From 2783820f98b02f01dd2d55aca1e05fc3e78a617d Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 3 May 2018 12:23:24 +0100 Subject: [PATCH 001/123] Initial draft of GET media/limits --- api/client-server/content-repo.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 03a25fd97..6e3ba273c 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -269,3 +269,30 @@ paths: "$ref": "definitions/error.yaml" tags: - Media + "/limits": + get: + summary: Get limits on what can be uploaded to the content repository. + Clients SHOULD use this as a guide when uploading content. + All values are intentionally left optional, the client MUST assume + that any field not given is not limited. + + **NOTE:** Reverse proxies may apply their own limits. + operationId: getLimits + produces: ["application/json"] + security: + - accessToken: [] + responses: + 200: + description: The limits exposed by the matrix server. + schema: + type: object + properties: + size: + type: number + description: "The maximum size a upload can be in bytes." + examples: + application/json: { + "size": 50000000 + } + tags: + - Media From c9abf36097f1f4a3a2086c6629e9ab8112c035d2 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 3 May 2018 16:58:22 +0100 Subject: [PATCH 002/123] size > upload_size --- api/client-server/content-repo.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 6e3ba273c..cfcb00dd4 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -287,12 +287,12 @@ paths: schema: type: object properties: - size: + upload_size: type: number description: "The maximum size a upload can be in bytes." examples: application/json: { - "size": 50000000 + "upload_size": 50000000 } tags: - Media From de6fe1b76e325380eef7110364c0e11ac2d014d2 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 3 May 2018 18:00:50 +0100 Subject: [PATCH 003/123] Add note about auth --- api/client-server/content-repo.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index cfcb00dd4..97a7d8f95 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -277,6 +277,9 @@ paths: that any field not given is not limited. **NOTE:** Reverse proxies may apply their own limits. + + If auth is not supplied, this endpoint gives the global limit of the server. + Otherwise it should give the limits applied to the authenticated user. operationId: getLimits produces: ["application/json"] security: From ffc8ee298adfc90a2b21801636927cbcff78b969 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 8 May 2018 11:50:34 +0100 Subject: [PATCH 004/123] a upload > an upload --- api/client-server/content-repo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 97a7d8f95..cf7b5f25d 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -292,7 +292,7 @@ paths: properties: upload_size: type: number - description: "The maximum size a upload can be in bytes." + description: "The maximum size an upload can be in bytes." examples: application/json: { "upload_size": 50000000 From c3f66e18147d4139ebc02ffae64cadc5217211a6 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Mon, 11 Jun 2018 12:40:04 +0100 Subject: [PATCH 005/123] basic spec for group IDs, mostly copied from User IDs as they share grammar --- .../appendices/identifier_grammar.rst | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/specification/appendices/identifier_grammar.rst b/specification/appendices/identifier_grammar.rst index e85bf4107..9487374b3 100644 --- a/specification/appendices/identifier_grammar.rst +++ b/specification/appendices/identifier_grammar.rst @@ -60,6 +60,7 @@ The sigil characters are as follows: * ``@``: User ID * ``!``: Room ID * ``$``: Event ID +* ``+``: Group ID * ``#``: Room alias The precise grammar defining the allowable format of an identifier depends on @@ -207,6 +208,35 @@ readable. .. TODO-spec What is the grammar for the opaque part? https://matrix.org/jira/browse/SPEC-389 + +Group Identifiers ++++++++++++++++++ + +Groups within Matrix are uniquely identified by their group ID. The group +ID is namespaced to the group server which hosts this group and has the +form:: + + +localpart:domain + +The ``localpart`` of a group ID is an opaque identifier for that group. It MUST +NOT be empty, and MUST contain only the characters ``a-z``, ``0-9``, ``.``, +``_``, ``=``, ``-``, and ``/``. + +The ``domain`` of a group ID is the `server name`_ of the group server which +hosts this group. + +The length of a group ID, including the ``+`` sigil and the domain, MUST NOT +exceed 255 characters. + +The complete grammar for a legal group ID is:: + + group_id = "+" group_id_localpart ":" server_name + group_id_localpart = 1*group_id_char + group_id_char = DIGIT + / %x61-7A ; a-z + / "-" / "." / "=" / "_" / "/" + + Room Aliases ++++++++++++ From 077cd04a6d42b852f0dd8e29ad26ffda843e48ef Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 20 Jun 2018 16:01:06 +0100 Subject: [PATCH 006/123] /limits => /config --- api/client-server/content-repo.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index cf7b5f25d..3c2003444 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -269,9 +269,9 @@ paths: "$ref": "definitions/error.yaml" tags: - Media - "/limits": + "/config": get: - summary: Get limits on what can be uploaded to the content repository. + summary: Get the config for the media repository. Clients SHOULD use this as a guide when uploading content. All values are intentionally left optional, the client MUST assume that any field not given is not limited. From 6b9640bcb499e95cb87ca6784b2b68e5bdf73d7a Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 20 Jun 2018 17:06:17 +0100 Subject: [PATCH 007/123] upload_size => m.upload.size --- api/client-server/content-repo.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 3c2003444..b86b6596b 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -290,12 +290,12 @@ paths: schema: type: object properties: - upload_size: + m.upload.size: type: number description: "The maximum size an upload can be in bytes." examples: application/json: { - "upload_size": 50000000 + "m.upload.size": 50000000 } tags: - Media From 6d0a56d190cddab7b0e710f6e1c43b04392088e0 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 22 Jun 2018 09:38:53 +0100 Subject: [PATCH 008/123] Suggestions by Travis tend to be sensible ones --- api/client-server/content-repo.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index b86b6596b..4a61ccec2 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -286,7 +286,7 @@ paths: - accessToken: [] responses: 200: - description: The limits exposed by the matrix server. + description: The public content repository configuration for the matrix server. schema: type: object properties: @@ -295,7 +295,7 @@ paths: description: "The maximum size an upload can be in bytes." examples: application/json: { - "m.upload.size": 50000000 - } + "m.upload.size": 50000000 + } tags: - Media From 20200264e1ba2a31aa45b5ed67f38f1403577546 Mon Sep 17 00:00:00 2001 From: Remi Rampin Date: Sat, 23 Jun 2018 11:11:38 -0400 Subject: [PATCH 009/123] Consolidate backwards-compatible client changes --- changelogs/client_server.rst | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index feabecab5..f375fc97d 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -15,6 +15,22 @@ Unreleased changes - Sticker messages: - Add sticker message event definition. (`#1158 `_). + - Add 'token' parameter to /keys/query endpoint + (`#1104 `_). + - Add the room visibility options for the room directory + (`#1141 `_). + - Add spec for ignoring users + (`#1142 `_). + - Add the ``/register/available`` endpoint for username availability + (`#1151 `_). + - Add ``allow_remote`` to the content repo to avoid routing loops + (`#1265 `_). + - Add report content API + (`#1264 `_). + - Document ``/logout/all`` endpoint + (`#1263 `_). + - Document `highlights` field in /search response + (`#1274 `_). - Spec clarifications: @@ -40,25 +56,6 @@ Unreleased changes - Describe ``StateEvent`` for ``/createRoom`` (`#1329 `_). -- Changes to the API which will be backwards-compatible for clients: - - - Add 'token' parameter to /keys/query endpoint - (`#1104 `_). - - Add the room visibility options for the room directory - (`#1141 `_). - - Add spec for ignoring users - (`#1142 `_). - - Add the ``/register/available`` endpoint for username availability - (`#1151 `_). - - Add ``allow_remote`` to the content repo to avoid routing loops - (`#1265 `_). - - Add report content API - (`#1264 `_). - - Document ``/logout/all`` endpoint - (`#1263 `_). - - Document `highlights` field in /search response - (`#1274 `_). - r0.3.0 ====== From ddc152347bfa6c472ffc30d508b0a2fdbd3e8cb6 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 27 Jun 2018 20:14:29 +0100 Subject: [PATCH 010/123] Extending words and rate limiting --- api/client-server/content-repo.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 4a61ccec2..57585ce88 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -271,16 +271,17 @@ paths: - Media "/config": get: - summary: Get the config for the media repository. + summary: Get the configuration for the media repository. Clients SHOULD use this as a guide when uploading content. All values are intentionally left optional, the client MUST assume that any field not given is not limited. - **NOTE:** Reverse proxies may apply their own limits. + **NOTE:** Reverse proxies may apply their own configuration. + - If auth is not supplied, this endpoint gives the global limit of the server. - Otherwise it should give the limits applied to the authenticated user. - operationId: getLimits + If an accessToken is supplied, the configuration applied to the authenticated user. + Otherwise it should give the configuration applied globally to the server. + operationId: getConfig produces: ["application/json"] security: - accessToken: [] @@ -297,5 +298,10 @@ paths: application/json: { "m.upload.size": 50000000 } + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + tags: - Media From 80935eadcf87f6dd8e1a3669e5519844e77e34fb Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 28 Jun 2018 00:14:42 +0100 Subject: [PATCH 011/123] Update content-repo.yaml --- api/client-server/content-repo.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 57585ce88..c80961386 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -273,13 +273,13 @@ paths: get: summary: Get the configuration for the media repository. Clients SHOULD use this as a guide when uploading content. - All values are intentionally left optional, the client MUST assume - that any field not given is not limited. + All values are intentionally left optional. Clients SHOULD follow + the advise given in the field description when the field is not avaliable. **NOTE:** Reverse proxies may apply their own configuration. - If an accessToken is supplied, the configuration applied to the authenticated user. + If an accessToken is supplied, the configuration applied to the authenticated user is returned. Otherwise it should give the configuration applied globally to the server. operationId: getConfig produces: ["application/json"] @@ -293,7 +293,9 @@ paths: properties: m.upload.size: type: number - description: "The maximum size an upload can be in bytes." + description: |- + The maximum size an upload can be in bytes. If not listed or null, + the upload limit should be treated as unknown. examples: application/json: { "m.upload.size": 50000000 From eeaf4385d73f221e896917432ec6ed336bbabeaf Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 28 Jun 2018 09:04:01 +0100 Subject: [PATCH 012/123] Speling is my weakniss --- api/client-server/content-repo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index c80961386..be003a2b2 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -274,7 +274,7 @@ paths: summary: Get the configuration for the media repository. Clients SHOULD use this as a guide when uploading content. All values are intentionally left optional. Clients SHOULD follow - the advise given in the field description when the field is not avaliable. + the advice given in the field description when the field is not available. **NOTE:** Reverse proxies may apply their own configuration. From 091b2a677190c1a120dd31c2ef4e4c0bec6469e6 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 28 Jun 2018 16:42:36 +0100 Subject: [PATCH 013/123] Hard limit on requiring accessToken. Rephrased some sections --- api/client-server/content-repo.yaml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index be003a2b2..b48068f23 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -272,15 +272,13 @@ paths: "/config": get: summary: Get the configuration for the media repository. - Clients SHOULD use this as a guide when uploading content. + Clients SHOULD use this as a guide when using media endpoints. All values are intentionally left optional. Clients SHOULD follow the advice given in the field description when the field is not available. - **NOTE:** Reverse proxies may apply their own configuration. - - - If an accessToken is supplied, the configuration applied to the authenticated user is returned. - Otherwise it should give the configuration applied globally to the server. + **NOTE:** The /config endpoint is a guide. Other middleware such as + reverse proxies may apply their own configuration not described on this + endpoint. operationId: getConfig produces: ["application/json"] security: @@ -294,8 +292,9 @@ paths: m.upload.size: type: number description: |- - The maximum size an upload can be in bytes. If not listed or null, - the upload limit should be treated as unknown. + The maximum size an upload can be in bytes. + Clients SHOULD use this as a guide when uploading content. + If not listed or null, the size limit should be treated as unknown. examples: application/json: { "m.upload.size": 50000000 From 1820df02d9faebf042bb3cf81e24d2036e844f74 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 28 Jun 2018 17:07:44 +0100 Subject: [PATCH 014/123] media => content repository --- api/client-server/content-repo.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index b48068f23..582d06956 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -271,8 +271,8 @@ paths: - Media "/config": get: - summary: Get the configuration for the media repository. - Clients SHOULD use this as a guide when using media endpoints. + summary: Get the configuration for the content repository. + Clients SHOULD use this as a guide when using content endpoints. All values are intentionally left optional. Clients SHOULD follow the advice given in the field description when the field is not available. From 21e957edfba242107eb871a960fc1af334b8ce49 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 28 Jun 2018 17:23:57 +0100 Subject: [PATCH 015/123] Better note for /config --- api/client-server/content-repo.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 582d06956..8ccc7684e 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -276,9 +276,11 @@ paths: All values are intentionally left optional. Clients SHOULD follow the advice given in the field description when the field is not available. - **NOTE:** The /config endpoint is a guide. Other middleware such as - reverse proxies may apply their own configuration not described on this - endpoint. + **NOTE:** Both clients and server administrators should be aware that proxies + between the client and the server may affect the apparent behaviour of content + repository APIs, for example, proxies may enforce a lower upload size limit + than is advertised by the server on this endpoint. Clients should handle such + situations gracefully. operationId: getConfig produces: ["application/json"] security: From 41c18bed0f954e43e65993b7487a573bf65152f0 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 28 Jun 2018 17:53:57 +0100 Subject: [PATCH 016/123] +repository --- api/client-server/content-repo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 8ccc7684e..e2549c658 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -272,7 +272,7 @@ paths: "/config": get: summary: Get the configuration for the content repository. - Clients SHOULD use this as a guide when using content endpoints. + Clients SHOULD use this as a guide when using content repository endpoints. All values are intentionally left optional. Clients SHOULD follow the advice given in the field description when the field is not available. From a1309d636c8af90ef04499c392ce836fb07d614f Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 29 Jun 2018 09:58:07 +0100 Subject: [PATCH 017/123] Remove "Clients should handle gracefully" sentence --- api/client-server/content-repo.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index e2549c658..0f4ef2da6 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -279,8 +279,7 @@ paths: **NOTE:** Both clients and server administrators should be aware that proxies between the client and the server may affect the apparent behaviour of content repository APIs, for example, proxies may enforce a lower upload size limit - than is advertised by the server on this endpoint. Clients should handle such - situations gracefully. + than is advertised by the server on this endpoint. operationId: getConfig produces: ["application/json"] security: From d73b247688606d767667c5e80a1b35c69513c566 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Jun 2018 10:57:35 +0100 Subject: [PATCH 018/123] 3PE lookup service --- .../application_service.yaml | 391 +++++++++++++++++- changelogs/client_server.rst | 12 + specification/targets.yaml | 1 + 3 files changed, 401 insertions(+), 3 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index c39ce198f..a0713d45f 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -1,4 +1,5 @@ # Copyright 2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -91,7 +92,13 @@ paths: } schema: type: object - + 500: + description: There was a problem completing the request. + examples: + application/json: { + } + schema: + type: object "/rooms/{roomAlias}": get: summary: Query if a room alias should exist on the application service. @@ -147,7 +154,14 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + "errcode": "M_NOT_FOUND" + } + schema: + type: object + 500: + description: There was a problem completing the request. + examples: + application/json: { } schema: type: object @@ -203,7 +217,378 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + "errcode": "M_NOT_FOUND" + } + schema: + type: object + 500: + description: There was a problem completing the request. + examples: + application/json: { + } + schema: + type: object + "/_matrix/app/unstable/thirdparty/{protocol}": + get: + summary: Retrieve metadata about a specific protocol that the application service supports. + description: |- + This API is called by the HS when it wants to present clients with + specific information about the various thirdparty networks that an AS + supports. + operationId: queryMetadata + parameters: + - in: path + name: protocol + type: string + description: |- + The name of the protocol. + required: true + x-example: "irc" + responses: + 200: + description: The protocol was found and metadata returned. + examples: + application/json: { + "user_fields": ["network", "nickname"], + "location_fields": ["network", "channel"], + "icon": "mxc://example.org/aBcDeFgH", + "field_types": { + "network": { + "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", + "placeholder": "irc.example.org" + }, + "nickname": { + "regexp": "[^\\s]+", + "placeholder": "username" + }, + "channel": { + "regexp": "#[^\\s]+", + "placeholder": "#foobar" + } + }, + "instances": [ + { + "desc": "Freenode", + "icon": "mxc://example.org/JkLmNoPq", + "fields": { + "network": "freenode.net", + } + } + ] + } + schema: + type: object + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + type: object + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: { + "errcode": "M_FORBIDDEN" + } + schema: + type: object + 404: + description: No protocol was found with the given path. + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + 500: + description: There was a problem completing the request. + examples: + application/json: { + } + schema: + type: object + "/_matrix/app/unstable/thirdparty/user/{protocol}": + get: + summary: Retrieve the Matrix ID of a corresponding thirdparty user. + description: |- + This API is called by the HS in order to retrieve a Matrix ID linked + to a user on the external service, given a set of user parameters. + operationId: queryUserByProtocol + parameters: + - in: path + name: protocol + type: string + description: |- + The name of the protocol. + required: true + x-example: irc + - in: query + name: field1, field2... + type: string + description: |- + One or more custom fields that are passed to the AS to help identify the user. + responses: + 200: + description: The Matrix IDs found with the given parameters. + examples: + application/json: [{ + "userid": "@_gitter_jim:matrix.org", + "protocol": "gitter", + "fields": { + "user": "jim" + } + }] + schema: + type: array + description: Matched users. + items: + type: object + title: User + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + type: object + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: { + "errcode": "M_FORBIDDEN" + } + schema: + type: object + 404: + description: No users were found with the given parameters. + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + 500: + description: There was a problem completing the request. + examples: + application/json: { + } + schema: + type: object + "/_matrix/app/unstable/thirdparty/location/{protocol}": + get: + summary: Retreive Matrix-side portals rooms leading to a thirdparty location. + description: |- + Requesting this endpoint with a valid protocol name results in a list + of successful mapping results in a JSON array. Each result contains + objects to represent the Matrix room or rooms that represent a portal + to this 3PN. Each has the Matrix room alias string, an identifier for + the particular 3PN protocol, and an object containing the + network-specific fields that comprise this identifier. It should + attempt to canonicalise the identifier as much as reasonably possible + given the network type. + operationId: queryLocationByProtocol + parameters: + - in: path + name: protocol + type: string + description: The protocol used to communicate to the thirdparty network. + required: true + x-example: "irc" + - in: query + name: field1, field2... + type: string + description: |- + One or more custom fields that are passed to the AS to help + identify the thirdparty location. + responses: + 200: + description: At least one portal room was found. + examples: + application/json: [{ + "alias": "#freenode_#matrix:matrix.org", + "protocol": "irc", + "fields": { + "network": "freenode", + "channel": "#matrix" + } + }] + schema: + type: array + description: |- + Array of portal rooms leading to the requested thirdparty + location. + items: + type: object + title: Portal Room + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + type: object + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: { + "errcode": "M_FORBIDDEN" } schema: type: object + 404: + description: No mappings were found with the given parameters. + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + 500: + description: There was a problem completing the request. + examples: + application/json: { + } + schema: + type: object + "/_matrix/app/unstable/thirdparty/location": + get: + summary: Reverse-lookup thirdparty locations given a Matrix room alias. + description: |- + Retreive an array of thirdparty network locations from a Matrix room + alias. + operationId: queryLocationByAlias + parameters: + - in: path + name: alias + type: string + description: The Matrix room alias to look up. + responses: + 200: + description: |- + All found thirdparty locations. Same response format as the + forward lookup response. + examples: + application/json: [{ + "alias": "#freenode_#matrix:matrix.org", + "protocol": "irc", + "fields": { + "network": "freenode", + "channel": "#matrix" + } + }] + schema: + type: array + description: Matched thirdparty locations. + items: + type: object + title: Location + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + type: object + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: { + "errcode": "M_FORBIDDEN" + } + schema: + type: object + 404: + description: No mappings were found with the given parameters. + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + 500: + description: There was a problem completing the request. + examples: + application/json: { + } + schema: + type: object + "/_matrix/app/unstable/thirdparty/user": + get: + summary: Reverse-lookup thirdparty users given a Matrix ID. + description: |- + Retreive an array of thirdparty users from a Matrix ID. + operationId: queryUserByID + paramters: + - in: path + name: userid + type: string + description: The Matrix ID to look up. + responses: + 200: + description: |- + An array of thirdparty users. + examples: + application/json: [{ + "userid": "@_gitter_jim:matrix.org", + "protocol": "gitter", + "fields": { + "user": "jim" + } + }] + schema: + type: array + description: Matched thirdparty users + items: + type: object + title: User + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + type: object + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: { + "errcode": "M_FORBIDDEN" + } + schema: + type: object + 404: + description: No mappings were found with the given parameters. + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + 500: + description: There was a problem completing the request. + examples: + application/json: { + } + schema: + type: object \ No newline at end of file diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index feabecab5..f7e94dbe4 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -11,6 +11,18 @@ Unreleased changes (`#1110 `_). - ``POST /delete_devices`` (`#1239 `_). + - ``GET /thirdparty/protocols`` + (`#1353 `_). + - ``GET /thirdparty/protocol/{protocol}`` + (`#1353 `_). + - ``GET /thirdparty/location/{protocol}`` + (`#1353 `_). + - ``GET /thirdparty/user/{protocol}`` + (`#1353 `_). + - ``GET /thirdparty/location`` + (`#1353 `_). + - ``GET /thirdparty/user`` + (`#1353 `_). - Sticker messages: - Add sticker message event definition. diff --git a/specification/targets.yaml b/specification/targets.yaml index bc9d94c8f..c6c9cd234 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -12,6 +12,7 @@ targets: - { 1: modules.rst } - { 2: feature_profiles.rst } - { 2: "group:modules" } # reference a group of files + - { 1: thirdparty_lookup.rst } version_label: "%CLIENT_RELEASE_LABEL%" application_service: files: From 7d94aaace8a799927673d53fa49b5f28c7d83acd Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 14:59:41 -0600 Subject: [PATCH 019/123] Explicitly declare that the kick/ban reason will be on the membership event Fixes https://github.com/matrix-org/matrix-doc/issues/798 --- api/client-server/banning.yaml | 3 ++- api/client-server/kicking.yaml | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/api/client-server/banning.yaml b/api/client-server/banning.yaml index 3877f0e90..4030f46b1 100644 --- a/api/client-server/banning.yaml +++ b/api/client-server/banning.yaml @@ -61,7 +61,8 @@ paths: description: The fully qualified user ID of the user being banned. reason: type: string - description: The reason the user has been banned. + description: The reason the user has been banned. This will be supplied as the + ``reason`` on the target's updated `m.room.member`_ event. required: ["user_id"] responses: 200: diff --git a/api/client-server/kicking.yaml b/api/client-server/kicking.yaml index ef43c0ee9..bacc806ee 100644 --- a/api/client-server/kicking.yaml +++ b/api/client-server/kicking.yaml @@ -34,6 +34,10 @@ paths: Kick a user from the room. The caller must have the required power level in order to perform this operation. + + Kicking a user adjusts the target member's membership state to be ``leave`` with an + optional ``reason``. Like with other membership changes, a user can directly adjust + the target member's state by making a request to ``/rooms//state/m.room.member/``. operationId: kick security: - accessToken: [] @@ -59,7 +63,9 @@ paths: description: The fully qualified user ID of the user being kicked. reason: type: string - description: The reason the user has been kicked. + description: |- + The reason the user has been kicked. This will be supplied as the + ``reason`` on the target's updated `m.room.member`_ event. required: ["user_id"] responses: 200: From 5b6ad6f39d5792c987b4bee60989b8a2b026f9b3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 15:19:33 -0600 Subject: [PATCH 020/123] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index feabecab5..a16b96ae0 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -39,6 +39,8 @@ Unreleased changes (`#1245 `_). - Describe ``StateEvent`` for ``/createRoom`` (`#1329 `_). + - Describe how the ``reason`` is handled for kicks/bans + (`#1362 `_). - Changes to the API which will be backwards-compatible for clients: From d279fdafa6e649f4dd8ea2dd4a89e559c633a3f3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 15:49:16 -0600 Subject: [PATCH 021/123] Document the server_name query parameter on /join/{roomIdOrAlias} Fixes https://github.com/matrix-org/matrix-doc/issues/904 --- api/client-server/joining.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/api/client-server/joining.yaml b/api/client-server/joining.yaml index da4e43371..bd3e6885f 100644 --- a/api/client-server/joining.yaml +++ b/api/client-server/joining.yaml @@ -143,6 +143,16 @@ paths: description: The room identifier or alias to join. required: true x-example: "#monkeys:matrix.org" + - in: query + type: array + items: + type: string + explode: true + name: server_name + description: |- + The servers to attempt to join the room through. One of the servers + must be participating in the room. + x-example: ["matrix.org", "elsewhere.ca"] - in: body name: third_party_signed schema: From cbbdcbcf01cd5c9d18bb0ca0aa80257dad8fccfe Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 15:50:01 -0600 Subject: [PATCH 022/123] Show arrays in the query string as "[type]" rather than "array" Otherwise the question "array of what?" is raised. --- scripts/templating/matrix_templates/units.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index d8cfe1e34..dac183d1f 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -502,6 +502,11 @@ class MatrixUnits(Units): # assign value expected for this param val_type = param.get("type") # integer/string + if val_type == "array": + items = param.get("items") + if items: + val_type = "[%s]" % items.get("type") + if param.get("enum"): val_type = "enum" desc += ( From 625913d797cda10d9916934351e598161adaced0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 15:53:43 -0600 Subject: [PATCH 023/123] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index feabecab5..01670c744 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -15,6 +15,8 @@ Unreleased changes - Sticker messages: - Add sticker message event definition. (`#1158 `_). + - Document the ``server_name`` parameter on ``/join/{roomIdOrAlias}`` + (`#1364 `_). - Spec clarifications: From 4ca54404fa4ae3b2ae58c34688204883e4c94903 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 16:15:29 -0600 Subject: [PATCH 024/123] Document the CORS/preflight headers Fixes https://github.com/matrix-org/matrix-doc/issues/1006 --- specification/client_server_api.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index dec3a4f44..f7a7d509e 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -164,6 +164,26 @@ recommended. {{versions_cs_http_api}} +Web Browser Clients +------------------- + +It is realistic to expect that some clients will be written to be run within a +web browser or similar environment. In these cases, the homeserver should respond +to pre-flight requests and supply Cross-Origin Resource Sharing (CORS) headers. + +When a client approaches the server with a pre-flight (``OPTIONS``) request, the +server should respond with the CORS headers for that route. If the route does not +exist, the server should return an ``M_NOT_FOUND`` error with a 404 status code. + +The standard CORS headers to be returned by servers on all requests are: + +.. code:: + + Access-Control-Allow-Origin: * + Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS + Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, Authorization + + Client Authentication --------------------- From b8c8d859268874cd548aff3be9fe58eed50a15a6 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 16:18:04 -0600 Subject: [PATCH 025/123] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index feabecab5..e4e7552f3 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -15,6 +15,8 @@ Unreleased changes - Sticker messages: - Add sticker message event definition. (`#1158 `_). + - Document the CORS/preflight headers + (`#1365 `_). - Spec clarifications: From a6b0125da62e9c911ff15b8cbcf267499f95ee99 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 16:19:31 -0600 Subject: [PATCH 026/123] Our swagger doesn't support explode yet --- api/client-server/joining.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/api/client-server/joining.yaml b/api/client-server/joining.yaml index bd3e6885f..471c50382 100644 --- a/api/client-server/joining.yaml +++ b/api/client-server/joining.yaml @@ -147,7 +147,6 @@ paths: type: array items: type: string - explode: true name: server_name description: |- The servers to attempt to join the room through. One of the servers From 2fc292646147b236a3740184a589f457206b3db5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 11:33:51 -0600 Subject: [PATCH 027/123] Clearly say that the server/server API is unstable and subject to change --- specification/server_server_api.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 3738ea053..f2b3204ab 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -16,6 +16,10 @@ Federation API ============== +.. WARNING:: + This API is unstable and will change without warning or discussion while + we work towards a r0 release (scheduled for August 2018). + Matrix homeservers use the Federation APIs (also known as server-server APIs) to communicate with each other. Homeservers use these APIs to push messages to each other in real-time, to From 1f12637d3f8973301e0ec6b7ebdfb167603b7d49 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 12:23:51 -0600 Subject: [PATCH 028/123] Getting presence requires authentication --- api/client-server/presence.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/client-server/presence.yaml b/api/client-server/presence.yaml index ba202c2da..ab32f9833 100644 --- a/api/client-server/presence.yaml +++ b/api/client-server/presence.yaml @@ -83,6 +83,8 @@ paths: description: |- Get the given user's presence state. operationId: getPresence + security: + - accessToken: [] parameters: - in: path type: string From b61fe201718ca121c62a8edde9093bb2c0611bf7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 13:02:32 -0600 Subject: [PATCH 029/123] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 44d41f4e4..eb0387c11 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -39,6 +39,8 @@ Unreleased changes (`#1152 `_). - Mark ``GET /rooms/{roomId}/members`` as requiring authentication (`#1245 `_). + - Mark ``GET /presence/{userId}/status`` as requiring authentication + (`#1371 `_). - Describe ``StateEvent`` for ``/createRoom`` (`#1329 `_). From 7cd22f5690b50cc4132328e3ba030b49f117e83c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 13:45:34 -0600 Subject: [PATCH 030/123] Describe the rate limit error everywhere Fixes https://github.com/matrix-org/matrix-doc/issues/1153 --- api/client-server/content-repo.yaml | 10 +++---- .../definitions/{ => errors}/error.yaml | 2 ++ .../definitions/errors/rate_limited.yaml | 26 +++++++++++++++++++ api/client-server/inviting.yaml | 2 +- api/client-server/joining.yaml | 4 +-- api/client-server/leaving.yaml | 4 +-- api/client-server/login.yaml | 4 +-- api/client-server/presence.yaml | 4 +-- api/client-server/profile.yaml | 4 +-- api/client-server/pusher.yaml | 2 +- api/client-server/pushrules.yaml | 2 +- api/client-server/receipts.yaml | 2 +- api/client-server/registration.yaml | 8 +++--- api/client-server/search.yaml | 2 +- api/client-server/third_party_membership.yaml | 2 +- api/client-server/typing.yaml | 2 +- api/client-server/users.yaml | 2 +- api/client-server/voip.yaml | 2 +- api/client-server/whoami.yaml | 2 +- 19 files changed, 57 insertions(+), 29 deletions(-) rename api/client-server/definitions/{ => errors}/error.yaml (90%) create mode 100644 api/client-server/definitions/errors/rate_limited.yaml diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index f76b2fb95..7ded75d0f 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -72,7 +72,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Media "/download/{serverName}/{mediaId}": @@ -119,7 +119,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Media "/download/{serverName}/{mediaId}/{fileName}": @@ -173,7 +173,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Media "/thumbnail/{serverName}/{mediaId}": @@ -238,7 +238,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Media "/preview_url": @@ -293,6 +293,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Media diff --git a/api/client-server/definitions/error.yaml b/api/client-server/definitions/errors/error.yaml similarity index 90% rename from api/client-server/definitions/error.yaml rename to api/client-server/definitions/errors/error.yaml index fa5cada7a..7471da6f6 100644 --- a/api/client-server/definitions/error.yaml +++ b/api/client-server/definitions/errors/error.yaml @@ -17,7 +17,9 @@ properties: errcode: type: string description: An error code. + example: M_UNKNOWN error: type: string description: A human-readable error message. + example: An unknown error occurred required: ["errcode"] \ No newline at end of file diff --git a/api/client-server/definitions/errors/rate_limited.yaml b/api/client-server/definitions/errors/rate_limited.yaml new file mode 100644 index 000000000..6b4932ee6 --- /dev/null +++ b/api/client-server/definitions/errors/rate_limited.yaml @@ -0,0 +1,26 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +$ref: error.yaml +type: object +description: The rate limit was reached for this request +properties: + errcode: + type: string + description: The M_LIMIT_EXCEEDED error code + example: M_LIMIT_EXCEEDED + error: + type: string + description: A human-readable error message. + example: Too many requests +required: ["errcode"] \ No newline at end of file diff --git a/api/client-server/inviting.yaml b/api/client-server/inviting.yaml index 47f51bd47..1e6694dc8 100644 --- a/api/client-server/inviting.yaml +++ b/api/client-server/inviting.yaml @@ -96,6 +96,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Room membership diff --git a/api/client-server/joining.yaml b/api/client-server/joining.yaml index da4e43371..9890e73ca 100644 --- a/api/client-server/joining.yaml +++ b/api/client-server/joining.yaml @@ -113,7 +113,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Room membership "/join/{roomIdOrAlias}": @@ -209,6 +209,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Room membership diff --git a/api/client-server/leaving.yaml b/api/client-server/leaving.yaml index 36351fd44..1a2515a6c 100644 --- a/api/client-server/leaving.yaml +++ b/api/client-server/leaving.yaml @@ -64,7 +64,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Room membership "/rooms/{roomId}/forget": @@ -102,6 +102,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Room membership diff --git a/api/client-server/login.yaml b/api/client-server/login.yaml index c0db2299b..e64ff953d 100644 --- a/api/client-server/login.yaml +++ b/api/client-server/login.yaml @@ -61,7 +61,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Session management post: @@ -176,6 +176,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Session management diff --git a/api/client-server/presence.yaml b/api/client-server/presence.yaml index ba202c2da..2a97e1bae 100644 --- a/api/client-server/presence.yaml +++ b/api/client-server/presence.yaml @@ -75,7 +75,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Presence get: @@ -176,7 +176,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Presence get: diff --git a/api/client-server/profile.yaml b/api/client-server/profile.yaml index 0cada0cad..c8dc40563 100644 --- a/api/client-server/profile.yaml +++ b/api/client-server/profile.yaml @@ -67,7 +67,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - User data get: @@ -141,7 +141,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - User data get: diff --git a/api/client-server/pusher.yaml b/api/client-server/pusher.yaml index 9cf40a061..826b69d1f 100644 --- a/api/client-server/pusher.yaml +++ b/api/client-server/pusher.yaml @@ -232,6 +232,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Push notifications diff --git a/api/client-server/pushrules.yaml b/api/client-server/pushrules.yaml index 801349efa..22738c3e3 100644 --- a/api/client-server/pushrules.yaml +++ b/api/client-server/pushrules.yaml @@ -442,7 +442,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Push notifications "/pushrules/{scope}/{kind}/{ruleId}/enabled": diff --git a/api/client-server/receipts.yaml b/api/client-server/receipts.yaml index e46359a99..a3e9789e9 100644 --- a/api/client-server/receipts.yaml +++ b/api/client-server/receipts.yaml @@ -76,6 +76,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Room participation diff --git a/api/client-server/registration.yaml b/api/client-server/registration.yaml index 1c544afed..5c2059996 100644 --- a/api/client-server/registration.yaml +++ b/api/client-server/registration.yaml @@ -185,7 +185,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - User data "/register/email/requestToken": @@ -296,7 +296,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - User data "/account/password/email/requestToken": @@ -363,7 +363,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - User data "/register/available": @@ -423,6 +423,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - User data diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index e5f6a872b..3dcd719fa 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -355,6 +355,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Search diff --git a/api/client-server/third_party_membership.yaml b/api/client-server/third_party_membership.yaml index 612b22d05..3050ee3a9 100644 --- a/api/client-server/third_party_membership.yaml +++ b/api/client-server/third_party_membership.yaml @@ -129,6 +129,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Room membership diff --git a/api/client-server/typing.yaml b/api/client-server/typing.yaml index e2a8f9bd1..e7cbe2d7d 100644 --- a/api/client-server/typing.yaml +++ b/api/client-server/typing.yaml @@ -82,6 +82,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Room participation diff --git a/api/client-server/users.yaml b/api/client-server/users.yaml index 1734e3bb8..a682b4358 100644 --- a/api/client-server/users.yaml +++ b/api/client-server/users.yaml @@ -95,6 +95,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - User data diff --git a/api/client-server/voip.yaml b/api/client-server/voip.yaml index 6d6136a54..75ace4c3a 100644 --- a/api/client-server/voip.yaml +++ b/api/client-server/voip.yaml @@ -73,6 +73,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - VOIP diff --git a/api/client-server/whoami.yaml b/api/client-server/whoami.yaml index 8f5abdf79..4d178e073 100644 --- a/api/client-server/whoami.yaml +++ b/api/client-server/whoami.yaml @@ -79,6 +79,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - User data From caa6d4d9ad082003cb4c481795327c5cc22569c3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 13:51:17 -0600 Subject: [PATCH 031/123] Give all errors a schema reference This just helps keep an overall structure --- api/client-server/administrative_contact.yaml | 2 ++ api/client-server/banning.yaml | 4 ++++ api/client-server/create_room.yaml | 2 ++ api/client-server/directory.yaml | 4 ++++ api/client-server/inviting.yaml | 2 ++ api/client-server/joining.yaml | 4 ++++ api/client-server/kicking.yaml | 2 ++ api/client-server/list_public_rooms.yaml | 4 ++++ api/client-server/login.yaml | 4 ++++ api/client-server/pusher.yaml | 2 +- api/client-server/pushrules.yaml | 2 +- api/client-server/registration.yaml | 6 +++++- api/client-server/third_party_membership.yaml | 2 ++ api/client-server/whoami.yaml | 4 ++-- 14 files changed, 39 insertions(+), 5 deletions(-) diff --git a/api/client-server/administrative_contact.yaml b/api/client-server/administrative_contact.yaml index e7381a555..8f0319d5a 100644 --- a/api/client-server/administrative_contact.yaml +++ b/api/client-server/administrative_contact.yaml @@ -129,6 +129,8 @@ paths: "errcode": "M_THREEPID_AUTH_FAILED", "error": "The third party credentials could not be verified by the identity server." } + schema: + "$ref": "definitions/errors/error.yaml" tags: - User data "/account/3pid/email/requestToken": diff --git a/api/client-server/banning.yaml b/api/client-server/banning.yaml index 3877f0e90..9aceb2db8 100644 --- a/api/client-server/banning.yaml +++ b/api/client-server/banning.yaml @@ -82,6 +82,8 @@ paths: "errcode": "M_FORBIDDEN", "error": "You do not have a high enough power level to ban from this room." } + schema: + "$ref": "definitions/errors/error.yaml" tags: - Room membership "/rooms/{roomId}/unban": @@ -133,5 +135,7 @@ paths: "errcode": "M_FORBIDDEN", "error": "You do not have a high enough power level to unban from this room." } + schema: + "$ref": "definitions/errors/error.yaml" tags: - Room membership diff --git a/api/client-server/create_room.yaml b/api/client-server/create_room.yaml index 3e88ee0c0..59f7f75f0 100644 --- a/api/client-server/create_room.yaml +++ b/api/client-server/create_room.yaml @@ -218,6 +218,8 @@ paths: invalid: for example, the user's ``power_level`` is set below that necessary to set the room name (``errcode`` set to ``M_INVALID_ROOM_STATE``). + schema: + "$ref": "definitions/errors/error.yaml" tags: - Room creation diff --git a/api/client-server/directory.yaml b/api/client-server/directory.yaml index a50bab629..ee42cf845 100644 --- a/api/client-server/directory.yaml +++ b/api/client-server/directory.yaml @@ -68,6 +68,8 @@ paths: "errcode": "M_UNKNOWN", "error": "Room alias #monkeys:matrix.org already exists." } + schema: + "$ref": "definitions/errors/error.yaml" tags: - Room directory get: @@ -118,6 +120,8 @@ paths: "errcode": "M_NOT_FOUND", "error": "Room alias #monkeys:matrix.org not found." } + schema: + "$ref": "definitions/errors/error.yaml" tags: - Room directory delete: diff --git a/api/client-server/inviting.yaml b/api/client-server/inviting.yaml index 1e6694dc8..f312d5ceb 100644 --- a/api/client-server/inviting.yaml +++ b/api/client-server/inviting.yaml @@ -93,6 +93,8 @@ paths: examples: application/json: { "errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"} + schema: + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: diff --git a/api/client-server/joining.yaml b/api/client-server/joining.yaml index 9890e73ca..2a2a4c687 100644 --- a/api/client-server/joining.yaml +++ b/api/client-server/joining.yaml @@ -110,6 +110,8 @@ paths: examples: application/json: { "errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} + schema: + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: @@ -206,6 +208,8 @@ paths: examples: application/json: { "errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} + schema: + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: diff --git a/api/client-server/kicking.yaml b/api/client-server/kicking.yaml index ef43c0ee9..9be5dc79f 100644 --- a/api/client-server/kicking.yaml +++ b/api/client-server/kicking.yaml @@ -81,5 +81,7 @@ paths: "errcode": "M_FORBIDDEN", "error": "You do not have a high enough power level to kick from this room." } + schema: + "$ref": "definitions/errors/error.yaml" tags: - Room membership diff --git a/api/client-server/list_public_rooms.yaml b/api/client-server/list_public_rooms.yaml index 334d528c3..45034488a 100644 --- a/api/client-server/list_public_rooms.yaml +++ b/api/client-server/list_public_rooms.yaml @@ -59,6 +59,8 @@ paths: "errcode": "M_NOT_FOUND", "error": "Room not found" } + schema: + "$ref": "definitions/errors/error.yaml" put: summary: Sets the visibility of a room in the room directory description: |- @@ -107,6 +109,8 @@ paths: "errcode": "M_NOT_FOUND", "error": "Room not found" } + schema: + "$ref": "definitions/errors/error.yaml" "/publicRooms": get: summary: Lists the public rooms on the server. diff --git a/api/client-server/login.yaml b/api/client-server/login.yaml index e64ff953d..289b17e78 100644 --- a/api/client-server/login.yaml +++ b/api/client-server/login.yaml @@ -167,12 +167,16 @@ paths: "errcode": "M_UNKNOWN", "error": "Bad login type." } + schema: + "$ref": "definitions/errors/error.yaml" 403: description: |- The login attempt failed. For example, the password may have been incorrect. examples: application/json: { "errcode": "M_FORBIDDEN"} + schema: + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: diff --git a/api/client-server/pusher.yaml b/api/client-server/pusher.yaml index 826b69d1f..938014c60 100644 --- a/api/client-server/pusher.yaml +++ b/api/client-server/pusher.yaml @@ -228,7 +228,7 @@ paths: "errcode": "M_MISSING_PARAM" } schema: - type: object + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: diff --git a/api/client-server/pushrules.yaml b/api/client-server/pushrules.yaml index 22738c3e3..ceb9954b1 100644 --- a/api/client-server/pushrules.yaml +++ b/api/client-server/pushrules.yaml @@ -438,7 +438,7 @@ paths: "errcode": "M_UNKNOWN" } schema: - type: object + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: diff --git a/api/client-server/registration.yaml b/api/client-server/registration.yaml index 5c2059996..6ae4ddd37 100644 --- a/api/client-server/registration.yaml +++ b/api/client-server/registration.yaml @@ -177,6 +177,8 @@ paths: "errcode": "M_USER_IN_USE", "error": "Desired user ID is already taken." } + schema: + "$ref": "definitions/errors/error.yaml" 401: description: |- The homeserver requires additional authentication information. @@ -249,7 +251,7 @@ paths: "error": "The specified address is already in use" } schema: - type: object + "$ref": "definitions/errors/error.yaml" "/account/password": post: summary: "Changes a user's password." @@ -420,6 +422,8 @@ paths: "errcode": "M_USER_IN_USE", "error": "Desired user ID is already taken." } + schema: + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: diff --git a/api/client-server/third_party_membership.yaml b/api/client-server/third_party_membership.yaml index 3050ee3a9..66c14c4d3 100644 --- a/api/client-server/third_party_membership.yaml +++ b/api/client-server/third_party_membership.yaml @@ -126,6 +126,8 @@ paths: examples: application/json: { "errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"} + schema: + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: diff --git a/api/client-server/whoami.yaml b/api/client-server/whoami.yaml index 4d178e073..ad40eb869 100644 --- a/api/client-server/whoami.yaml +++ b/api/client-server/whoami.yaml @@ -65,7 +65,7 @@ paths: "error": "Unrecognised access token." } schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/error.yaml" 403: description: The appservice cannot masquerade as the user or has not registered them. @@ -75,7 +75,7 @@ paths: "error": "Application service has not registered this user." } schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: From 705423086af6ae273f8ce140f572b6e1cffa860b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 13:55:36 -0600 Subject: [PATCH 032/123] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 44d41f4e4..c06c46b32 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -41,6 +41,8 @@ Unreleased changes (`#1245 `_). - Describe ``StateEvent`` for ``/createRoom`` (`#1329 `_). + - Describe the rate limit error response schema + (`#1373 `_). - Changes to the API which will be backwards-compatible for clients: From 981d1004bc9d7a82e09ca413eee87a64d80d59f1 Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Thu, 3 May 2018 23:18:47 +0900 Subject: [PATCH 033/123] Add a missing $ref --- api/client-server/message_pagination.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/api/client-server/message_pagination.yaml b/api/client-server/message_pagination.yaml index 714066534..941e61fb9 100644 --- a/api/client-server/message_pagination.yaml +++ b/api/client-server/message_pagination.yaml @@ -107,6 +107,7 @@ paths: items: type: object title: RoomEvent + "$ref": "definitions/event-schemas/schema/core-event-schema/room_event.yaml" examples: application/json: { "start": "t47429-4392820_219380_26003_2265", From 0779d81e52556c28bed61b82c602fd154a8f184d Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 14:30:39 -0600 Subject: [PATCH 034/123] Clarify which requests should have CORS headers Spoilers: all of them. --- specification/client_server_api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index f7a7d509e..33ee8b1ac 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -169,7 +169,8 @@ Web Browser Clients It is realistic to expect that some clients will be written to be run within a web browser or similar environment. In these cases, the homeserver should respond -to pre-flight requests and supply Cross-Origin Resource Sharing (CORS) headers. +to pre-flight requests and supply Cross-Origin Resource Sharing (CORS) headers on +all requests. When a client approaches the server with a pre-flight (``OPTIONS``) request, the server should respond with the CORS headers for that route. If the route does not From 05c85fd840b3feda4916596365387e1e75cb1e8b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 14:37:21 -0600 Subject: [PATCH 035/123] Spec the 403 on GET /presence/{userId}/status --- api/client-server/presence.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/api/client-server/presence.yaml b/api/client-server/presence.yaml index ab32f9833..461344308 100644 --- a/api/client-server/presence.yaml +++ b/api/client-server/presence.yaml @@ -123,6 +123,15 @@ paths: description: |- There is no presence state for this user. This user may not exist or isn't exposing presence information to you. + 403: + description: You are not allowed to see this user's presence status. + examples: + application/json: { + "errcode": "M_FORBIDDEN", + "error": "You are not allowed to see their presence" + } + schema: + "$ref": "definitions/error.yaml" tags: - Presence "/presence/list/{userId}": From 9e796192e45aad16019f29d892cf875db7eeb7c5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 15:31:23 -0600 Subject: [PATCH 036/123] Document that clients must leave before forgetting rooms Fixes https://github.com/matrix-org/matrix-doc/issues/1011 --- api/client-server/leaving.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/leaving.yaml b/api/client-server/leaving.yaml index 36351fd44..e8e58397e 100644 --- a/api/client-server/leaving.yaml +++ b/api/client-server/leaving.yaml @@ -78,8 +78,8 @@ paths: for this room. If all users on a homeserver forget a room, the room is eligible for deletion from that homeserver. - If the user is currently joined to the room, they will implicitly leave - the room as part of this API call. + If the user is currently joined to the room, they must leave the room + before calling this API. operationId: forgetRoom security: - accessToken: [] From a19bbc80897fe0b4e184263200203b4b2e2c707c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 15:32:53 -0600 Subject: [PATCH 037/123] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 44d41f4e4..697b02c1a 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -41,6 +41,8 @@ Unreleased changes (`#1245 `_). - Describe ``StateEvent`` for ``/createRoom`` (`#1329 `_). + - Clarify that clients must leave rooms before forgetting them + (`#1378 `_). - Changes to the API which will be backwards-compatible for clients: From 9b1f3a1f4768fe153fa733e8db6d10b0c3c917e0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 18:16:24 -0600 Subject: [PATCH 038/123] Document guest_access in /createRoom presets; Make presets a table Previously the presets ended up being rendered as one long string (the indentation was not respected). This was hard to read and wasn't very clear. Now that there's a larger amount of information, it makes some sense to put it into a table. Tables don't work nicely when nested in another table, so the preset definitions have been moved into the general route description. This commit also updates the documentation to reference `preset`, not `presets` given the param is singular. Fixes https://github.com/matrix-org/matrix-doc/issues/881 --- api/client-server/create_room.yaml | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/api/client-server/create_room.yaml b/api/client-server/create_room.yaml index 3e88ee0c0..37c8ec210 100644 --- a/api/client-server/create_room.yaml +++ b/api/client-server/create_room.yaml @@ -1,4 +1,5 @@ # Copyright 2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,7 +41,7 @@ paths: 0. A default ``m.room.power_levels`` event, giving the room creator (and not other members) permission to send state events. - 1. Events set by ``presets``. + 1. Events set by the ``preset``. 2. Events listed in ``initial_state``, in the order that they are listed. @@ -49,6 +50,16 @@ paths: 4. Invite events implied by ``invite`` and ``invite_3pid``. + The available presets do the following with respect to room state: + + ======================== ============== ====================== ================ ========= + Preset ``join_rules`` ``history_visibility`` ``guest_access`` Other + ======================== ============== ====================== ================ ========= + ``private_chat`` ``invite`` ``shared`` ``can_join`` + ``trusted_private_chat`` ``invite`` ``shared`` ``can_join`` All invitees are given the same power level as the room creator. + ``public_chat`` ``public`` ``shared`` ``forbidden`` + ======================== ============== ====================== ================ ========= + operationId: createRoom security: - accessToken: [] @@ -142,7 +153,7 @@ paths: room. The expected format of the state events are an object with type, state_key and content keys set. - Takes precedence over events set by ``presets``, but gets + Takes precedence over events set by ``preset``, but gets overriden by ``name`` and ``topic`` keys. items: type: object @@ -163,20 +174,7 @@ paths: enum: ["private_chat", "public_chat", "trusted_private_chat"] description: |- Convenience parameter for setting various default state events - based on a preset. Must be either: - - ``private_chat`` => - ``join_rules`` is set to ``invite``. - ``history_visibility`` is set to ``shared``. - - ``trusted_private_chat`` => - ``join_rules`` is set to ``invite``. - ``history_visibility`` is set to ``shared``. - All invitees are given the same power level as the room creator. - - ``public_chat``: => - ``join_rules`` is set to ``public``. - ``history_visibility`` is set to ``shared``. + based on a preset. is_direct: type: boolean description: |- From e409b40fad7f456f962e6005d8265de4da5cbbd0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 18:22:10 -0600 Subject: [PATCH 039/123] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 44d41f4e4..781e2f90f 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -17,6 +17,8 @@ Unreleased changes (`#1158 `_). - Document the GET version of ``/login`` (`#1361 `_). + - Document guest access in ``/createRoom`` presets + (`#1379 `_). - Spec clarifications: From 95e8f1bddebc927fff3bf2b03ea795a79312b2ed Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 19:21:03 -0600 Subject: [PATCH 040/123] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 44d41f4e4..34e42d0b4 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -39,6 +39,8 @@ Unreleased changes (`#1152 `_). - Mark ``GET /rooms/{roomId}/members`` as requiring authentication (`#1245 `_). + - Define what a ``RoomEvent`` is on ``/rooms/{roomId}/messages`` + (`#1380 `_). - Describe ``StateEvent`` for ``/createRoom`` (`#1329 `_). From 436b4106980663b595570f5878033765195be6b2 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 20:02:30 -0600 Subject: [PATCH 041/123] Document the error code when someone tries to forget a room they are in --- api/client-server/leaving.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/api/client-server/leaving.yaml b/api/client-server/leaving.yaml index e8e58397e..57a148c62 100644 --- a/api/client-server/leaving.yaml +++ b/api/client-server/leaving.yaml @@ -99,6 +99,15 @@ paths: } schema: type: object + 400: + description: The user has not left the room + examples: + application/json: { + "errcode": "M_UNKNOWN", + "error": "User @example:matrix.org is in room !au1ba7o:matrix.org" + } + schema: + "$ref": "definitions/error.yaml" 429: description: This request was rate-limited. schema: From dfdc5f9bef899c356695c9cc0c5b7f9e8c45b2d5 Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Thu, 3 May 2018 23:19:55 +0900 Subject: [PATCH 042/123] search.yaml: Rename some titles to avoid confusion between request and response schemas Signed-off-by: Alexey Rusakov --- api/client-server/search.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index e5f6a872b..f7c961805 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -74,7 +74,7 @@ paths: properties: room_events: type: object - title: "Room Events" + title: Room Events Criteria description: Mapping of category name to search criteria. properties: search_term: @@ -103,7 +103,7 @@ paths: The order in which to search for results. By default, this is ``"rank"``. event_context: - title: "Event Context" + title: "Include Event Context" type: object description: |- Configures whether any context for the events @@ -169,13 +169,13 @@ paths: properties: search_categories: type: object - title: Categories + title: Result Categories description: Describes which categories to search in and their criteria. properties: room_events: type: object - title: Room Event Results + title: Result Room Events description: Mapping of category name to search criteria. properties: count: From 0867ab2365e403241343d45bc06c966d73442ea3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 20:14:19 -0600 Subject: [PATCH 043/123] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 44d41f4e4..f732a6d83 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -41,6 +41,8 @@ Unreleased changes (`#1245 `_). - Describe ``StateEvent`` for ``/createRoom`` (`#1329 `_). + - Clarify the request and result types on ``/search`` + (`#1381 `_). - Changes to the API which will be backwards-compatible for clients: From cc3724b54a44bef1ee26577e74b2f167aebdcc2a Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 Jul 2018 11:55:25 +0100 Subject: [PATCH 044/123] fix indentation, error messages. add rest of PR --- .../application_service.yaml | 331 ++++---- api/client-server/third_party_lookup.yaml | 338 ++++++++ changelogs/client_server.rst | 2 - specification/proposals.rst | 770 +++++++++++++++++- specification/targets.yaml | 2 +- specification/third_party_lookup.rst | 26 + 6 files changed, 1306 insertions(+), 163 deletions(-) create mode 100644 api/client-server/third_party_lookup.yaml create mode 100644 specification/third_party_lookup.rst diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index a0713d45f..5efc058b4 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -30,8 +30,8 @@ paths: put: summary: Send some events to the application service. description: |- - This API is called by the HS when the HS wants to push an event (or - batch of events) to the AS. + This API is called by the homeserver when it wants to push an event + (or batch of events) to the application service. operationId: sendTransaction parameters: - in: path @@ -48,33 +48,33 @@ paths: schema: type: object example: { - "events": [ - { - "age": 32, - "content": { - "body": "incoming message", - "msgtype": "m.text" - }, - "event_id": "$14328055551tzaee:localhost", - "origin_server_ts": 1432804485886, - "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", - "type": "m.room.message", - "user_id": "@bob:localhost" + "events": [ + { + "age": 32, + "content": { + "body": "incoming message", + "msgtype": "m.text" }, - { - "age": 1984, - "content": { - "body": "another incoming message", - "msgtype": "m.text" - }, - "event_id": "$1228055551ffsef:localhost", - "origin_server_ts": 1432804485886, - "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", - "type": "m.room.message", - "user_id": "@bob:localhost" - } - ] - } + "event_id": "$14328055551tzaee:localhost", + "origin_server_ts": 1432804485886, + "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", + "type": "m.room.message", + "user_id": "@bob:localhost" + }, + { + "age": 1984, + "content": { + "body": "another incoming message", + "msgtype": "m.text" + }, + "event_id": "$1228055551ffsef:localhost", + "origin_server_ts": 1432804485886, + "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", + "type": "m.room.message", + "user_id": "@bob:localhost" + } + ] + } description: "Transaction informations" properties: events: @@ -89,14 +89,14 @@ paths: description: The transaction was processed successfully. examples: application/json: { - } + } schema: type: object 500: description: There was a problem completing the request. examples: application/json: { - } + } schema: type: object "/rooms/{roomAlias}": @@ -126,7 +126,7 @@ paths: before responding. examples: application/json: { - } + } schema: type: object 401: @@ -135,8 +135,8 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } schema: type: object 403: @@ -144,8 +144,8 @@ paths: The credentials supplied by the homeserver were rejected. examples: application/json: { - "errcode": "M_FORBIDDEN" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } schema: type: object 404: @@ -154,15 +154,15 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "M_NOT_FOUND" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } schema: type: object 500: description: There was a problem completing the request. examples: application/json: { - } + } schema: type: object "/users/{userId}": @@ -189,7 +189,7 @@ paths: service MUST create the user using the client-server API. examples: application/json: { - } + } schema: type: object 401: @@ -198,8 +198,8 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } schema: type: object 403: @@ -207,8 +207,8 @@ paths: The credentials supplied by the homeserver were rejected. examples: application/json: { - "errcode": "M_FORBIDDEN" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } schema: type: object 404: @@ -217,24 +217,24 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "M_NOT_FOUND" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } schema: type: object 500: description: There was a problem completing the request. examples: application/json: { - } + } schema: type: object "/_matrix/app/unstable/thirdparty/{protocol}": get: summary: Retrieve metadata about a specific protocol that the application service supports. description: |- - This API is called by the HS when it wants to present clients with - specific information about the various thirdparty networks that an AS - supports. + This API is called by the homeserver when it wants to present clients + with specific information about the various third party networks that + an application service supports. operationId: queryMetadata parameters: - in: path @@ -249,33 +249,33 @@ paths: description: The protocol was found and metadata returned. examples: application/json: { - "user_fields": ["network", "nickname"], - "location_fields": ["network", "channel"], - "icon": "mxc://example.org/aBcDeFgH", - "field_types": { - "network": { - "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", - "placeholder": "irc.example.org" - }, - "nickname": { - "regexp": "[^\\s]+", - "placeholder": "username" - }, - "channel": { - "regexp": "#[^\\s]+", - "placeholder": "#foobar" - } + "user_fields": ["network", "nickname"], + "location_fields": ["network", "channel"], + "icon": "mxc://example.org/aBcDeFgH", + "field_types": { + "network": { + "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", + "placeholder": "irc.example.org" + }, + "nickname": { + "regexp": "[^\\s]+", + "placeholder": "username" }, - "instances": [ - { - "desc": "Freenode", - "icon": "mxc://example.org/JkLmNoPq", - "fields": { - "network": "freenode.net", - } + "channel": { + "regexp": "#[^\\s]+", + "placeholder": "#foobar" + } + }, + "instances": [ + { + "desc": "Freenode", + "icon": "mxc://example.org/JkLmNoPq", + "fields": { + "network": "freenode.net", } - ] - } + } + ] + } schema: type: object 401: @@ -284,8 +284,8 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } schema: type: object 403: @@ -293,30 +293,30 @@ paths: The credentials supplied by the homeserver were rejected. examples: application/json: { - "errcode": "M_FORBIDDEN" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } schema: type: object 404: description: No protocol was found with the given path. examples: application/json: { - "errcode": "M_NOT_FOUND" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } schema: type: object 500: description: There was a problem completing the request. examples: application/json: { - } + } schema: type: object "/_matrix/app/unstable/thirdparty/user/{protocol}": get: - summary: Retrieve the Matrix ID of a corresponding thirdparty user. + summary: Retrieve the Matrix ID of a corresponding third party user. description: |- - This API is called by the HS in order to retrieve a Matrix ID linked + This API is called by the homeserver in order to retrieve a Matrix ID linked to a user on the external service, given a set of user parameters. operationId: queryUserByProtocol parameters: @@ -331,32 +331,47 @@ paths: name: field1, field2... type: string description: |- - One or more custom fields that are passed to the AS to help identify the user. + One or more custom fields that are passed to the application + service to help identify the user. responses: 200: description: The Matrix IDs found with the given parameters. examples: application/json: [{ - "userid": "@_gitter_jim:matrix.org", - "protocol": "gitter", - "fields": { - "user": "jim" - } - }] + "userid": "@_gitter_jim:matrix.org", + "protocol": "gitter", + "fields": { + "user": "jim" + } + }] schema: type: array description: Matched users. items: type: object title: User + properties: + userid: + type: string + description: The Matrix ID of the matched user. + protocol: + type: string + description: The third party protocol. + fields: + type: object + description: The third party network fields used to identify this user. + properties: + user: + type: string + description: An example field, in this case the username for a Gitter user. 401: description: |- The homeserver has not supplied credentials to the application service. Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } schema: type: object 403: @@ -364,67 +379,67 @@ paths: The credentials supplied by the homeserver were rejected. examples: application/json: { - "errcode": "M_FORBIDDEN" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } schema: type: object 404: description: No users were found with the given parameters. examples: application/json: { - "errcode": "M_NOT_FOUND" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } schema: type: object 500: description: There was a problem completing the request. examples: application/json: { - } + } schema: type: object "/_matrix/app/unstable/thirdparty/location/{protocol}": get: - summary: Retreive Matrix-side portals rooms leading to a thirdparty location. + summary: Retreive Matrix-side portal rooms leading to a third party location. description: |- Requesting this endpoint with a valid protocol name results in a list of successful mapping results in a JSON array. Each result contains objects to represent the Matrix room or rooms that represent a portal - to this 3PN. Each has the Matrix room alias string, an identifier for - the particular 3PN protocol, and an object containing the - network-specific fields that comprise this identifier. It should - attempt to canonicalise the identifier as much as reasonably possible - given the network type. + to this third party network. Each has the Matrix room alias string, + an identifier for the particular third party network protocol, and an + object containing the network-specific fields that comprise this + identifier. It should attempt to canonicalise the identifier as much + as reasonably possible given the network type. operationId: queryLocationByProtocol parameters: - in: path name: protocol type: string - description: The protocol used to communicate to the thirdparty network. + description: The protocol used to communicate to the third party network. required: true x-example: "irc" - in: query name: field1, field2... type: string description: |- - One or more custom fields that are passed to the AS to help - identify the thirdparty location. + One or more custom fields that are passed to the application + service to help identify the third party location. responses: 200: description: At least one portal room was found. examples: application/json: [{ - "alias": "#freenode_#matrix:matrix.org", - "protocol": "irc", - "fields": { - "network": "freenode", - "channel": "#matrix" - } - }] + "alias": "#freenode_#matrix:matrix.org", + "protocol": "irc", + "fields": { + "network": "freenode", + "channel": "#matrix" + } + }] schema: type: array description: |- - Array of portal rooms leading to the requested thirdparty + Array of portal rooms leading to the requested third party location. items: type: object @@ -435,8 +450,8 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } schema: type: object 403: @@ -444,54 +459,54 @@ paths: The credentials supplied by the homeserver were rejected. examples: application/json: { - "errcode": "M_FORBIDDEN" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } schema: type: object 404: description: No mappings were found with the given parameters. examples: application/json: { - "errcode": "M_NOT_FOUND" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } schema: type: object 500: description: There was a problem completing the request. examples: application/json: { - } + } schema: type: object "/_matrix/app/unstable/thirdparty/location": get: - summary: Reverse-lookup thirdparty locations given a Matrix room alias. + summary: Reverse-lookup third party locations given a Matrix room alias. description: |- - Retreive an array of thirdparty network locations from a Matrix room + Retreive an array of third party network locations from a Matrix room alias. operationId: queryLocationByAlias parameters: - - in: path + - in: query name: alias type: string description: The Matrix room alias to look up. responses: 200: description: |- - All found thirdparty locations. Same response format as the + All found third party locations. Same response format as the forward lookup response. examples: application/json: [{ - "alias": "#freenode_#matrix:matrix.org", - "protocol": "irc", - "fields": { - "network": "freenode", - "channel": "#matrix" - } - }] + "alias": "#freenode_#matrix:matrix.org", + "protocol": "irc", + "fields": { + "network": "freenode", + "channel": "#matrix" + } + }] schema: type: array - description: Matched thirdparty locations. + description: Matched third party locations. items: type: object title: Location @@ -501,8 +516,8 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } schema: type: object 403: @@ -510,51 +525,51 @@ paths: The credentials supplied by the homeserver were rejected. examples: application/json: { - "errcode": "M_FORBIDDEN" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } schema: type: object 404: description: No mappings were found with the given parameters. examples: application/json: { - "errcode": "M_NOT_FOUND" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } schema: type: object 500: description: There was a problem completing the request. examples: application/json: { - } + } schema: type: object "/_matrix/app/unstable/thirdparty/user": get: - summary: Reverse-lookup thirdparty users given a Matrix ID. + summary: Reverse-lookup third party users given a Matrix ID. description: |- - Retreive an array of thirdparty users from a Matrix ID. + Retreive an array of third party users from a Matrix ID. operationId: queryUserByID paramters: - - in: path + - in: query name: userid type: string description: The Matrix ID to look up. responses: 200: description: |- - An array of thirdparty users. + An array of third party users. examples: application/json: [{ - "userid": "@_gitter_jim:matrix.org", - "protocol": "gitter", - "fields": { - "user": "jim" - } - }] + "userid": "@_gitter_jim:matrix.org", + "protocol": "gitter", + "fields": { + "user": "jim" + } + }] schema: type: array - description: Matched thirdparty users + description: Matched third party users items: type: object title: User @@ -564,8 +579,8 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } schema: type: object 403: @@ -573,22 +588,22 @@ paths: The credentials supplied by the homeserver were rejected. examples: application/json: { - "errcode": "M_FORBIDDEN" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } schema: type: object 404: description: No mappings were found with the given parameters. examples: application/json: { - "errcode": "M_NOT_FOUND" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } schema: type: object 500: description: There was a problem completing the request. examples: application/json: { - } + } schema: type: object \ No newline at end of file diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml new file mode 100644 index 000000000..41b5df16b --- /dev/null +++ b/api/client-server/third_party_lookup.yaml @@ -0,0 +1,338 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Client-Server Third Party Lookup API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/%CLIENT_MAJOR_VERSION% +consumes: + - application/json +produces: + - application/json +paths: + "/thirdparty/protocols": + get: + summary: Retrieve metadata about all protocols that a homeserver supports. + description: |- + Fetches the overall metadata about protocols supported by the + homeserver. Includes both the available protocols and all fields + required for queries against each protocol. + operationId: queryMetadata + responses: + 200: + description: The protocols supported by the homeserver. + examples: + application/json: { + "irc": { + "user_fields": ["network", "nickname"], + "location_fields": ["network", "channel"], + "icon": "mxc://example.org/aBcDeFgH", + "field_types": { + "network": { + "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", + "placeholder": "irc.example.org" + }, + "nickname": { + "regexp": "[^\\s]+", + "placeholder": "username" + }, + "channel": { + "regexp": "#[^\\s]+", + "placeholder": "#foobar" + } + }, + "instances": [ + { + "desc": "Freenode", + "icon": "mxc://example.org/JkLmNoPq", + "fields": { + "network": "freenode.net", + } + } + ] + }, + "gitter": { + "user_fields": ["username"], + "location_fields": ["room"], + "field_types": { + "username": { + "regexp": "@[^\\s]+", + "placeholder": "@username" + }, + "room": { + "regexp": "[^\\s]+\\/[^\\s]+", + "placeholder": "matrix-org/matrix-doc" + } + }, + "instances": [ + { + "desc": "Gitter", + "icon": "mxc://example.org/zXyWvUt", + "fields": {} + } + ] + } + } + schema: + type: object + "/thirdparty/protocol/{protocol}": + get: + summary: Retrieve metadata about a specific protocol that the homeserver supports. + description: |- + Fetches the metadata from the homeserver about a particular third party protocol. + operationId: queryMetadata + parameters: + - in: path + name: protocol + type: string + description: |- + The name of the protocol. + required: true + x-example: "irc" + responses: + 200: + description: The protocol was found and metadata returned. + examples: + application/json: { + "user_fields": ["network", "nickname"], + "location_fields": ["network", "channel"], + "icon": "mxc://example.org/aBcDeFgH", + "field_types": { + "network": { + "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", + "placeholder": "irc.example.org" + }, + "nickname": { + "regexp": "[^\\s#]+", + "placeholder": "username" + }, + "channel": { + "regexp": "#[^\\s]+", + "placeholder": "#foobar" + } + }, + "instances": [ + { + "desc": "Freenode", + "icon": "mxc://example.org/JkLmNoPq", + "fields": { + "network": "freenode.net", + } + } + ] + } + schema: + type: object + 404: + description: The protocol is unknown + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + "/thirdparty/location/{protocol}": + get: + summary: Retreive Matrix-side portals rooms leading to a third party location. + description: |- + Requesting this endpoint with a valid protocol name results in a list + of successful mapping results in a JSON array. Each result contains + objects to represent the Matrix room or rooms that represent a portal + to this third party network. Each has the Matrix room alias string, + an identifier for the particular third party network protocol, and an + object containing the network-specific fields that comprise this + identifier. It should attempt to canonicalise the identifier as much + as reasonably possible given the network type. + operationId: queryLocationByProtocol + parameters: + - in: path + name: protocol + type: string + description: The protocol used to communicate to the third party network. + required: true + x-example: "irc" + - in: query + name: field1, field2... + type: string + description: |- + One or more custom fields to help identify the third party + location. + responses: + 200: + description: At least one portal room was found. + examples: + application/json: [{ + "alias": "#freenode_#matrix:matrix.org", + "protocol": "irc", + "fields": { + "network": "freenode", + "channel": "#matrix" + } + }] + schema: + type: array + description: |- + Array of portal rooms leading to the requested third party + location. + items: + type: object + title: Portal Room + 404: + description: The Matrix room alias was not found + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + "/thirdparty/user/{protocol}": + get: + summary: Retrieve the Matrix ID of a corresponding third party user. + description: |- + Retrieve a Matrix ID linked to a user on the third party service, given + a set of user parameters. + operationId: queryUserByProtocol + parameters: + - in: path + name: protocol + type: string + description: |- + The name of the protocol. + required: true + x-example: irc + - in: query + name: field1, field2... + type: string + description: |- + One or more custom fields that are passed to the AS to help identify the user. + responses: + 200: + description: The Matrix IDs found with the given parameters. + examples: + application/json: [{ + "userid": "@_gitter_jim:matrix.org", + "protocol": "gitter", + "fields": { + "user": "jim" + } + }] + schema: + type: array + description: Matched users. + items: + type: object + title: User + properties: + userid: + type: string + description: The Matrix ID of the matched user. + protocol: + type: string + description: The third party protocol. + fields: + type: object + description: The third party network fields used to identify this user. + properties: + user: + type: string + description: An example field, in this case the username for a Gitter user. + 404: + description: The Matrix ID was not found + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + "/thirdparty/location": + get: + summary: Reverse-lookup third party locations given a Matrix room alias. + description: |- + Retreive an array of third party network locations from a Matrix room + alias. + operationId: queryLocationByAlias + parameters: + - in: path + name: alias + type: string + description: The Matrix room alias to look up. + responses: + 200: + description: |- + All found third party locations. Same response format as the + forward lookup response. + examples: + application/json: [{ + "alias": "#freenode_#matrix:matrix.org", + "protocol": "irc", + "fields": { + "network": "freenode", + "channel": "#matrix" + } + }] + schema: + type: array + description: Matched third party locations. + items: + type: object + title: Location + 404: + description: The Matrix room alias was not found + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + "/thirdparty/user": + get: + summary: Reverse-lookup third party users given a Matrix ID. + description: |- + Retreive an array of third party users from a Matrix ID. + operationId: queryUserByID + paramters: + - in: path + name: userid + type: string + description: The Matrix ID to look up. + responses: + 200: + description: |- + An array of third party users. + examples: + application/json: [{ + "userid": "@_gitter_jim:matrix.org", + "protocol": "gitter", + "fields": { + "user": "jim" + } + }] + schema: + type: array + description: Matched third party users + items: + type: object + title: User + 404: + description: The Matrix ID was not found + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object \ No newline at end of file diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index f7e94dbe4..7f31aa751 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -11,8 +11,6 @@ Unreleased changes (`#1110 `_). - ``POST /delete_devices`` (`#1239 `_). - - ``GET /thirdparty/protocols`` - (`#1353 `_). - ``GET /thirdparty/protocol/{protocol}`` (`#1353 `_). - ``GET /thirdparty/location/{protocol}`` diff --git a/specification/proposals.rst b/specification/proposals.rst index 371850ab3..d04edfa84 100644 --- a/specification/proposals.rst +++ b/specification/proposals.rst @@ -1,6 +1,772 @@ Tables of Tracked Proposals --------------------------- -This file is autogenerated by a jenkins build process +proposal-wip +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -View the current live version `at https://matrix.org/docs/spec/proposals `_ +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documentation + - Author + - Shepherd + - PRs + * - `MSC455 `_ + - Do we want to specify a matrix:// URI scheme for rooms? (SPEC-5) + - 2014-09-15 + - 2018-05-15 + - `455-1 `_ + - `@KitsuneRal`_ + - None + - + * - `MSC586 `_ + - Federation API for canonicalising MXIDs + - 2014-10-27 + - 2018-05-15 + - `586-1 `_ + - `@ara4n`_ + - None + - + * - `MSC489 `_ + - Extensible Profiles. (SPEC-93) + - 2015-01-19 + - 2018-05-15 + - `489-1 `_ + - `@erikjohnston`_ + - None + - + * - `MSC1196 `_ + - Matrix Escape Hatch (Versioned Rooms) + - 2015-10-22 + - 2018-05-15 + - `1196-1 `_ + - `@leonerd`_ + - None + - + * - `MSC1148 `_ + - Support for websockets + - 2015-11-16 + - 2018-06-04 + - `1148-1 `_, `1148-2 `_ + - `@richvdh`_, `@krombel`_ + - None + - + * - `MSC1238 `_ + - Push to Talk + - 2016-04-13 + - 2018-05-15 + - `1238-1 `_ + - `@aviraldg`_ + - None + - `PR#310`_ + * - `MSC1198 `_ + - Threading API + - 2016-05-23 + - 2018-05-15 + - `1198-1 `_ + - `@Half-Shot`_ + - None + - + * - `MSC1207 `_ + - Publishing Room Lists for 3rd party networks + - 2016-10-21 + - 2018-05-31 + - `1207-1 `_ + - `@erikjohnston`_ + - None + - + * - `MSC441 `_ + - Support for Reactions / Aggregations + - 2016-12-25 + - 2018-05-15 + - `441-1 `_ + - `@pik`_ + - `@ara4n`_ + - + * - `MSC1237 `_ + - Improving m.location with GeoJSON and accuracy + - 2017-05-19 + - 2018-05-15 + - `1237-1 `_ + - `@Half-Shot`_ + - None + - `PR#919`_ + * - `MSC971 `_ + - Add groups stuff to spec + - 2017-08-10 + - 2018-05-20 + - `971-1 `_, `971-2 `_, `971-3 `_ + - `@erikjohnston`_ + - None + - + * - `MSC1215 `_ + - Groups as Rooms + - 2017-10-17 + - 2018-05-15 + - `1215-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1217 `_ + - Visibility of groups to group members and their non-members + - 2017-10-30 + - 2018-05-15 + - `1217-1 `_ + - `@lampholder`_ + - None + - + * - `MSC1218 `_ + - Bridging group membership with 3rd party group sources + - 2017-11-15 + - 2018-05-15 + - `1218-1 `_ + - `@lukebarnard1`_ + - None + - + * - `MSC1219 `_ + - Proposal for storing megolm keys serverside + - 2017-11-23 + - 2018-05-15 + - `1219-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1221 `_ + - Improving Presence + - 2017-12-26 + - 2018-05-24 + - `1221-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1222 `_ + - Pushing updates about Groups (Communities) to clients + - 2018-01-02 + - 2018-05-24 + - `1222-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1225 `_ + - Extensible event types & fallback in Matrix + - 2018-02-18 + - 2018-05-15 + - `1225-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1206 `_ + - Proposal for improved bot support + - 2018-03-14 + - 2018-06-08 + - `1206-1 `_ + - `@turt2live`_ + - None + - + * - `MSC1228 `_ + - Removing MXIDs from events + - 2018-04-19 + - 2018-05-15 + - `1228-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1231 `_ + - Handling Consent for Privacy Policy + - 2018-05-02 + - 2018-05-15 + - `1231-1 `_ + - `@neilisfragile`_ + - None + - + * - `MSC1267 `_ + - Interactive Key Verification + - 2018-05-28 + - 2018-05-28 + - `1267-1 `_ + - `@uhoreg`_ + - None + - + * - `MSC1280 `_ + - Mechanisms for communicating erasure requests to bots and federated homeservers + - 2018-06-05 + - 2018-06-05 + - `1280-1 `_ + - `@richvdh`_ + - None + - + * - `MSC688 `_ + - Calculate room names server-side + - 2018-06-10 + - 2018-06-15 + - `688-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1324 `_ + - Custom protocol definitions for application services + - 2018-06-20 + - 2018-06-20 + - `1324-1 `_ + - `@anoadragon453`_ + - None + - + * - `MSC1323 `_ + - AS traffic rate-limiting + - 2018-06-20 + - 2018-07-03 + - `1323-1 `_ + - `@anoadragon453`_ + - None + - + * - `MSC1322 `_ + - Mechanism to communicate 3PID binding updates or deletions to homeservers + - 2018-06-20 + - 2018-06-20 + - `1322-1 `_ + - `@babolivier`_ + - None + - + + + +proposal-ready-for-review +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documentation + - Author + - Shepherd + - PRs + * - `MSC1227 `_ + - Proposal for lazy-loading room members to improve initial sync speed and client RAM usage + - 2018-03-05 + - 2018-06-10 + - `1227-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1206 `_ + - Proposal for improved bot support + - 2018-03-14 + - 2018-06-08 + - `1206-1 `_ + - `@turt2live`_ + - None + - + * - `MSC1256 `_ + - Custom emoji and sticker packs in matrix + - 2018-05-23 + - 2018-05-24 + - `1256-1 `_ + - `@turt2live`_ + - None + - + * - `MSC1270 `_ + - Proposal Add /_matrix/media/v1/resolve_url to Client-Server API: download and preview urls in the clients despite CORS + - 2018-05-31 + - 2018-06-19 + - `1270-1 `_ + - `@oivoodoo`_ + - None + - + * - `MSC701 `_ + - Auth for content repo (and enforcing GDPR erasure) + - 2018-06-04 + - 2018-06-07 + - `701-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1304 `_ + - Proposal to simplify the auth rules of m.room.power_level events. + - 2018-06-13 + - 2018-06-14 + - `1304-1 `_ + - `@richvdh`_, `@ara4n`_ + - None + - + * - `MSC1301 `_ + - Proposal for improving authorization for the matrix profile API + - 2018-06-13 + - 2018-06-13 + - `1301-1 `_ + - `@turt2live`_ + - None + - + * - `MSC1309 `_ + - Proposal for an application service management API + - 2018-06-14 + - 2018-06-15 + - `1309-1 `_ + - `@turt2live`_ + - None + - + * - `MSC1308 `_ + - Proposal for speeding up review of simple spec changes + - 2018-06-14 + - 2018-06-24 + - + - `@ara4n`_ + - None + - + * - `MSC1306 `_ + - Proposal to filter out traffic to Appservices based on filters + - 2018-06-14 + - 2018-06-14 + - `1306-1 `_ + - `@Half-Shot`_ + - None + - + + + +proposal-in-review +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documentation + - Author + - Shepherd + - PRs + * - `MSC1194 `_ + - A way for HSes to remove bindings from ISes (aka unbind) + - 2018-05-09 + - 2018-06-05 + - `1194-1 `_ + - `@dbkr`_ + - None + - + + + +proposal-passed-review +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documentation + - Author + - Shepherd + - PRs + * - `MSC433 `_ + - Support for discovering API endpoints via .well-known URIs (SPEC-121) + - 2015-03-08 + - 2018-07-01 + - `433-1 `_, `433-2 `_ + - `@maxidor`_, `others`_ + - `@uhoreg`_ + - + * - `MSC1226 `_ + - State Reset mitigation proposal + - 2018-02-20 + - 2018-05-15 + - `1226-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1229 `_ + - Mitigating abuse of the event depth parameter over federation + - 2018-04-30 + - 2018-05-15 + - `1229-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1232 `_ + - Media limits API + - 2018-05-04 + - 2018-06-21 + - `1232-1 `_ + - `@Half-Shot`_ + - None + - `PR#1189`_ + * - `MSC1236 `_ + - Matrix Widget API v2 + - 2018-05-13 + - 2018-05-15 + - `1236-1 `_ + - `@rxl881`_ + - None + - + + + +spec-pr-missing +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documentation + - Author + - Shepherd + - PRs + * - `MSC1200 `_ + - Configuration of E2E encryption in a room + - 2016-06-16 + - 2018-05-31 + - `1200-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1201 `_ + - Device Management API + - 2016-07-14 + - 2018-05-15 + - `1201-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1203 `_ + - 3rd Party Entity lookup API + - 2016-07-21 + - 2018-07-02 + - `1203-1 `_ + - `@leonerd`_ + - None + - + * - `MSC1208 `_ + - Encrypted attachment format + - 2016-10-26 + - 2018-05-15 + - `1208-1 `_ + - `@NegativeMjark`_ + - None + - + * - `MSC739 `_ + - Reporting inappropriate content in Matrix + - 2016-11-21 + - 2018-05-31 + - `739-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1211 `_ + - Megolm session export format + - 2017-01-03 + - 2018-05-15 + - `1211-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1212 `_ + - Device List Update Stream + - 2017-01-18 + - 2018-05-15 + - `1212-1 `_ + - `@richvdh`_ + - None + - + * - `MSC829 `_ + - Need to spec msisdn login API + - 2017-03-08 + - 2018-05-15 + - `829-1 `_ + - `@dbkr`_ + - None + - + * - `MSC855 `_ + - spec m.login.msisdn UI auth type + - 2017-03-24 + - 2018-05-15 + - `855-1 `_ + - `@dbkr`_ + - None + - + * - `MSC910 `_ + - Add new Read Marker API to docs + - 2017-05-08 + - 2018-05-15 + - + - `@lukebarnard1`_ + - None + - + * - `MSC1067 `_ + - Spec @mentions + - 2017-07-14 + - 2018-05-15 + - `1067-1 `_ + - `@lukebarnard1`_ + - None + - + * - `MSC1214 `_ + - Related Groups (i.e. flair) + - 2017-10-03 + - 2018-05-15 + - `1214-1 `_ + - `@lukebarnard1`_ + - None + - + * - `MSC1033 `_ + - Doc @room notifications + - 2017-10-23 + - 2018-05-31 + - + - `@dbkr`_ + - None + - + * - `MSC1183 `_ + - Document key-share requests + - 2018-04-30 + - 2018-05-31 + - `1183-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1230 `_ + - Temporary mitigation for depth parameter abuse + - 2018-05-01 + - 2018-05-15 + - `1230-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1234 `_ + - Rich Replies format + - 2018-05-12 + - 2018-05-18 + - `1234-1 `_ + - `@t3chguy`_ + - None + - + + + +merged +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documentation + - Author + - Shepherd + - PRs + * - `MSC1197 `_ + - Ignoring Users + - 2016-05-03 + - 2018-05-18 + - `1197-1 `_ + - `@erikjohnston`_ + - None + - `PR#1142`_ + * - `MSC1199 `_ + - Notifications API + - 2016-05-23 + - 2018-06-25 + - `1199-1 `_ + - `@dbkr`_ + - None + - + * - `MSC1204 `_ + - Access Token Semantics (refresh and macaroons) - aka Auth Sept 2016 Edition + - 2016-09-29 + - 2018-06-25 + - `1204-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1205 `_ + - Proposal for multi-device deletion API + - 2016-10-12 + - 2018-06-25 + - `1205-1 `_ + - `@richvdh`_ + - None + - `PR#1239`_ + * - `MSC953 `_ + - Add /user_directory/search API + - 2017-05-31 + - 2018-05-10 + - `953-1 `_ + - `@erikjohnston`_ + - None + - + * - `MSC1233 `_ + - A proposal for organising spec proposals + - 2018-05-10 + - 2018-06-25 + - `1233-1 `_ + - `@ara4n`_ + - None + - `PR#1240`_ + + + +abandoned +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documentation + - Author + - Shepherd + - PRs + * - `MSC531 `_ + - Homeservers as OAuth authorization endpoints (resource owners) (SPEC-206) + - 2015-07-25 + - 2018-05-15 + - `531-1 `_ + - `@Kegsay`_ + - None + - + * - `MSC1202 `_ + - Profile Personae + - 2016-07-15 + - 2018-05-15 + - `1202-1 `_ + - `@erikjohnston`_ + - None + - + * - `MSC1209 `_ + - Server Side Profile API + - 2016-11-01 + - 2018-05-15 + - `1209-1 `_ + - `@erikjohnston`_ + - None + - + * - `MSC1213 `_ + - Set defaults for m.federate + - 2017-04-10 + - 2018-05-18 + - `1213-1 `_ + - `@psaavedra`_ + - None + - + + + +obsolete +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documentation + - Author + - Shepherd + - PRs + * - `MSC1223 `_ + - Replies event format + - 2018-02-01 + - 2018-05-15 + - `1223-1 `_ + - `@t3chguy`_ + - None + - + * - `MSC1224 `_ + - Replies - next steps + - 2018-02-03 + - 2018-05-15 + - `1224-1 `_ + - `@t3chguy`_ + - None + - + * - `MSC1235 `_ + - Proposal for Calendar Events + - 2018-02-06 + - 2018-05-15 + - `1235-1 `_ + - `@Half-Shot`_ + - None + - + * - `MSC1220 `_ + - Rich quoting proposal + - 2018-05-10 + - 2018-05-15 + - `1220-1 `_ + - `@t3chguy`_ + - None + - + + + + + +.. _@rxl881: https://github.com/rxl881 +.. _@turt2live: https://github.com/turt2live +.. _@erikjohnston: https://github.com/erikjohnston +.. _@anoadragon453: https://github.com/anoadragon453 +.. _@t3chguy: https://github.com/t3chguy +.. _@Kegsay: https://github.com/Kegsay +.. _@KitsuneRal: https://github.com/KitsuneRal +.. _@leonerd: https://github.com/leonerd +.. _@psaavedra: https://github.com/psaavedra +.. _@ara4n: https://github.com/ara4n +.. _@krombel: https://github.com/krombel +.. _@maxidor: https://github.com/maxidor +.. _@uhoreg: https://github.com/uhoreg +.. _@pik: https://github.com/pik +.. _@neilisfragile: https://github.com/neilisfragile +.. _@babolivier: https://github.com/babolivier +.. _@lukebarnard1: https://github.com/lukebarnard1 +.. _others: https://github.com/thers +.. _@Half-Shot: https://github.com/Half-Shot +.. _@aviraldg: https://github.com/aviraldg +.. _@oivoodoo: https://github.com/oivoodoo +.. _@richvdh: https://github.com/richvdh +.. _@NegativeMjark: https://github.com/NegativeMjark +.. _@lampholder: https://github.com/lampholder +.. _@dbkr: https://github.com/dbkr +.. _PR#1189: https://github.com/matrix-org/matrix-doc/pull/1189 +.. _PR#310: https://github.com/matrix-org/matrix-doc/pull/310 +.. _PR#1142: https://github.com/matrix-org/matrix-doc/pull/1142 +.. _PR#1239: https://github.com/matrix-org/matrix-doc/pull/1239 +.. _PR#1240: https://github.com/matrix-org/matrix-doc/pull/1240 +.. _PR#919: https://github.com/matrix-org/matrix-doc/pull/919 \ No newline at end of file diff --git a/specification/targets.yaml b/specification/targets.yaml index c6c9cd234..96199d291 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -12,7 +12,7 @@ targets: - { 1: modules.rst } - { 2: feature_profiles.rst } - { 2: "group:modules" } # reference a group of files - - { 1: thirdparty_lookup.rst } + - { 1: third_party_lookup.rst } version_label: "%CLIENT_RELEASE_LABEL%" application_service: files: diff --git a/specification/third_party_lookup.rst b/specification/third_party_lookup.rst new file mode 100644 index 000000000..1279f9fed --- /dev/null +++ b/specification/third_party_lookup.rst @@ -0,0 +1,26 @@ +Application Services +==================== + +An application service is a separate program that interacts with a homeserver +and provides various bits of functionality that would otherwise not make +sense to include directly in the homeserver. This ranges from bots, which can +often be interacted with, to bridges, which allow Matrix users to communicate +with users on third party networks. The following describes endpoints that a +Matrix client can use to interact with an application service through the +facilitation of the homeserver. + +Third Party Lookups +------------------ + +Application services can provide access to third party networks via bridging. +This allows Matrix users to communicate with users on other communication +platforms, with messages ferried back and forth by the application service. A +single application service may bridge multiple third party networks, and many +individual locations within those networks. A single third party network +location may be bridged to multiple Matrix rooms. + +In order for a client to join one of these bridged rooms, or communicate +directly with a user on a third party network, the following endpoints can be +used. + +{{third_party_lookup_cs_http_api}} \ No newline at end of file From a6dad76c018925867b4d714a2a87e502aff623b9 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 Jul 2018 11:56:30 +0100 Subject: [PATCH 045/123] Fix indentation --- .../application_service.yaml | 28 +++++++++---------- api/client-server/third_party_lookup.yaml | 28 +++++++++---------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 5efc058b4..5edb10121 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -350,20 +350,20 @@ paths: items: type: object title: User - properties: - userid: - type: string - description: The Matrix ID of the matched user. - protocol: - type: string - description: The third party protocol. - fields: - type: object - description: The third party network fields used to identify this user. - properties: - user: - type: string - description: An example field, in this case the username for a Gitter user. + properties: + userid: + type: string + description: The Matrix ID of the matched user. + protocol: + type: string + description: The third party protocol. + fields: + type: object + description: The third party network fields used to identify this user. + properties: + user: + type: string + description: An example field, in this case the username for a Gitter user. 401: description: |- The homeserver has not supplied credentials to the application service. diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 41b5df16b..2b07484aa 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -237,20 +237,20 @@ paths: items: type: object title: User - properties: - userid: - type: string - description: The Matrix ID of the matched user. - protocol: - type: string - description: The third party protocol. - fields: - type: object - description: The third party network fields used to identify this user. - properties: - user: - type: string - description: An example field, in this case the username for a Gitter user. + properties: + userid: + type: string + description: The Matrix ID of the matched user. + protocol: + type: string + description: The third party protocol. + fields: + type: object + description: The third party network fields used to identify this user. + properties: + user: + type: string + description: An example field, in this case the username for a Gitter user. 404: description: The Matrix ID was not found examples: From e6218b56abe447c3ecd4ed2c3afa975db32583d1 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 Jul 2018 11:59:10 +0100 Subject: [PATCH 046/123] Switch properties to schema --- api/application-service/application_service.yaml | 2 +- api/client-server/third_party_lookup.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 5edb10121..8c49043e4 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -350,7 +350,7 @@ paths: items: type: object title: User - properties: + schema: userid: type: string description: The Matrix ID of the matched user. diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 2b07484aa..0cc18ecde 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -237,7 +237,7 @@ paths: items: type: object title: User - properties: + schema: userid: type: string description: The Matrix ID of the matched user. From 8e22e9e85ab9da2e90b957f98c3155865fda8517 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 Jul 2018 12:00:20 +0100 Subject: [PATCH 047/123] Fix title underline length --- specification/third_party_lookup.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/third_party_lookup.rst b/specification/third_party_lookup.rst index 1279f9fed..6325dca0c 100644 --- a/specification/third_party_lookup.rst +++ b/specification/third_party_lookup.rst @@ -10,7 +10,7 @@ Matrix client can use to interact with an application service through the facilitation of the homeserver. Third Party Lookups ------------------- +------------------- Application services can provide access to third party networks via bridging. This allows Matrix users to communicate with users on other communication From 4c38012122b2c6efc4943f7da55d4f3b3be3ae8e Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 Jul 2018 12:58:42 +0100 Subject: [PATCH 048/123] I can spell --- api/application-service/application_service.yaml | 2 +- api/client-server/third_party_lookup.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 8c49043e4..4951a29ff 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -550,7 +550,7 @@ paths: description: |- Retreive an array of third party users from a Matrix ID. operationId: queryUserByID - paramters: + parameters: - in: query name: userid type: string diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 0cc18ecde..343cf2b77 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -305,7 +305,7 @@ paths: description: |- Retreive an array of third party users from a Matrix ID. operationId: queryUserByID - paramters: + parameters: - in: path name: userid type: string From 16860ddd22b8364ea501bbcf0034c5ae189b3bf3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 5 Jul 2018 10:46:53 -0600 Subject: [PATCH 049/123] De-quote --- api/client-server/search.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index f7c961805..687140b3e 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -103,7 +103,7 @@ paths: The order in which to search for results. By default, this is ``"rank"``. event_context: - title: "Include Event Context" + title: Include Event Context type: object description: |- Configures whether any context for the events From e81e0328c22aa2c0cb64b41791d692bb352d7683 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 Jul 2018 17:50:34 +0100 Subject: [PATCH 050/123] Fix single protocol metadata endpoint path, remove 500s --- .../application_service.yaml | 58 +------------------ 1 file changed, 1 insertion(+), 57 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 4951a29ff..86411ffa2 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -92,13 +92,6 @@ paths: } schema: type: object - 500: - description: There was a problem completing the request. - examples: - application/json: { - } - schema: - type: object "/rooms/{roomAlias}": get: summary: Query if a room alias should exist on the application service. @@ -158,13 +151,6 @@ paths: } schema: type: object - 500: - description: There was a problem completing the request. - examples: - application/json: { - } - schema: - type: object "/users/{userId}": get: summary: Query if a user should exist on the application service. @@ -221,14 +207,7 @@ paths: } schema: type: object - 500: - description: There was a problem completing the request. - examples: - application/json: { - } - schema: - type: object - "/_matrix/app/unstable/thirdparty/{protocol}": + "/_matrix/app/unstable/thirdparty/protocol/{protocol}": get: summary: Retrieve metadata about a specific protocol that the application service supports. description: |- @@ -305,13 +284,6 @@ paths: } schema: type: object - 500: - description: There was a problem completing the request. - examples: - application/json: { - } - schema: - type: object "/_matrix/app/unstable/thirdparty/user/{protocol}": get: summary: Retrieve the Matrix ID of a corresponding third party user. @@ -391,13 +363,6 @@ paths: } schema: type: object - 500: - description: There was a problem completing the request. - examples: - application/json: { - } - schema: - type: object "/_matrix/app/unstable/thirdparty/location/{protocol}": get: summary: Retreive Matrix-side portal rooms leading to a third party location. @@ -471,13 +436,6 @@ paths: } schema: type: object - 500: - description: There was a problem completing the request. - examples: - application/json: { - } - schema: - type: object "/_matrix/app/unstable/thirdparty/location": get: summary: Reverse-lookup third party locations given a Matrix room alias. @@ -537,13 +495,6 @@ paths: } schema: type: object - 500: - description: There was a problem completing the request. - examples: - application/json: { - } - schema: - type: object "/_matrix/app/unstable/thirdparty/user": get: summary: Reverse-lookup third party users given a Matrix ID. @@ -598,12 +549,5 @@ paths: application/json: { "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } - schema: - type: object - 500: - description: There was a problem completing the request. - examples: - application/json: { - } schema: type: object \ No newline at end of file From f9a0a49e96a98eb0d96b2ef424868858235ab007 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 5 Jul 2018 20:22:59 +0100 Subject: [PATCH 051/123] Add a bunch more people to the speculator trusted list --- scripts/speculator/main.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 5e1225037..12ec2aec2 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -701,8 +701,8 @@ func ignoreExitCodeOne(err error) error { func main() { flag.Parse() - // It would be great to read this from github, but there's no convenient way to do so. - // Most of these memberships are "private", so would require some kind of auth. + // It would be great to read this from github + // cf https://github.com/matrix-org/matrix-doc/issues/1384 allowedMembers = map[string]bool{ "dbkr": true, "erikjohnston": true, @@ -715,6 +715,11 @@ func main() { "rxl881": true, "uhoreg": true, "turt2live": true, + "Half-Shot": true, + "anoadragon453": true, + "mujx": true, + "benparsons": true, + "KitsuneRal": true, } if err := initCache(); err != nil { log.Fatal(err) From bbf63d116cfca00ea9947547db5c84830736e94d Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 5 Jul 2018 16:55:11 -0600 Subject: [PATCH 052/123] Fix reference to error.yaml in forget room API --- api/client-server/leaving.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/leaving.yaml b/api/client-server/leaving.yaml index 7d8e9c895..513b5b4d1 100644 --- a/api/client-server/leaving.yaml +++ b/api/client-server/leaving.yaml @@ -107,7 +107,7 @@ paths: "error": "User @example:matrix.org is in room !au1ba7o:matrix.org" } schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: From d1c8ec9e489134a804834a049cd9f80794b48abb Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 5 Jul 2018 21:18:28 -0600 Subject: [PATCH 053/123] add circle config --- .circleci/config.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..e8df5956b --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,19 @@ +version: 2 +jobs: + build-docs: + docker: + - image: uhoreg/matrix-doc-build + steps: + - checkout + - run: python scripts/gendoc.py + +workflows: + version: 2 + + egg-info: + jobs: + - build-docs + +notify: + webhooks: + - url: https://giles.cadair.com/circleci From 86b00451953d96e7af2eadb1c013415a488fc8c4 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 5 Jul 2018 21:25:15 -0600 Subject: [PATCH 054/123] virtualenvs are the worst --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e8df5956b..c11e36bad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,9 +2,10 @@ version: 2 jobs: build-docs: docker: - - image: uhoreg/matrix-doc-build + - image: circleci/python:2.7 steps: - checkout + - run: pip install -r scripts/requirements.txt - run: python scripts/gendoc.py workflows: From fe43b79e73f84708ad7ae5b641f7a69be8020170 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 5 Jul 2018 21:29:12 -0600 Subject: [PATCH 055/123] what's even the point --- .circleci/config.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c11e36bad..236a1d813 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,12 +1,17 @@ +gendoc: &gendoc + name: Generate the docs + command: | + source /env/bin/activate + scripts/gendoc.py + version: 2 jobs: build-docs: docker: - - image: circleci/python:2.7 + - image: uhoreg/matrix-doc-build steps: - checkout - - run: pip install -r scripts/requirements.txt - - run: python scripts/gendoc.py + - run: *gendoc workflows: version: 2 From b7c3a7402282eb1b9787d07158e27c93521dec9e Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 5 Jul 2018 21:31:57 -0600 Subject: [PATCH 056/123] artifacts --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 236a1d813..06696633b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,6 +12,11 @@ jobs: steps: - checkout - run: *gendoc + - store_artifacts: + path: scripts/gen + - run: + name: "Doc build is available at:" + command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/scripts/gen/index.html"; echo $DOCS_URL workflows: version: 2 From be5232914a704808f74ac9a5f32b1591fc7988ff Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 5 Jul 2018 21:39:45 -0600 Subject: [PATCH 057/123] Maybe a swagger --- .circleci/config.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 06696633b..c11eb70dc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,6 +4,12 @@ gendoc: &gendoc source /env/bin/activate scripts/gendoc.py +gendoc: &genswagger + name: Generate the swagger + command: | + source /env/bin/activate + scripts/dump-swagger.py + version: 2 jobs: build-docs: @@ -18,12 +24,25 @@ jobs: name: "Doc build is available at:" command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/scripts/gen/index.html"; echo $DOCS_URL + build-swagger: + docker: + - image: uhoreg/matrix-doc-build + steps: + - checkout + - run: *genswagger + - store_artifacts: + path: scripts/swagger/api-docs.json + - run: + name: "Swagger UI is available at:" + command: SWAGGER_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/scripts/swagger/api-docs.json"; echo "https://matrix.org/docs/api/client-server/?url="$SWAGGER_URL + workflows: version: 2 - egg-info: + build-spec: jobs: - build-docs + - build-swagger notify: webhooks: From d5ce87e9567a38c9fbaafa2cbbfd90304794053b Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 6 Jul 2018 10:27:03 +0100 Subject: [PATCH 058/123] Split summary up into description and summary --- api/client-server/content-repo.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 0f4ef2da6..3340808c5 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -272,14 +272,14 @@ paths: "/config": get: summary: Get the configuration for the content repository. - Clients SHOULD use this as a guide when using content repository endpoints. - All values are intentionally left optional. Clients SHOULD follow - the advice given in the field description when the field is not available. + description: Clients SHOULD use this as a guide when using content repository endpoints. + All values are intentionally left optional. Clients SHOULD follow + the advice given in the field description when the field is not available. - **NOTE:** Both clients and server administrators should be aware that proxies - between the client and the server may affect the apparent behaviour of content - repository APIs, for example, proxies may enforce a lower upload size limit - than is advertised by the server on this endpoint. + **NOTE:** Both clients and server administrators should be aware that proxies + between the client and the server may affect the apparent behaviour of content + repository APIs, for example, proxies may enforce a lower upload size limit + than is advertised by the server on this endpoint. operationId: getConfig produces: ["application/json"] security: From 2e6cc8045cc3f0942b9c23caab1edd25f52e4915 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 6 Jul 2018 10:29:05 +0100 Subject: [PATCH 059/123] Better indent of description --- api/client-server/content-repo.yaml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 3340808c5..1f1d6abc9 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -272,14 +272,15 @@ paths: "/config": get: summary: Get the configuration for the content repository. - description: Clients SHOULD use this as a guide when using content repository endpoints. - All values are intentionally left optional. Clients SHOULD follow - the advice given in the field description when the field is not available. + description: |- + Clients SHOULD use this as a guide when using content repository endpoints. + All values are intentionally left optional. Clients SHOULD follow + the advice given in the field description when the field is not available. - **NOTE:** Both clients and server administrators should be aware that proxies - between the client and the server may affect the apparent behaviour of content - repository APIs, for example, proxies may enforce a lower upload size limit - than is advertised by the server on this endpoint. + **NOTE:** Both clients and server administrators should be aware that proxies + between the client and the server may affect the apparent behaviour of content + repository APIs, for example, proxies may enforce a lower upload size limit + than is advertised by the server on this endpoint. operationId: getConfig produces: ["application/json"] security: From 77f4ac54aaddee927812a6c3e10d844d8df00ef7 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 6 Jul 2018 10:36:17 +0100 Subject: [PATCH 060/123] Wordy intro sentence for description --- api/client-server/content-repo.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 1f1d6abc9..d88961b7d 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -273,6 +273,8 @@ paths: get: summary: Get the configuration for the content repository. description: |- + This endpoint allows clients to retrieve the configuration of the content + repository, such as upload limitations. Clients SHOULD use this as a guide when using content repository endpoints. All values are intentionally left optional. Clients SHOULD follow the advice given in the field description when the field is not available. From 516000bdc3888dce6844bebd066961ab5f50916e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 08:53:04 -0600 Subject: [PATCH 061/123] Fix error reference in media repo --- api/client-server/content-repo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 1397c8eee..b3e9517b1 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -336,7 +336,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/error.yaml" tags: - Media From f54d5a40398c3c6a1c252b01d38e50a850685dee Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 15:19:14 -0600 Subject: [PATCH 062/123] Convert build scripts to python 3 --- .gitignore | 2 +- README.rst | 2 +- api/check_examples.py | 6 +-- scripts/dump-swagger.py | 8 +-- scripts/gendoc.py | 38 +++++++------- scripts/proposals.py | 2 +- scripts/swagger-http-server.py | 12 ++--- scripts/templating/batesian/__init__.py | 7 ++- scripts/templating/batesian/sections.py | 8 +-- scripts/templating/batesian/units.py | 4 +- scripts/templating/build.py | 23 ++++----- .../templating/matrix_templates/__init__.py | 4 +- .../templating/matrix_templates/sections.py | 4 +- scripts/templating/matrix_templates/units.py | 49 ++++++++++--------- 14 files changed, 86 insertions(+), 83 deletions(-) diff --git a/.gitignore b/.gitignore index 84ac4951c..58b6099e1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ /api/node_modules /assets /assets.tar.gz -/env +/env* /scripts/gen /scripts/continuserv/continuserv /scripts/speculator/speculator diff --git a/README.rst b/README.rst index 6c87201c4..c0949d9e6 100644 --- a/README.rst +++ b/README.rst @@ -41,7 +41,7 @@ specs and event schemas in this repository. Preparation ----------- -To use the scripts, it is best to create a Python 2.x virtualenv as follows:: +To use the scripts, it is best to create a Python 3.5+ virtualenv as follows:: virtualenv env env/bin/pip install -r scripts/requirements.txt diff --git a/api/check_examples.py b/api/check_examples.py index be0676bb4..3ecd5a4c6 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -88,9 +88,9 @@ def check_swagger_file(filepath): with open(filepath) as f: swagger = yaml.load(f) - for path, path_api in swagger.get('paths', {}).items(): + for path, path_api in list(swagger.get('paths', {}).items()): - for method, request_api in path_api.items(): + for method, request_api in list(path_api.items()): request = "%s %s" % (method.upper(), path) for parameter in request_api.get('parameters', ()): if parameter['in'] == 'body': @@ -100,7 +100,7 @@ def check_swagger_file(filepath): responses = request_api['responses'] except KeyError: raise ValueError("No responses for %r" % (request,)) - for code, response in responses.items(): + for code, response in list(responses.items()): check_response(filepath, request, code, response) diff --git a/scripts/dump-swagger.py b/scripts/dump-swagger.py index 110c44466..6dd9864f9 100755 --- a/scripts/dump-swagger.py +++ b/scripts/dump-swagger.py @@ -94,16 +94,16 @@ for filename in os.listdir(cs_api_dir): api = units.resolve_references(filepath, api) basePath = api['basePath'] - for path, methods in api["paths"].items(): + for path, methods in list(api["paths"].items()): path = (basePath + path).replace('%CLIENT_MAJOR_VERSION%', major_version) - for method, spec in methods.items(): - if "tags" in spec.keys(): + for method, spec in list(methods.items()): + if "tags" in list(spec.keys()): if path not in output["paths"]: output["paths"][path] = {} output["paths"][path][method] = spec -print "Generating %s" % output_file +print("Generating %s" % output_file) try: os.makedirs(os.path.dirname(output_file)) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 2b35f8015..0c5486114 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -31,6 +31,7 @@ script_dir = os.path.dirname(os.path.abspath(__file__)) docs_dir = os.path.dirname(script_dir) spec_dir = os.path.join(docs_dir, "specification") tmp_dir = os.path.join(script_dir, "tmp") +changelog_dir = os.path.join(docs_dir, "changelogs") VERBOSE = False @@ -151,7 +152,7 @@ def is_title_line(prev_line, line, title_styles): def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): # string are file paths to RST blobs - if isinstance(file_info, basestring): + if isinstance(file_info, str): log("%s %s" % (">" * (1 + title_level), file_info)) with open(os.path.join(spec_dir, file_info), "r") as f: rst = None @@ -194,7 +195,7 @@ def build_spec(target, out_filename): spec_dir=spec_dir, adjust_titles=True ) - outfile.write(section) + outfile.write(section.encode('UTF-8')) """ @@ -279,15 +280,16 @@ def rst2html(i, o, stylesheets): def addAnchors(path): log("add anchors %s" % path) - with open(path, "r") as f: + with open(path, "rb") as f: lines = f.readlines() replacement = replacement = r'

\n\1' - with open(path, "w") as f: + with open(path, "wb") as f: for line in lines: + line = line.decode("UTF-8") line = re.sub(r'()', replacement, line.rstrip()) line = re.sub(r'(
)', replacement, line.rstrip()) - f.write(line + "\n") + f.write((line + "\n").encode('UTF-8')) def run_through_template(input_files, set_verbose, substitutions): @@ -297,7 +299,7 @@ def run_through_template(input_files, set_verbose, substitutions): "-i", "matrix_templates", ] - for k, v in substitutions.items(): + for k, v in list(substitutions.items()): args.append("--substitution=%s=%s" % (k, v)) if set_verbose: @@ -357,14 +359,14 @@ def get_build_target(all_targets, target_name): for i, entry in enumerate(group): if isinstance(entry, dict): group[i] = { - (rel_depth + depth): v for (rel_depth, v) in entry.items() + (rel_depth + depth): v for (rel_depth, v) in list(entry.items()) } return group resolved_files = [] for file_entry in target["files"]: # file_entry is a group id - if isinstance(file_entry, basestring) and file_entry.startswith("group:"): + if isinstance(file_entry, str) and file_entry.startswith("group:"): group = get_group(file_entry, 0) # The group may be resolved to a list of file entries, in which case # we want to extend the array to insert each of them rather than @@ -376,8 +378,8 @@ def get_build_target(all_targets, target_name): # file_entry is a dict which has more file entries as values elif isinstance(file_entry, dict): resolved_entry = {} - for (depth, entry) in file_entry.iteritems(): - if not isinstance(entry, basestring): + for (depth, entry) in list(file_entry.items()): + if not isinstance(entry, str): raise Exception( "Double-nested depths are not supported. Entry: %s" % (file_entry,) ) @@ -395,11 +397,11 @@ def get_build_target(all_targets, target_name): return build_target def log(line): - print "gendoc: %s" % line + print("gendoc: %s" % line) def logv(line): if VERBOSE: - print "gendoc:V: %s" % line + print("gendoc:V: %s" % line) def cleanup_env(): @@ -427,7 +429,7 @@ def main(targets, dest_dir, keep_intermediates, substitutions): target_defs = yaml.load(targ_file.read()) if targets == ["all"]: - targets = target_defs["targets"].keys() + targets = list(target_defs["targets"].keys()) log("Building spec [targets=%s]" % targets) @@ -441,17 +443,17 @@ def main(targets, dest_dir, keep_intermediates, substitutions): templated_files[target_name] = templated_file # we do all the templating at once, because it's slow - run_through_template(templated_files.values(), VERBOSE, substitutions) + run_through_template(list(templated_files.values()), VERBOSE, substitutions) stylesheets = glob.glob(os.path.join(script_dir, "css", "*.css")) - for target_name, templated_file in templated_files.iteritems(): + for target_name, templated_file in list(templated_files.items()): target = target_defs["targets"].get(target_name) version_label = None if target: version_label = target.get("version_label") if version_label: - for old, new in substitutions.items(): + for old, new in list(substitutions.items()): version_label = version_label.replace(old, new) rst_file = os.path.join(tmp_dir, "spec_%s.rst" % (target_name,)) @@ -479,8 +481,8 @@ def main(targets, dest_dir, keep_intermediates, substitutions): def list_targets(): with open(os.path.join(spec_dir, "targets.yaml"), "r") as targ_file: target_defs = yaml.load(targ_file.read()) - targets = target_defs["targets"].keys() - print "\n".join(targets) + targets = list(target_defs["targets"].keys()) + print("\n".join(targets)) def extract_major(s): diff --git a/scripts/proposals.py b/scripts/proposals.py index b489f0d27..156d53518 100755 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -14,7 +14,7 @@ prs = set() def getpage(url, page): resp = requests.get(url + str(page)) - for link in resp.links.values(): + for link in list(resp.links.values()): if link['rel'] == 'last': pagecount = re.search('page=(.+?)', link['url']).group(1) diff --git a/scripts/swagger-http-server.py b/scripts/swagger-http-server.py index 5ec001019..06d764aa0 100755 --- a/scripts/swagger-http-server.py +++ b/scripts/swagger-http-server.py @@ -19,14 +19,14 @@ import argparse import os -import SimpleHTTPServer -import SocketServer +import http.server +import socketserver # Thanks to http://stackoverflow.com/a/13354482 -class MyHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): +class MyHTTPRequestHandler(http.server.SimpleHTTPRequestHandler): def end_headers(self): self.send_my_headers() - SimpleHTTPServer.SimpleHTTPRequestHandler.end_headers(self) + http.server.SimpleHTTPRequestHandler.end_headers(self) def send_my_headers(self): self.send_header("Access-Control-Allow-Origin", "*") @@ -49,7 +49,7 @@ if __name__ == '__main__': os.chdir(args.swagger_dir) - httpd = SocketServer.TCPServer(("localhost", args.port), + httpd = socketserver.TCPServer(("localhost", args.port), MyHTTPRequestHandler) - print "Serving at http://localhost:%i/api-docs.json" % args.port + print("Serving at http://localhost:%i/api-docs.json" % args.port) httpd.serve_forever() diff --git a/scripts/templating/batesian/__init__.py b/scripts/templating/batesian/__init__.py index da41b31bf..f5d83a1ed 100644 --- a/scripts/templating/batesian/__init__.py +++ b/scripts/templating/batesian/__init__.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from sets import Set class AccessKeyStore(object): @@ -22,10 +21,10 @@ class AccessKeyStore(object): if not existing_data: existing_data = {} self.data = existing_data - self.accessed_set = Set() + self.accessed_set = set() def keys(self): - return self.data.keys() + return list(self.data.keys()) def add(self, key, unit_dict): self.data[key] = unit_dict @@ -35,5 +34,5 @@ class AccessKeyStore(object): return self.data[key] def get_unaccessed_set(self): - data_list = Set(self.data.keys()) + data_list = set(self.data.keys()) return data_list - self.accessed_set \ No newline at end of file diff --git a/scripts/templating/batesian/sections.py b/scripts/templating/batesian/sections.py index c541d771d..2e5593d64 100644 --- a/scripts/templating/batesian/sections.py +++ b/scripts/templating/batesian/sections.py @@ -29,7 +29,7 @@ class Sections(object): def log(self, text): if self.debug: - print "batesian:sections: %s" % text + print("batesian:sections: %s" % text) def get_sections(self): render_list = inspect.getmembers(self, predicate=inspect.ismethod) @@ -40,7 +40,7 @@ class Sections(object): section_key = func_name[len("render_"):] self.log("Generating section '%s'" % section_key) section = func() - if isinstance(section, basestring): + if isinstance(section, str): if section_key in section_dict: raise Exception( ("%s : Section %s already exists. It must have been " + @@ -54,8 +54,8 @@ class Sections(object): ) elif isinstance(section, dict): self.log(" Generated multiple sections:") - for (k, v) in section.iteritems(): - if not isinstance(k, basestring) or not isinstance(v, basestring): + for (k, v) in list(section.items()): + if not isinstance(k, str) or not isinstance(v, str): raise Exception( ("Method %s returned multiple sections as a dict but " + "expected the dict elements to be strings but they aren't.") % diff --git a/scripts/templating/batesian/units.py b/scripts/templating/batesian/units.py index 8f748f6da..82cc52f92 100644 --- a/scripts/templating/batesian/units.py +++ b/scripts/templating/batesian/units.py @@ -41,7 +41,7 @@ class Units(object): trace = inspect.stack() if len(trace) > 1 and len(trace[1]) > 2: func_name = trace[1][3] + ":" - print "batesian:units:%s %s" % (func_name, text) + print("batesian:units:%s %s" % (func_name, text)) def get_units(self, debug=False): unit_list = inspect.getmembers(self, predicate=inspect.ismethod) @@ -50,7 +50,7 @@ class Units(object): if not func_name.startswith("load_"): continue unit_key = func_name[len("load_"):] - if len(inspect.getargs(func.func_code).args) > 1: + if len(inspect.getargs(func.__code__).args) > 1: unit_dict[unit_key] = func(self.substitutions) else: unit_dict[unit_key] = func() diff --git a/scripts/templating/build.py b/scripts/templating/build.py index d18569b66..de9ead35f 100755 --- a/scripts/templating/build.py +++ b/scripts/templating/build.py @@ -63,6 +63,7 @@ import sys from textwrap import TextWrapper from matrix_templates.units import TypeTableRow +from functools import reduce def create_from_template(template, sections): @@ -138,7 +139,7 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={} return reduce(max, rowwidths, default if default is not None else default_width) - results = map(colwidth, keys, defaults) + results = list(map(colwidth, keys, defaults)) return results # make Jinja aware of the templates and filters @@ -167,16 +168,16 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={} # print out valid section keys if no file supplied if not files: - print "\nValid template variables:" - for key in sections.keys(): + print("\nValid template variables:") + for key in list(sections.keys()): sec_text = "" if (len(sections[key]) > 75) else ( "(Value: '%s')" % sections[key] ) sec_info = "%s characters" % len(sections[key]) if sections[key].count("\n") > 0: sec_info += ", %s lines" % sections[key].count("\n") - print " %s" % key - print " %s %s" % (sec_info, sec_text) + print(" %s" % key) + print(" %s %s" % (sec_info, sec_text)) return # check the input files and substitute in sections where required @@ -190,8 +191,8 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={} def process_file(env, sections, filename, output_filename): log("Parsing input template: %s" % filename) - with open(filename, "r") as file_stream: - temp_str = file_stream.read().decode("utf-8") + with open(filename, "rb") as file_stream: + temp_str = file_stream.read().decode('UTF-8') # do sanity checking on the template to make sure they aren't reffing things # which will never be replaced with a section. @@ -210,16 +211,16 @@ def process_file(env, sections, filename, output_filename): # Do these substitutions outside of the ordinary templating system because # we want them to apply to things like the underlying swagger used to # generate the templates, not just the top-level sections. - for old, new in substitutions.items(): + for old, new in list(substitutions.items()): output = output.replace(old, new) - with open(output_filename, "w") as f: - f.write(output.encode("utf-8")) + with open(output_filename, "wb") as f: + f.write(output.encode('UTF-8')) log("Output file for: %s" % output_filename) def log(line): - print "batesian: %s" % line + print("batesian: %s" % line) if __name__ == '__main__': parser = ArgumentParser( diff --git a/scripts/templating/matrix_templates/__init__.py b/scripts/templating/matrix_templates/__init__.py index 6b46192ca..b81c5a30d 100644 --- a/scripts/templating/matrix_templates/__init__.py +++ b/scripts/templating/matrix_templates/__init__.py @@ -11,8 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from sections import MatrixSections -from units import MatrixUnits +from .sections import MatrixSections +from .units import MatrixUnits import os exports = { diff --git a/scripts/templating/matrix_templates/sections.py b/scripts/templating/matrix_templates/sections.py index 64e32aa44..71c1acf35 100644 --- a/scripts/templating/matrix_templates/sections.py +++ b/scripts/templating/matrix_templates/sections.py @@ -86,7 +86,7 @@ class MatrixSections(Sections): # the key is the section name and the value is the value of the section def render_group_http_apis(self): # map all swagger_apis to the form $GROUP_http_api - swagger_groups = self.units.get("swagger_apis").keys() + swagger_groups = list(self.units.get("swagger_apis").keys()) renders = {} for group in swagger_groups: sortFnOrPathList = None @@ -134,7 +134,7 @@ class MatrixSections(Sections): "m.room.message#m.file" ] other_msgtypes = [ - k for k in schemas.keys() if k.startswith("m.room.message#") and + k for k in list(schemas.keys()) if k.startswith("m.room.message#") and k not in msgtype_order ] for event_name in (msgtype_order + other_msgtypes): diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index dac183d1f..40fd96a23 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -29,11 +29,12 @@ import os.path import re import subprocess import sys -import urllib +import urllib.request, urllib.parse, urllib.error import yaml +from functools import reduce matrix_doc_dir=reduce(lambda acc,_: os.path.dirname(acc), - range(1, 5), os.path.abspath(__file__)) + list(range(1, 5)), os.path.abspath(__file__)) HTTP_APIS = { os.path.join(matrix_doc_dir, "api/application-service"): "as", @@ -124,7 +125,7 @@ def resolve_references(path, schema): else: result = OrderedDict() - for key, value in schema.items(): + for key, value in list(schema.items()): result[key] = resolve_references(path, value) return result elif isinstance(schema, list): @@ -147,7 +148,7 @@ def inherit_parents(obj): # settings defined in the child take priority over the parents, so we # iterate through the parents first, and then overwrite with the settings # from the child. - for p in map(inherit_parents, parents) + [obj]: + for p in list(map(inherit_parents, parents)) + [obj]: # child blats out type, title and description for key in ('type', 'title', 'description'): if p.get(key): @@ -209,7 +210,7 @@ def get_json_schema_object_fields(obj, enforce_title=False): props = obj.get("patternProperties") if props: # try to replace horrible regex key names with pretty x-pattern ones - for key_name in props.keys(): + for key_name in list(props.keys()): pretty_key = props[key_name].get("x-pattern") if pretty_key: props[pretty_key] = props[key_name] @@ -250,12 +251,12 @@ def get_json_schema_object_fields(obj, enforce_title=False): tables.extend(res["tables"]) logger.debug("Done property %s" % key_name) - except Exception, e: + except Exception as e: e2 = Exception("Error reading property %s.%s: %s" % (obj_title, key_name, str(e))) # throw the new exception with the old stack trace, so that # we don't lose information about where the error occurred. - raise e2, None, sys.exc_info()[2] + raise e2.with_traceback(sys.exc_info()[2]) tables.insert(0, TypeTable(title=obj_title, rows=first_table_rows)) @@ -380,7 +381,7 @@ def get_example_for_schema(schema): if 'properties' not in schema: raise Exception('"object" property has neither properties nor example') res = OrderedDict() - for prop_name, prop in schema['properties'].iteritems(): + for prop_name, prop in list(schema['properties'].items()): logger.debug("Parsing property %r" % prop_name) prop_example = get_example_for_schema(prop) res[prop_name] = prop_example @@ -523,7 +524,7 @@ class MatrixUnits(Units): if param_loc == "path": path_template = path_template.replace( - "{%s}" % param_name, urllib.quote(example) + "{%s}" % param_name, urllib.parse.quote(example) ) elif param_loc == "query": if type(example) == list: @@ -532,7 +533,7 @@ class MatrixUnits(Units): else: example_query_params.append((param_name, example)) - except Exception, e: + except Exception as e: raise Exception("Error handling parameter %s" % param_name, e) # endfor[param] good_response = None @@ -556,14 +557,14 @@ class MatrixUnits(Units): ) if "headers" in good_response: headers = TypeTable() - for (header_name, header) in good_response["headers"].iteritems(): + for (header_name, header) in list(good_response["headers"].items()): headers.add_row( TypeTableRow(key=header_name, title=header["type"], desc=header["description"]), ) endpoint["res_headers"] = headers query_string = "" if len( - example_query_params) == 0 else "?" + urllib.urlencode( + example_query_params) == 0 else "?" + urllib.parse.urlencode( example_query_params) if example_body: endpoint["example"][ @@ -605,17 +606,17 @@ class MatrixUnits(Units): body_tables = req_body_tables[1:] endpoint_data['req_body_tables'].extend(body_tables) - except Exception, e: + except Exception as e: e2 = Exception( "Error decoding body of API endpoint %s %s: %s" % (endpoint_data["method"], endpoint_data["path"], e) ) - raise e2, None, sys.exc_info()[2] + raise e2.with_traceback(sys.exc_info()[2]) def load_swagger_apis(self): apis = {} - for path, suffix in HTTP_APIS.items(): + for path, suffix in list(HTTP_APIS.items()): for filename in os.listdir(path): if not filename.endswith(".yaml"): continue @@ -711,12 +712,12 @@ class MatrixUnits(Units): if filename != event_name: examples[event_name] = examples.get(event_name, []) examples[event_name].append(example) - except Exception, e: + except Exception as e: e2 = Exception("Error reading event example "+filepath+": "+ str(e)) # throw the new exception with the old stack trace, so that # we don't lose information about where the error occurred. - raise e2, None, sys.exc_info()[2] + raise e2.with_traceback(sys.exc_info()[2]) return examples @@ -730,12 +731,12 @@ class MatrixUnits(Units): filepath = os.path.join(path, filename) try: schemata[filename] = self.read_event_schema(filepath) - except Exception, e: + except Exception as e: e2 = Exception("Error reading event schema "+filepath+": "+ str(e)) # throw the new exception with the old stack trace, so that # we don't lose information about where the error occurred. - raise e2, None, sys.exc_info()[2] + raise e2.with_traceback(sys.exc_info()[2]) return schemata @@ -871,7 +872,7 @@ class MatrixUnits(Units): ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], stderr=null, cwd=cwd, - ).strip() + ).strip().decode('UTF-8') except subprocess.CalledProcessError: git_branch = "" try: @@ -879,7 +880,7 @@ class MatrixUnits(Units): ['git', 'describe', '--exact-match'], stderr=null, cwd=cwd, - ).strip() + ).strip().decode('UTF-8') git_tag = "tag=" + git_tag except subprocess.CalledProcessError: git_tag = "" @@ -888,7 +889,7 @@ class MatrixUnits(Units): ['git', 'rev-parse', '--short', 'HEAD'], stderr=null, cwd=cwd, - ).strip() + ).strip().decode('UTF-8') except subprocess.CalledProcessError: git_commit = "" try: @@ -897,7 +898,7 @@ class MatrixUnits(Units): ['git', 'describe', '--dirty=' + dirty_string, "--all"], stderr=null, cwd=cwd, - ).strip().endswith(dirty_string) + ).strip().decode('UTF-8').endswith(dirty_string) git_dirty = "dirty" if is_dirty else "" except subprocess.CalledProcessError: git_dirty = "" @@ -908,7 +909,7 @@ class MatrixUnits(Units): s for s in (git_branch, git_tag, git_commit, git_dirty,) if s - ).encode("ascii") + ).encode("ascii").decode('ascii') return { "string": git_version, "revision": git_commit From 9277e4c52d70472cd58d7b2bf0ec6329e8228b8a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 16:32:27 -0600 Subject: [PATCH 063/123] Add initial towncrier support --- .gitignore | 1 + changelogs/client_server/pyproject.toml | 30 ++++++++++++++++++++ scripts/requirements.txt | 1 + scripts/templating/matrix_templates/units.py | 23 +++++++++++++-- 4 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 changelogs/client_server/pyproject.toml diff --git a/.gitignore b/.gitignore index 58b6099e1..a850d2faa 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ /templating/out *.pyc *.swp +_rendered.rst diff --git a/changelogs/client_server/pyproject.toml b/changelogs/client_server/pyproject.toml new file mode 100644 index 000000000..d5daface6 --- /dev/null +++ b/changelogs/client_server/pyproject.toml @@ -0,0 +1,30 @@ +[tool.towncrier] + filename = "_rendered.rst" + directory = "newsfragments" + issue_format = "`#{issue} `_" + title_format = "{version}" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 37d99dd3d..6f7951d62 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -7,3 +7,4 @@ Jinja2 >= 2.9.6 jsonschema >= 2.6.0 PyYAML >= 3.12 requests >= 2.18.4 +towncrier == 18.6.0rc1 diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 40fd96a23..0dbf81047 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -832,12 +832,28 @@ class MatrixUnits(Units): path = os.path.join(CHANGELOG_DIR, f) name = f[:-4] + # If there's a directory with the same name, we'll try to generate + # a towncrier changelog and prepend it to the general changelog. + tc_path = os.path.join(CHANGELOG_DIR, name) + tc_lines = [] + if os.path.isdir(tc_path): + logger.info("Generating towncrier changelog for: %s" % name) + try: + raw_log = subprocess.check_output( + ['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'], + stderr=subprocess.PIPE, + cwd=tc_path, + ).strip().decode('UTF-8') + except subprocess.CalledProcessError: + raw_log = "" + tc_lines = raw_log.splitlines() + title_part = None changelog_lines = [] with open(path, "r") as f: lines = f.readlines() prev_line = None - for line in lines: + for line in (tc_lines + lines): if prev_line is None: prev_line = line continue @@ -853,7 +869,10 @@ class MatrixUnits(Units): # then bail out. changelog_lines.pop() break - changelog_lines.append(" " + line) + # Don't generate subheadings (we'll keep the title though) + if re.match("^[-]{3,}$", line.strip()): + continue + changelog_lines.append(" " + line + '\n') changelogs[name] = "".join(changelog_lines) return changelogs From ee9abc5fd76c4e8e62a9a737fbf4ea33afddead5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 16:41:54 -0600 Subject: [PATCH 064/123] Convert unstable changelog to towncrier --- changelogs/client_server.rst | 70 ------------------- .../client_server/newsfragments/1096.new | 1 + .../newsfragments/1097.clarification | 1 + .../client_server/newsfragments/1104.feature | 1 + .../newsfragments/1106.clarification | 1 + .../newsfragments/1109.clarification | 1 + .../client_server/newsfragments/1110.new | 1 + .../newsfragments/1137.clarification | 1 + .../newsfragments/1139.clarification | 1 + .../client_server/newsfragments/1141.feature | 1 + .../client_server/newsfragments/1142.feature | 1 + .../client_server/newsfragments/1151.feature | 1 + .../newsfragments/1152.clarification | 1 + .../newsfragments/1158.clarification | 3 + .../client_server/newsfragments/1158.feature | 1 + .../client_server/newsfragments/1239.new | 1 + .../newsfragments/1245.clarification | 1 + .../client_server/newsfragments/1263.feature | 1 + .../client_server/newsfragments/1264.feature | 1 + .../client_server/newsfragments/1265.feature | 1 + .../client_server/newsfragments/1274.feature | 1 + .../newsfragments/1329.clarification | 1 + .../client_server/newsfragments/1361.feature | 1 + .../newsfragments/1362.clarification | 1 + .../client_server/newsfragments/1364.feature | 1 + .../newsfragments/1378.clarification | 1 + .../newsfragments/1380.clarification | 1 + .../newsfragments/1381.clarification | 1 + 28 files changed, 29 insertions(+), 70 deletions(-) create mode 100644 changelogs/client_server/newsfragments/1096.new create mode 100644 changelogs/client_server/newsfragments/1097.clarification create mode 100644 changelogs/client_server/newsfragments/1104.feature create mode 100644 changelogs/client_server/newsfragments/1106.clarification create mode 100644 changelogs/client_server/newsfragments/1109.clarification create mode 100644 changelogs/client_server/newsfragments/1110.new create mode 100644 changelogs/client_server/newsfragments/1137.clarification create mode 100644 changelogs/client_server/newsfragments/1139.clarification create mode 100644 changelogs/client_server/newsfragments/1141.feature create mode 100644 changelogs/client_server/newsfragments/1142.feature create mode 100644 changelogs/client_server/newsfragments/1151.feature create mode 100644 changelogs/client_server/newsfragments/1152.clarification create mode 100644 changelogs/client_server/newsfragments/1158.clarification create mode 100644 changelogs/client_server/newsfragments/1158.feature create mode 100644 changelogs/client_server/newsfragments/1239.new create mode 100644 changelogs/client_server/newsfragments/1245.clarification create mode 100644 changelogs/client_server/newsfragments/1263.feature create mode 100644 changelogs/client_server/newsfragments/1264.feature create mode 100644 changelogs/client_server/newsfragments/1265.feature create mode 100644 changelogs/client_server/newsfragments/1274.feature create mode 100644 changelogs/client_server/newsfragments/1329.clarification create mode 100644 changelogs/client_server/newsfragments/1361.feature create mode 100644 changelogs/client_server/newsfragments/1362.clarification create mode 100644 changelogs/client_server/newsfragments/1364.feature create mode 100644 changelogs/client_server/newsfragments/1378.clarification create mode 100644 changelogs/client_server/newsfragments/1380.clarification create mode 100644 changelogs/client_server/newsfragments/1381.clarification diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 32654025b..d2ef7f2db 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -1,73 +1,3 @@ -Unreleased changes -================== - -- Changes to the API which will be backwards-compatible for clients: - - - New endpoints: - - - ``POST /user_directory/search`` - (`#1096 `_). - - ``GET /rooms/{roomId}/event/{eventId}`` - (`#1110 `_). - - ``POST /delete_devices`` - (`#1239 `_). - - - Sticker messages: - - Add sticker message event definition. - (`#1158 `_). - - Document the ``server_name`` parameter on ``/join/{roomIdOrAlias}`` - (`#1364 `_). - - Add 'token' parameter to /keys/query endpoint - (`#1104 `_). - - Add the room visibility options for the room directory - (`#1141 `_). - - Add spec for ignoring users - (`#1142 `_). - - Add the ``/register/available`` endpoint for username availability - (`#1151 `_). - - Add ``allow_remote`` to the content repo to avoid routing loops - (`#1265 `_). - - Add report content API - (`#1264 `_). - - Document ``/logout/all`` endpoint - (`#1263 `_). - - Document `highlights` field in /search response - (`#1274 `_). - - Document the GET version of ``/login`` - (`#1361 `_). - -- Spec clarifications: - - - Update ``ImageInfo`` and ``ThumbnailInfo`` dimension schema descriptions - to clarify that they relate to intended display size, as opposed to the - intrinsic size of the image file. - (`#1158 `_). - - Mark ``home_server`` return field for ``/login`` and ``/register`` - endpoints as deprecated - (`#1097 `_). - - Fix response format of ``/keys/changes`` endpoint - (`#1106 `_). - - Clarify default values for some fields on the /search API - (`#1109 `_). - - Fix the representation of ``m.presence`` events - (`#1137 `_). - - Clarify that ``m.tag`` ordering is done with numbers, not strings - (`#1139 `_). - - Clarify that ``/account/whoami`` should consider application services - (`#1152 `_). - - Mark ``GET /rooms/{roomId}/members`` as requiring authentication - (`#1245 `_). - - Define what a ``RoomEvent`` is on ``/rooms/{roomId}/messages`` - (`#1380 `_). - - Describe ``StateEvent`` for ``/createRoom`` - (`#1329 `_). - - Describe how the ``reason`` is handled for kicks/bans - (`#1362 `_). - - Clarify that clients must leave rooms before forgetting them - (`#1378 `_). - - Clarify the request and result types on ``/search`` - (`#1381 `_). - r0.3.0 ====== diff --git a/changelogs/client_server/newsfragments/1096.new b/changelogs/client_server/newsfragments/1096.new new file mode 100644 index 000000000..50d868799 --- /dev/null +++ b/changelogs/client_server/newsfragments/1096.new @@ -0,0 +1 @@ +``POST /user_directory/search`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1097.clarification b/changelogs/client_server/newsfragments/1097.clarification new file mode 100644 index 000000000..2a7cb93a5 --- /dev/null +++ b/changelogs/client_server/newsfragments/1097.clarification @@ -0,0 +1 @@ +Mark ``home_server`` return field for ``/login`` and ``/register`` endpoints as deprecated \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1104.feature b/changelogs/client_server/newsfragments/1104.feature new file mode 100644 index 000000000..9b85343f7 --- /dev/null +++ b/changelogs/client_server/newsfragments/1104.feature @@ -0,0 +1 @@ +Add ``token`` parameter to the ``/keys/query`` endpoint \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1106.clarification b/changelogs/client_server/newsfragments/1106.clarification new file mode 100644 index 000000000..f7a1fe3e7 --- /dev/null +++ b/changelogs/client_server/newsfragments/1106.clarification @@ -0,0 +1 @@ +Fix response format of ``/keys/changes`` endpoint \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1109.clarification b/changelogs/client_server/newsfragments/1109.clarification new file mode 100644 index 000000000..176d94030 --- /dev/null +++ b/changelogs/client_server/newsfragments/1109.clarification @@ -0,0 +1 @@ +Clarify default values for some fields on the ``/search`` API \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1110.new b/changelogs/client_server/newsfragments/1110.new new file mode 100644 index 000000000..e1b80b8b6 --- /dev/null +++ b/changelogs/client_server/newsfragments/1110.new @@ -0,0 +1 @@ +``GET /rooms/{roomId}/event/{eventId}`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1137.clarification b/changelogs/client_server/newsfragments/1137.clarification new file mode 100644 index 000000000..5ad8ec34f --- /dev/null +++ b/changelogs/client_server/newsfragments/1137.clarification @@ -0,0 +1 @@ +Fix the representation of ``m.presence`` events \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1139.clarification b/changelogs/client_server/newsfragments/1139.clarification new file mode 100644 index 000000000..b5193ad3c --- /dev/null +++ b/changelogs/client_server/newsfragments/1139.clarification @@ -0,0 +1 @@ +Clarify that ``m.tag`` ordering is done with numbers, not strings \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1141.feature b/changelogs/client_server/newsfragments/1141.feature new file mode 100644 index 000000000..da0418191 --- /dev/null +++ b/changelogs/client_server/newsfragments/1141.feature @@ -0,0 +1 @@ +Add the room visibility options for the room directory \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1142.feature b/changelogs/client_server/newsfragments/1142.feature new file mode 100644 index 000000000..0a0842c42 --- /dev/null +++ b/changelogs/client_server/newsfragments/1142.feature @@ -0,0 +1 @@ +Add spec for ignoring users \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1151.feature b/changelogs/client_server/newsfragments/1151.feature new file mode 100644 index 000000000..8875812bd --- /dev/null +++ b/changelogs/client_server/newsfragments/1151.feature @@ -0,0 +1 @@ +Add the ``/register/available`` endpoint for username availability \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1152.clarification b/changelogs/client_server/newsfragments/1152.clarification new file mode 100644 index 000000000..bbecc9b29 --- /dev/null +++ b/changelogs/client_server/newsfragments/1152.clarification @@ -0,0 +1 @@ +Clarify that ``/account/whoami`` should consider application services \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1158.clarification b/changelogs/client_server/newsfragments/1158.clarification new file mode 100644 index 000000000..dc1f6d149 --- /dev/null +++ b/changelogs/client_server/newsfragments/1158.clarification @@ -0,0 +1,3 @@ +Update ``ImageInfo`` and ``ThumbnailInfo`` dimension schema descriptions +to clarify that they relate to intended display size, as opposed to the +intrinsic size of the image file. \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1158.feature b/changelogs/client_server/newsfragments/1158.feature new file mode 100644 index 000000000..a55df4fb3 --- /dev/null +++ b/changelogs/client_server/newsfragments/1158.feature @@ -0,0 +1 @@ +Add sticker messages \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1239.new b/changelogs/client_server/newsfragments/1239.new new file mode 100644 index 000000000..9bcf357df --- /dev/null +++ b/changelogs/client_server/newsfragments/1239.new @@ -0,0 +1 @@ +``POST /delete_devices`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1245.clarification b/changelogs/client_server/newsfragments/1245.clarification new file mode 100644 index 000000000..e0a638345 --- /dev/null +++ b/changelogs/client_server/newsfragments/1245.clarification @@ -0,0 +1 @@ +Mark ``GET /rooms/{roomId}/members`` as requiring authentication \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1263.feature b/changelogs/client_server/newsfragments/1263.feature new file mode 100644 index 000000000..04964a7d6 --- /dev/null +++ b/changelogs/client_server/newsfragments/1263.feature @@ -0,0 +1 @@ +Document ``/logout/all`` endpoint \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1264.feature b/changelogs/client_server/newsfragments/1264.feature new file mode 100644 index 000000000..9cb06f719 --- /dev/null +++ b/changelogs/client_server/newsfragments/1264.feature @@ -0,0 +1 @@ +Add report content API \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1265.feature b/changelogs/client_server/newsfragments/1265.feature new file mode 100644 index 000000000..1e270fa8b --- /dev/null +++ b/changelogs/client_server/newsfragments/1265.feature @@ -0,0 +1 @@ +Add ``allow_remote`` to the content repo to avoid routing loops \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1274.feature b/changelogs/client_server/newsfragments/1274.feature new file mode 100644 index 000000000..d49581314 --- /dev/null +++ b/changelogs/client_server/newsfragments/1274.feature @@ -0,0 +1 @@ +Document `highlights` field in /search response \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1329.clarification b/changelogs/client_server/newsfragments/1329.clarification new file mode 100644 index 000000000..970d3d947 --- /dev/null +++ b/changelogs/client_server/newsfragments/1329.clarification @@ -0,0 +1 @@ +Describe ``StateEvent`` for ``/createRoom`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1361.feature b/changelogs/client_server/newsfragments/1361.feature new file mode 100644 index 000000000..b1d4c2f11 --- /dev/null +++ b/changelogs/client_server/newsfragments/1361.feature @@ -0,0 +1 @@ +Document the GET version of ``/login`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1362.clarification b/changelogs/client_server/newsfragments/1362.clarification new file mode 100644 index 000000000..1deb45002 --- /dev/null +++ b/changelogs/client_server/newsfragments/1362.clarification @@ -0,0 +1 @@ +Describe how the ``reason`` is handled for kicks/bans \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1364.feature b/changelogs/client_server/newsfragments/1364.feature new file mode 100644 index 000000000..733d6a1f0 --- /dev/null +++ b/changelogs/client_server/newsfragments/1364.feature @@ -0,0 +1 @@ +Document the ``server_name`` parameter on ``/join/{roomIdOrAlias}`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1378.clarification b/changelogs/client_server/newsfragments/1378.clarification new file mode 100644 index 000000000..f952428b5 --- /dev/null +++ b/changelogs/client_server/newsfragments/1378.clarification @@ -0,0 +1 @@ +Clarify that clients must leave rooms before forgetting them \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1380.clarification b/changelogs/client_server/newsfragments/1380.clarification new file mode 100644 index 000000000..490a9a43c --- /dev/null +++ b/changelogs/client_server/newsfragments/1380.clarification @@ -0,0 +1 @@ +Define what a ``RoomEvent`` is on ``/rooms/{roomId}/messages`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1381.clarification b/changelogs/client_server/newsfragments/1381.clarification new file mode 100644 index 000000000..e5e599acf --- /dev/null +++ b/changelogs/client_server/newsfragments/1381.clarification @@ -0,0 +1 @@ +Clarify the request and result types on ``/search`` \ No newline at end of file From c2f85788ded8970cad38bc261314ae148c492e20 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 16:54:14 -0600 Subject: [PATCH 065/123] Add documentation about how to add to the changelog --- CONTRIBUTING.rst | 35 +++++++++++++++++++++++++++++++++++ scripts/gendoc.py | 3 ++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 2a7416b17..c592cf02b 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -69,6 +69,41 @@ For such changes, please do just open a `pull request`_. .. _pull request: https://help.github.com/articles/about-pull-requests + +Adding to the changelog +~~~~~~~~~~~~~~~~~~~~~~~ + +Currently only changes to the client-server API need to end up in a changelog. The +other APIs are not yet stable and therefore do not have a changelog. Adding to the +changelog can only be done after you've opened your pull request, so be sure to do +that first. + +The changelog is managed by Towncrier (https://github.com/hawkowl/towncrier) in the +form of "news fragments". The news fragments for the client-server API are stored +under ``changelogs/client_server/newsfragments``. + +To create a changelog entry, create a file named in the format ``prNumber.type`` in +the ``newsfragments`` directory. The ``type`` can be one of the following: + +* ``new`` - Used when adding new endpoints. Please have the file contents be the + method and route being added, surrounded in RST code tags. For example: ``POST + /accounts/whoami`` + +* ``feature`` - Used when adding backwards-compatible changes to the API. + +* ``clarification`` - Used when an area of the spec is being improved upon and does + not change or introduce any functionality. + +* ``breaking`` - Used when the change is not backwards compatible. + +* ``deprecation`` - Used when deprecating something + +All news fragments must have a brief summary explaining the change in the contents +of the file. + +Changes that do not change the spec, such as changes to the build script, formatting, +CSS, etc should not get a news fragment. + Sign off -------- diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 0c5486114..31ee9e696 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -405,7 +405,8 @@ def logv(line): def cleanup_env(): - shutil.rmtree(tmp_dir) + #shutil.rmtree(tmp_dir) + pass def mkdirp(d) : From 34089e302c8415a34f69a200f4aee689fd97a0c8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 17:06:08 -0600 Subject: [PATCH 066/123] Add general documentation for maintainers --- changelogs/README.md | 55 +++++++++++++++++++++++++ changelogs/client_server/pyproject.toml | 2 +- 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 changelogs/README.md diff --git a/changelogs/README.md b/changelogs/README.md new file mode 100644 index 000000000..10d69cb6b --- /dev/null +++ b/changelogs/README.md @@ -0,0 +1,55 @@ + + +# Changelogs + +[Towncrier](https://github.com/hawkowl/towncrier) is used to manage the changelog and +keep it up to date. Because of this, updating a changelog is really easy. + +## How to update a changelog when releasing an API + +1. Ensure you're in your Python 3 virtual environment +2. `cd` your way to the API you're releasing (eg: `cd changelogs/client_server`) +3. Run `towncrier --version "r0.4.0" --name "client-server" --yes` substituting the + variables as approprite. Note that `--name` is required although the value is ignored. +4. Commit the changes and finish the release process. + +## How to prepare a changelog for a new API + +For this example, we're going to pretend that the `server_server` API doesn't exist. + +1. Create the file `changelogs/server_server.rst` +2. Create the folder `changelogs/server_server` +3. In the new folder, create a `pyproject.toml` file with these contents: + ```toml + [tool.towncrier] + filename = "../server_server.rst" + directory = "newsfragments" + issue_format = "`#{issue} `_" + title_format = "{version}" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true + ``` +4. Use the changelog in whatever RST documents you need diff --git a/changelogs/client_server/pyproject.toml b/changelogs/client_server/pyproject.toml index d5daface6..8fa3f6b5a 100644 --- a/changelogs/client_server/pyproject.toml +++ b/changelogs/client_server/pyproject.toml @@ -1,5 +1,5 @@ [tool.towncrier] - filename = "_rendered.rst" + filename = "../client_server.rst" directory = "newsfragments" issue_format = "`#{issue} `_" title_format = "{version}" From 63b926a4aaae18293426cc10efa658a8644b7e42 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 17:19:04 -0600 Subject: [PATCH 067/123] Make Travis CI use python 3 --- .travis.yml | 4 ++++ scripts/test-and-build.sh | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7d84f237d..98dd14b83 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,10 @@ sudo: false git: depth: 1 +before_install: +- sudo apt-get update +- sudo apt-get install python3 + # test-and-build does the installation, so tell travis to skip the # installation step install: true diff --git a/scripts/test-and-build.sh b/scripts/test-and-build.sh index 7794f8264..e459a370a 100755 --- a/scripts/test-and-build.sh +++ b/scripts/test-and-build.sh @@ -4,7 +4,7 @@ set -ex cd `dirname $0`/.. -virtualenv env +virtualenv -p python3 env . env/bin/activate pip install -r scripts/requirements.txt From f68c82bd6c3d8959004ee564ae9dce6769aa99a8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 17:32:57 -0600 Subject: [PATCH 068/123] Don't let the newsfragments wink out of existence --- changelogs/README.md | 2 +- changelogs/client_server/newsfragments/.gitignore | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelogs/client_server/newsfragments/.gitignore diff --git a/changelogs/README.md b/changelogs/README.md index 10d69cb6b..a5fb1fb7d 100644 --- a/changelogs/README.md +++ b/changelogs/README.md @@ -52,4 +52,4 @@ For this example, we're going to pretend that the `server_server` API doesn't ex name = "Spec Clarifications" showcontent = true ``` -4. Use the changelog in whatever RST documents you need +4. Create a `.gitignore` in `changelogs/server_server/newsfragments` with the contents `!.gitignore` diff --git a/changelogs/client_server/newsfragments/.gitignore b/changelogs/client_server/newsfragments/.gitignore new file mode 100644 index 000000000..b722e9e13 --- /dev/null +++ b/changelogs/client_server/newsfragments/.gitignore @@ -0,0 +1 @@ +!.gitignore \ No newline at end of file From a4f8e0807acee23f58b2a9e4ae2b0b0ba9f21cfa Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 08:41:20 -0600 Subject: [PATCH 069/123] Print the python version being used in the CI build. --- scripts/test-and-build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/test-and-build.sh b/scripts/test-and-build.sh index e459a370a..710b03ddf 100755 --- a/scripts/test-and-build.sh +++ b/scripts/test-and-build.sh @@ -6,6 +6,11 @@ cd `dirname $0`/.. virtualenv -p python3 env . env/bin/activate + +# Print out the python versions for debugging purposes +python --version +pip --version + pip install -r scripts/requirements.txt # do sanity checks on the examples and swagger From 08f69128259bc3113614018e04b9342850388d2c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 08:50:26 -0600 Subject: [PATCH 070/123] Ask Travis to install 3.5 explicitly --- .travis.yml | 2 +- scripts/test-and-build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 98dd14b83..013efdd98 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ git: before_install: - sudo apt-get update -- sudo apt-get install python3 +- sudo apt-get install python3.5 # test-and-build does the installation, so tell travis to skip the # installation step diff --git a/scripts/test-and-build.sh b/scripts/test-and-build.sh index 710b03ddf..397b620ed 100755 --- a/scripts/test-and-build.sh +++ b/scripts/test-and-build.sh @@ -4,7 +4,7 @@ set -ex cd `dirname $0`/.. -virtualenv -p python3 env +virtualenv -p python3.5 env . env/bin/activate # Print out the python versions for debugging purposes From c65c6e2e70795dabd7717e195a72a39704e8a014 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 12:25:34 -0600 Subject: [PATCH 071/123] Add changelog entries removed in the merge from master --- changelogs/client_server/newsfragments/1371.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1371.clarification diff --git a/changelogs/client_server/newsfragments/1371.clarification b/changelogs/client_server/newsfragments/1371.clarification new file mode 100644 index 000000000..88552fcd5 --- /dev/null +++ b/changelogs/client_server/newsfragments/1371.clarification @@ -0,0 +1 @@ +Mark ``GET /presence/{userId}/status`` as requiring authentication \ No newline at end of file From 7ada96020677b9df88422ac38b2b82754d974321 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 12:28:27 -0600 Subject: [PATCH 072/123] Fix the Travis CI build to work on 3.5 --- .travis.yml | 9 +++------ scripts/requirements.txt | 1 + scripts/templating/matrix_templates/units.py | 7 ++++--- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 013efdd98..27e5cf202 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,13 +8,10 @@ sudo: false git: depth: 1 -before_install: +install: - sudo apt-get update -- sudo apt-get install python3.5 - -# test-and-build does the installation, so tell travis to skip the -# installation step -install: true +- sudo apt-get install python3.5 python3.5-dev +- sudo pip install --upgrade virtualenv script: - ./scripts/test-and-build.sh diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 6f7951d62..1f92c3dc9 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -8,3 +8,4 @@ jsonschema >= 2.6.0 PyYAML >= 3.12 requests >= 2.18.4 towncrier == 18.6.0rc1 +six >= 1.11.0 \ No newline at end of file diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 0dbf81047..acd85d66a 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -29,9 +29,10 @@ import os.path import re import subprocess import sys -import urllib.request, urllib.parse, urllib.error import yaml +import requests.utils from functools import reduce +from six.moves.urllib.parse import urlencode matrix_doc_dir=reduce(lambda acc,_: os.path.dirname(acc), list(range(1, 5)), os.path.abspath(__file__)) @@ -524,7 +525,7 @@ class MatrixUnits(Units): if param_loc == "path": path_template = path_template.replace( - "{%s}" % param_name, urllib.parse.quote(example) + "{%s}" % param_name, requests.utils.requote_uri(example) ) elif param_loc == "query": if type(example) == list: @@ -564,7 +565,7 @@ class MatrixUnits(Units): ) endpoint["res_headers"] = headers query_string = "" if len( - example_query_params) == 0 else "?" + urllib.parse.urlencode( + example_query_params) == 0 else "?" + urlencode( example_query_params) if example_body: endpoint["example"][ From 550f95570b327e8f57e832866427049603f48b06 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 15:17:28 -0600 Subject: [PATCH 073/123] Fix regex for some versions of python --- scripts/gendoc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 31ee9e696..25fd4577a 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -283,12 +283,12 @@ def addAnchors(path): with open(path, "rb") as f: lines = f.readlines() - replacement = replacement = r'

\n\1' + replacement = r'

\n\1' with open(path, "wb") as f: for line in lines: line = line.decode("UTF-8") line = re.sub(r'()', replacement, line.rstrip()) - line = re.sub(r'(
)', replacement, line.rstrip()) + line = re.sub(r'(
)', replacement, line.rstrip()) f.write((line + "\n").encode('UTF-8')) From 423d5593f5a48e8bd0b7dfea0d37278b957d7a56 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 15:40:03 -0600 Subject: [PATCH 074/123] Generify how OPTIONS and CORS are handled --- specification/client_server_api.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 33ee8b1ac..8dde43029 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -173,10 +173,8 @@ to pre-flight requests and supply Cross-Origin Resource Sharing (CORS) headers o all requests. When a client approaches the server with a pre-flight (``OPTIONS``) request, the -server should respond with the CORS headers for that route. If the route does not -exist, the server should return an ``M_NOT_FOUND`` error with a 404 status code. - -The standard CORS headers to be returned by servers on all requests are: +server should respond with the CORS headers for that route. The recommended CORS +headers to be returned by servers on all requests are: .. code:: From 3a9cfd490feaeb6a91d14ade329393b50cb1337b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 23:52:22 -0600 Subject: [PATCH 075/123] Improve pagination handling in proposals.py Previously if multiple pages were returned then the script would lose the results. A check to ensure the response is an array was also added. This helps avoid somewhat cryptic errors when iterating over non-lists (eg: error messages, bad output, etc) --- scripts/proposals.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index b489f0d27..eec6d6527 100755 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -12,13 +12,18 @@ authors = set() prs = set() def getpage(url, page): - resp = requests.get(url + str(page)) + url = url + str(page) + resp = requests.get(url) for link in resp.links.values(): if link['rel'] == 'last': pagecount = re.search('page=(.+?)', link['url']).group(1) - return resp.json() + val = resp.json() + if not isinstance(val, list): + print(val) # Just dump the raw (likely error) response to the log + raise Exception("Error calling %s" % url) + return val def getbylabel(label): pagecount = 1 @@ -27,7 +32,7 @@ def getbylabel(label): print(urlbase) json.extend(getpage(urlbase, 1)) for page in range(2, int(pagecount) + 1): - getpage(urlbase, page) + json.extend(getpage(urlbase, page)) return json From e9f7d2096e100c27c530ec4660670bdb9f8605a4 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 08:39:34 -0600 Subject: [PATCH 076/123] Add changelog entry for CORS --- changelogs/client_server/newsfragments/1365.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1365.feature diff --git a/changelogs/client_server/newsfragments/1365.feature b/changelogs/client_server/newsfragments/1365.feature new file mode 100644 index 000000000..d2864e965 --- /dev/null +++ b/changelogs/client_server/newsfragments/1365.feature @@ -0,0 +1 @@ +Document the CORS/preflight headers \ No newline at end of file From 55e4e90ab1155c5b4d4031fe5108c7b5ea63ac53 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 12:52:11 -0600 Subject: [PATCH 077/123] Document message formats as-is This is likely to later be replaced by mixins, however this is being documented now so clients aren't left in the dark. Fixes https://github.com/matrix-org/matrix-doc/issues/917 --- event-schemas/examples/m.room.message#m.emote | 4 +++- event-schemas/examples/m.room.message#m.text | 4 +++- event-schemas/schema/m.room.message#m.emote | 10 ++++++++++ event-schemas/schema/m.room.message#m.text | 10 ++++++++++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/event-schemas/examples/m.room.message#m.emote b/event-schemas/examples/m.room.message#m.emote index 4280928ee..79292ddf8 100644 --- a/event-schemas/examples/m.room.message#m.emote +++ b/event-schemas/examples/m.room.message#m.emote @@ -2,7 +2,9 @@ "age": 242352, "content": { "body": "thinks this is an example emote", - "msgtype": "m.emote" + "msgtype": "m.emote", + "format": "org.matrix.custom.html", + "formatted_body": "thinks this is an example emote" }, "origin_server_ts": 1431961217939, "event_id": "$WLGTSEFSEF:localhost", diff --git a/event-schemas/examples/m.room.message#m.text b/event-schemas/examples/m.room.message#m.text index e00c7aa56..48a97db80 100644 --- a/event-schemas/examples/m.room.message#m.text +++ b/event-schemas/examples/m.room.message#m.text @@ -2,7 +2,9 @@ "age": 242352, "content": { "body": "This is an example text message", - "msgtype": "m.text" + "msgtype": "m.text", + "format": "org.matrix.custom.html", + "formatted_body": "This is an example text message" }, "origin_server_ts": 1431961217939, "event_id": "$WLGTSEFSEF:localhost", diff --git a/event-schemas/schema/m.room.message#m.emote b/event-schemas/schema/m.room.message#m.emote index 88860cb21..f67a184b1 100644 --- a/event-schemas/schema/m.room.message#m.emote +++ b/event-schemas/schema/m.room.message#m.emote @@ -12,6 +12,16 @@ properties: enum: - m.emote type: string + format: + description: |- + The format used in the ``formatted_body``. Currently only + ``org.matrix.custom.html`` is supported. + type: string + formatted_body: + description: |- + The formatted version of the ``body``. This is required if ``format`` + is specified. + type: string required: - msgtype - body diff --git a/event-schemas/schema/m.room.message#m.text b/event-schemas/schema/m.room.message#m.text index 2720172dd..b481bceab 100644 --- a/event-schemas/schema/m.room.message#m.text +++ b/event-schemas/schema/m.room.message#m.text @@ -12,6 +12,16 @@ properties: enum: - m.text type: string + format: + description: |- + The format used in the ``formatted_body``. Currently only + ``org.matrix.custom.html`` is supported. + type: string + formatted_body: + description: |- + The formatted version of the ``body``. This is required if ``format`` + is specified. + type: string required: - msgtype - body From c999b7c2e21b61a8f5d217054555d6a317a76cf0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 13:00:37 -0600 Subject: [PATCH 078/123] Add newsfragment --- changelogs/client_server/newsfragments/1397.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1397.feature diff --git a/changelogs/client_server/newsfragments/1397.feature b/changelogs/client_server/newsfragments/1397.feature new file mode 100644 index 000000000..e4bd248a8 --- /dev/null +++ b/changelogs/client_server/newsfragments/1397.feature @@ -0,0 +1 @@ +Document message formats on ``m.text`` and ``m.emote`` messages From fbee83d5cf53ddb37ce4b54b6f12d7cbe0496038 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 15:08:28 -0600 Subject: [PATCH 079/123] Clarify some of the properties on the search result Fixes https://github.com/matrix-org/matrix-doc/issues/773 --- api/client-server/search.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index 687140b3e..e28461c4f 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -227,6 +227,9 @@ paths: description: |- The historic profile information of the users that sent the events returned. + + The ``string`` key is the user ID for which + the profile belongs to. additionalProperties: type: object title: User Profile @@ -260,15 +263,24 @@ paths: The current state for every room in the results. This is included if the request had the ``include_state`` key set with a value of ``true``. + + The ``string`` key is the room ID for which the ``State + Event`` array belongs to. additionalProperties: type: array title: Room State items: + type: object "$ref": "definitions/event-schemas/schema/core-event-schema/state_event.yaml" groups: type: object title: Groups - description: Any groups that were requested. + description: |- + Any groups that were requested. + + The outer ``string`` key is the group key requested (eg: ``room_id`` + or ``sender``). The inner ``string`` key is the grouped value (eg: + a room's ID or a user's ID). additionalProperties: type: object title: Group Key From b6f9e13d469718aa07c879b88136f4ec99719505 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 15:09:53 -0600 Subject: [PATCH 080/123] Add newsfragment --- changelogs/client_server/newsfragments/1400.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1400.clarification diff --git a/changelogs/client_server/newsfragments/1400.clarification b/changelogs/client_server/newsfragments/1400.clarification new file mode 100644 index 000000000..3fd29e920 --- /dev/null +++ b/changelogs/client_server/newsfragments/1400.clarification @@ -0,0 +1 @@ +Clarify some of the properties on the search result From 9e0fafbcd4436d3d245a942117eccb1d2396261d Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 15:13:22 -0600 Subject: [PATCH 081/123] Remove debugging line --- scripts/gendoc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 25fd4577a..af613b680 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -405,8 +405,7 @@ def logv(line): def cleanup_env(): - #shutil.rmtree(tmp_dir) - pass + shutil.rmtree(tmp_dir) def mkdirp(d) : From ebc7db12fbb9e5ff0f6193605da8182710d0937c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 16:52:17 -0600 Subject: [PATCH 082/123] Remove extraneous list casting --- api/check_examples.py | 6 +++--- scripts/dump-swagger.py | 6 +++--- scripts/gendoc.py | 16 ++++++++-------- scripts/proposals.py | 2 +- scripts/templating/batesian/__init__.py | 2 +- scripts/templating/batesian/sections.py | 2 +- scripts/templating/build.py | 4 ++-- scripts/templating/matrix_templates/sections.py | 4 ++-- scripts/templating/matrix_templates/units.py | 12 ++++++------ 9 files changed, 27 insertions(+), 27 deletions(-) diff --git a/api/check_examples.py b/api/check_examples.py index 3ecd5a4c6..be0676bb4 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -88,9 +88,9 @@ def check_swagger_file(filepath): with open(filepath) as f: swagger = yaml.load(f) - for path, path_api in list(swagger.get('paths', {}).items()): + for path, path_api in swagger.get('paths', {}).items(): - for method, request_api in list(path_api.items()): + for method, request_api in path_api.items(): request = "%s %s" % (method.upper(), path) for parameter in request_api.get('parameters', ()): if parameter['in'] == 'body': @@ -100,7 +100,7 @@ def check_swagger_file(filepath): responses = request_api['responses'] except KeyError: raise ValueError("No responses for %r" % (request,)) - for code, response in list(responses.items()): + for code, response in responses.items(): check_response(filepath, request, code, response) diff --git a/scripts/dump-swagger.py b/scripts/dump-swagger.py index 6dd9864f9..7994324f6 100755 --- a/scripts/dump-swagger.py +++ b/scripts/dump-swagger.py @@ -94,11 +94,11 @@ for filename in os.listdir(cs_api_dir): api = units.resolve_references(filepath, api) basePath = api['basePath'] - for path, methods in list(api["paths"].items()): + for path, methods in api["paths"].items(): path = (basePath + path).replace('%CLIENT_MAJOR_VERSION%', major_version) - for method, spec in list(methods.items()): - if "tags" in list(spec.keys()): + for method, spec in methods.items(): + if "tags" in spec.keys(): if path not in output["paths"]: output["paths"][path] = {} output["paths"][path][method] = spec diff --git a/scripts/gendoc.py b/scripts/gendoc.py index af613b680..16c40af5d 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -299,7 +299,7 @@ def run_through_template(input_files, set_verbose, substitutions): "-i", "matrix_templates", ] - for k, v in list(substitutions.items()): + for k, v in substitutions.items(): args.append("--substitution=%s=%s" % (k, v)) if set_verbose: @@ -359,7 +359,7 @@ def get_build_target(all_targets, target_name): for i, entry in enumerate(group): if isinstance(entry, dict): group[i] = { - (rel_depth + depth): v for (rel_depth, v) in list(entry.items()) + (rel_depth + depth): v for (rel_depth, v) in entry.items() } return group @@ -378,7 +378,7 @@ def get_build_target(all_targets, target_name): # file_entry is a dict which has more file entries as values elif isinstance(file_entry, dict): resolved_entry = {} - for (depth, entry) in list(file_entry.items()): + for (depth, entry) in file_entry.items(): if not isinstance(entry, str): raise Exception( "Double-nested depths are not supported. Entry: %s" % (file_entry,) @@ -429,7 +429,7 @@ def main(targets, dest_dir, keep_intermediates, substitutions): target_defs = yaml.load(targ_file.read()) if targets == ["all"]: - targets = list(target_defs["targets"].keys()) + targets = target_defs["targets"].keys() log("Building spec [targets=%s]" % targets) @@ -443,17 +443,17 @@ def main(targets, dest_dir, keep_intermediates, substitutions): templated_files[target_name] = templated_file # we do all the templating at once, because it's slow - run_through_template(list(templated_files.values()), VERBOSE, substitutions) + run_through_template(templated_files.values(), VERBOSE, substitutions) stylesheets = glob.glob(os.path.join(script_dir, "css", "*.css")) - for target_name, templated_file in list(templated_files.items()): + for target_name, templated_file in templated_files.items(): target = target_defs["targets"].get(target_name) version_label = None if target: version_label = target.get("version_label") if version_label: - for old, new in list(substitutions.items()): + for old, new in substitutions.items(): version_label = version_label.replace(old, new) rst_file = os.path.join(tmp_dir, "spec_%s.rst" % (target_name,)) @@ -481,7 +481,7 @@ def main(targets, dest_dir, keep_intermediates, substitutions): def list_targets(): with open(os.path.join(spec_dir, "targets.yaml"), "r") as targ_file: target_defs = yaml.load(targ_file.read()) - targets = list(target_defs["targets"].keys()) + targets = target_defs["targets"].keys() print("\n".join(targets)) diff --git a/scripts/proposals.py b/scripts/proposals.py index 156d53518..b489f0d27 100755 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -14,7 +14,7 @@ prs = set() def getpage(url, page): resp = requests.get(url + str(page)) - for link in list(resp.links.values()): + for link in resp.links.values(): if link['rel'] == 'last': pagecount = re.search('page=(.+?)', link['url']).group(1) diff --git a/scripts/templating/batesian/__init__.py b/scripts/templating/batesian/__init__.py index f5d83a1ed..e901590f5 100644 --- a/scripts/templating/batesian/__init__.py +++ b/scripts/templating/batesian/__init__.py @@ -24,7 +24,7 @@ class AccessKeyStore(object): self.accessed_set = set() def keys(self): - return list(self.data.keys()) + return self.data.keys() def add(self, key, unit_dict): self.data[key] = unit_dict diff --git a/scripts/templating/batesian/sections.py b/scripts/templating/batesian/sections.py index 2e5593d64..18a622f65 100644 --- a/scripts/templating/batesian/sections.py +++ b/scripts/templating/batesian/sections.py @@ -54,7 +54,7 @@ class Sections(object): ) elif isinstance(section, dict): self.log(" Generated multiple sections:") - for (k, v) in list(section.items()): + for (k, v) in section.items(): if not isinstance(k, str) or not isinstance(v, str): raise Exception( ("Method %s returned multiple sections as a dict but " + diff --git a/scripts/templating/build.py b/scripts/templating/build.py index de9ead35f..fae4db56f 100755 --- a/scripts/templating/build.py +++ b/scripts/templating/build.py @@ -169,7 +169,7 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={} # print out valid section keys if no file supplied if not files: print("\nValid template variables:") - for key in list(sections.keys()): + for key in sections.keys(): sec_text = "" if (len(sections[key]) > 75) else ( "(Value: '%s')" % sections[key] ) @@ -211,7 +211,7 @@ def process_file(env, sections, filename, output_filename): # Do these substitutions outside of the ordinary templating system because # we want them to apply to things like the underlying swagger used to # generate the templates, not just the top-level sections. - for old, new in list(substitutions.items()): + for old, new in substitutions.items(): output = output.replace(old, new) with open(output_filename, "wb") as f: diff --git a/scripts/templating/matrix_templates/sections.py b/scripts/templating/matrix_templates/sections.py index 71c1acf35..64e32aa44 100644 --- a/scripts/templating/matrix_templates/sections.py +++ b/scripts/templating/matrix_templates/sections.py @@ -86,7 +86,7 @@ class MatrixSections(Sections): # the key is the section name and the value is the value of the section def render_group_http_apis(self): # map all swagger_apis to the form $GROUP_http_api - swagger_groups = list(self.units.get("swagger_apis").keys()) + swagger_groups = self.units.get("swagger_apis").keys() renders = {} for group in swagger_groups: sortFnOrPathList = None @@ -134,7 +134,7 @@ class MatrixSections(Sections): "m.room.message#m.file" ] other_msgtypes = [ - k for k in list(schemas.keys()) if k.startswith("m.room.message#") and + k for k in schemas.keys() if k.startswith("m.room.message#") and k not in msgtype_order ] for event_name in (msgtype_order + other_msgtypes): diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index acd85d66a..63a4e5a84 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -35,7 +35,7 @@ from functools import reduce from six.moves.urllib.parse import urlencode matrix_doc_dir=reduce(lambda acc,_: os.path.dirname(acc), - list(range(1, 5)), os.path.abspath(__file__)) + range(1, 5), os.path.abspath(__file__)) HTTP_APIS = { os.path.join(matrix_doc_dir, "api/application-service"): "as", @@ -126,7 +126,7 @@ def resolve_references(path, schema): else: result = OrderedDict() - for key, value in list(schema.items()): + for key, value in schema.items(): result[key] = resolve_references(path, value) return result elif isinstance(schema, list): @@ -211,7 +211,7 @@ def get_json_schema_object_fields(obj, enforce_title=False): props = obj.get("patternProperties") if props: # try to replace horrible regex key names with pretty x-pattern ones - for key_name in list(props.keys()): + for key_name in props.keys(): pretty_key = props[key_name].get("x-pattern") if pretty_key: props[pretty_key] = props[key_name] @@ -382,7 +382,7 @@ def get_example_for_schema(schema): if 'properties' not in schema: raise Exception('"object" property has neither properties nor example') res = OrderedDict() - for prop_name, prop in list(schema['properties'].items()): + for prop_name, prop in schema['properties'].items(): logger.debug("Parsing property %r" % prop_name) prop_example = get_example_for_schema(prop) res[prop_name] = prop_example @@ -558,7 +558,7 @@ class MatrixUnits(Units): ) if "headers" in good_response: headers = TypeTable() - for (header_name, header) in list(good_response["headers"].items()): + for (header_name, header) in good_response["headers"].items(): headers.add_row( TypeTableRow(key=header_name, title=header["type"], desc=header["description"]), @@ -617,7 +617,7 @@ class MatrixUnits(Units): def load_swagger_apis(self): apis = {} - for path, suffix in list(HTTP_APIS.items()): + for path, suffix in HTTP_APIS.items(): for filename in os.listdir(path): if not filename.endswith(".yaml"): continue From 905ef6daddbaeff457c9f265d0d2bf64332c558b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 16:57:21 -0600 Subject: [PATCH 083/123] Have the towncrier generator complain if the output looks wrong --- scripts/templating/matrix_templates/units.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 63a4e5a84..9b3d621d6 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -845,7 +845,14 @@ class MatrixUnits(Units): stderr=subprocess.PIPE, cwd=tc_path, ).strip().decode('UTF-8') - except subprocess.CalledProcessError: + + # This is a bit of a hack, but it does mean that the log at least gets *something* + # to tell us it broke + if not raw_log.startswith("Unreleased Changes"): + logger.error(raw_log) + raw_log = "" + except subprocess.CalledProcessError as e: + logger.error(e) raw_log = "" tc_lines = raw_log.splitlines() From a8461e647f2c5c6bf8cdb4f5157736ed2589948a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 11 Jul 2018 08:35:55 -0600 Subject: [PATCH 084/123] Improve the error handling for towncrier The changelog shows up via stdout, everything else via stderr. We dump as much information as we can into the changelog to make debugging errors easier. --- scripts/templating/matrix_templates/units.py | 25 +++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 9b3d621d6..760aeb6b8 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -839,21 +839,28 @@ class MatrixUnits(Units): tc_lines = [] if os.path.isdir(tc_path): logger.info("Generating towncrier changelog for: %s" % name) - try: - raw_log = subprocess.check_output( - ['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'], - stderr=subprocess.PIPE, - cwd=tc_path, - ).strip().decode('UTF-8') + p = subprocess.run( + ['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'], + cwd=tc_path, + check=False, # We'll manually check the exit code + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + if p.returncode != 0: + # Something broke - dump as much information as we can + logger.error("Towncrier exited with code %s" % p.returncode) + logger.error(p.stdout.decode('UTF-8')) + logger.error(p.stderr.decode('UTF-8')) + raw_log = "" + else: + raw_log = p.stdout.decode('UTF-8') # This is a bit of a hack, but it does mean that the log at least gets *something* # to tell us it broke if not raw_log.startswith("Unreleased Changes"): + logger.error("Towncrier appears to have failed to generate a changelog") logger.error(raw_log) raw_log = "" - except subprocess.CalledProcessError as e: - logger.error(e) - raw_log = "" tc_lines = raw_log.splitlines() title_part = None From bd2c0b7c98612635c514567094caccda974fafba Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 11 Jul 2018 13:09:19 -0600 Subject: [PATCH 085/123] Convert server keys to swagger --- api/server-server/definitions/keys.yaml | 96 ++++++++++++++ .../definitions/keys_query_response.yaml | 27 ++++ api/server-server/examples/server_key.json | 23 ++++ api/server-server/keys.yaml | 121 ++++++++++++++++++ specification/server_server_api.rst | 34 +---- 5 files changed, 269 insertions(+), 32 deletions(-) create mode 100644 api/server-server/definitions/keys.yaml create mode 100644 api/server-server/definitions/keys_query_response.yaml create mode 100644 api/server-server/examples/server_key.json create mode 100644 api/server-server/keys.yaml diff --git a/api/server-server/definitions/keys.yaml b/api/server-server/definitions/keys.yaml new file mode 100644 index 000000000..5d1c39def --- /dev/null +++ b/api/server-server/definitions/keys.yaml @@ -0,0 +1,96 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +title: Server Keys +description: Server keys +example: + $ref: "../examples/server_key.json" +properties: + server_name: + type: string + description: DNS name of the homeserver + required: true # TODO: Verify + example: "example.org" + verify_keys: + type: object + description: Public keys of the homeserver for verifying digital signatures + required: true # TODO: Verify + additionalProperties: + type: object + title: Verify Key + example: { + "ed25519:auto2": { + "key": "Base+64+Encoded+Signature+Verification+Key" + } + } + properties: + key: + type: string + description: The key + required: true + example: "Base+64+Encoded+Signature+Verification+Key" + old_verify_keys: + type: object + description: The public keys that the server used to use and when it stopped using them + additionalProperties: + type: object + title: Old Verify Key + example: { + "ed25519:auto1": { + "expired_ts": 922834800000, + "key": "Base+64+Encoded+Signature+Verification+Key" + } + } + properties: + expired_ts: + type: integer + format: int64 + description: The expiration time + required: true + example: 922834800000 + key: + type: string + description: The key + required: true + example: "Base+64+Encoded+Signature+Verification+Key" + signatures: + type: object + description: Digital signatures for this object signed using the ``verify_keys`` + additionalProperties: + type: object + title: Signed Server + example: { + "example.org": { + "ad25519:auto2": "Base+64+Encoded+Signature+Verification+Key" + } + } + additionalProperties: + type: string + name: Encoded Signature Verification Key + tls_fingerprints: + type: array + description: Hashes of X.509 TLS certificates used by this server encoded as `Unpadded Base64`_ + items: + type: object + title: TLS Fingerprint + properties: + sha256: + type: string + description: The encoded fingerprint + example: Base+64+Encoded+SHA-256-Fingerprint + valid_until_ts: + type: integer + format: int64 + description: POSIX timestamp when the list of valid keys should be refreshed + example: 1052262000000 diff --git a/api/server-server/definitions/keys_query_response.yaml b/api/server-server/definitions/keys_query_response.yaml new file mode 100644 index 000000000..8223f9ec1 --- /dev/null +++ b/api/server-server/definitions/keys_query_response.yaml @@ -0,0 +1,27 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +description: Server keys +example: { + "server_keys": [{ + $ref: "../examples/server_key.json" + }] +} +properties: + server_keys: + type: array + title: Server Keys + description: The server keys + items: + $ref: "keys.yaml" diff --git a/api/server-server/examples/server_key.json b/api/server-server/examples/server_key.json new file mode 100644 index 000000000..a3934bd36 --- /dev/null +++ b/api/server-server/examples/server_key.json @@ -0,0 +1,23 @@ +{ + "server_name": "example.org", + "verify_keys": { + "ed25519:auto2": { + "key": "Base+64+Encoded+Signature+Verification+Key" + } + }, + "old_verify_keys": { + "ed25519:auto1": { + "expired_ts": 922834800000, + "key": "Base+64+Encoded+Old+Verify+Key" + } + }, + "signatures": { + "example.org": { + "ed25519:auto2": "Base+64+Encoded+Signature" + } + }, + "tls_fingerprints": [{ + "sha256": "Base+64+Encoded+SHA-256-Fingerprint" + }], + "valid_until_ts": 1052262000000 +} \ No newline at end of file diff --git a/api/server-server/keys.yaml b/api/server-server/keys.yaml new file mode 100644 index 000000000..e8e5167dd --- /dev/null +++ b/api/server-server/keys.yaml @@ -0,0 +1,121 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Key Exchange API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/key/v2 +produces: + - application/json +paths: + "/server/{keyId}": + get: + summary: Get the server's key + description: Get the server's key + operationId: getServerKey + parameters: + - in: path + name: keyId + type: string + description: Key ID + required: false + x-example: TODO # No examples in the spec so far + responses: + 200: + description: The server's keys + schema: + $ref: "definitions/keys.yaml" + "/query/{serverName}/{keyId}": + get: + summary: Retreive a server key + description: Retreive a server key + operationId: getQueryKeys + parameters: + - in: path + name: serverName + type: string + description: Server name + required: true + x-example: matrix.org + - in: path + name: keyId + type: string + description: Key ID + required: true + x-example: TODO # No examples in spec so far + - in: query + name: minimum_valid_until_ts + type: integer + format: int64 + description: Minimum Valid Until Milliseconds + required: true # TODO: Verify + x-example: 1234567890 + responses: + 200: + description: The keys for the server + schema: + $ref: "definitions/keys_query_response.yaml" + "/query": + post: + summary: Retreive a server key + description: Retreive a server key + operationId: postQueryKeys + parameters: + - in: body + name: body + schema: + type: object + # TODO: Improve example + example: { + "server_keys": { + "{server_name}": { + "{key_id}": { + "minimum_valid_until_ts": 1234567890 + } + } + } + } + properties: + server_keys: + type: object + description: The query criteria + required: true + additionalProperties: + type: object + name: ServerName + description: The server names to query + additionalProperties: + type: array + name: ServerKey + description: The server keys to query + items: + type: object + title: Query Criteria + description: The query criteria + properties: + minimum_valid_until_ts: + type: integer + format: int64 + description: Minimum Valid Until MS + required: true # TODO: Verify + x-example: 1234567890 + responses: + 200: + description: The keys for the server + schema: + $ref: "definitions/keys_query_response.yaml" diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f2b3204ab..861519910 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -1,5 +1,6 @@ .. Copyright 2016 OpenMarket Ltd .. Copyright 2017 New Vector Ltd +.. Copyright 2018 New Vector Ltd .. .. Licensed under the Apache License, Version 2.0 (the "License"); .. you may not use this file except in compliance with the License. @@ -232,38 +233,7 @@ This API can return keys for servers that are offline by using cached responses taken from when the server was online. Keys can be queried from multiple servers to mitigate against DNS spoofing. -Example request: - -.. code:: - - GET /_matrix/key/v2/query/{server_name}/{key_id}/?minimum_valid_until_ts={minimum_valid_until_ts} HTTP/1.1 - - POST /_matrix/key/v2/query HTTP/1.1 - Content-Type: application/json - - { - "server_keys": { - "{server_name}": { - "{key_id}": { - "minimum_valid_until_ts": {posix_timestamp} - } - } - } - } - - -Response: - -.. code:: - - HTTP/1.1 200 OK - Content-Type: application/json - { - "server_keys": [ - # List of responses with same format as /_matrix/key/v2/server - # signed by both the originating server and this server. - ] - } +{{keys_ss_http_api}} Version 1 +++++++++ From 96889f16e93aed1a273d0e9ef4621ab079b64f32 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 11 Jul 2018 13:20:44 -0600 Subject: [PATCH 086/123] Split out and fix the /server and /query key APIs --- .../{keys.yaml => keys_query.yaml} | 39 ++++------------ api/server-server/keys_server.yaml | 42 +++++++++++++++++ specification/server_server_api.rst | 46 +------------------ 3 files changed, 53 insertions(+), 74 deletions(-) rename api/server-server/{keys.yaml => keys_query.yaml} (72%) create mode 100644 api/server-server/keys_server.yaml diff --git a/api/server-server/keys.yaml b/api/server-server/keys_query.yaml similarity index 72% rename from api/server-server/keys.yaml rename to api/server-server/keys_query.yaml index e8e5167dd..c0811dbc2 100644 --- a/api/server-server/keys.yaml +++ b/api/server-server/keys_query.yaml @@ -23,23 +23,6 @@ basePath: /_matrix/key/v2 produces: - application/json paths: - "/server/{keyId}": - get: - summary: Get the server's key - description: Get the server's key - operationId: getServerKey - parameters: - - in: path - name: keyId - type: string - description: Key ID - required: false - x-example: TODO # No examples in the spec so far - responses: - 200: - description: The server's keys - schema: - $ref: "definitions/keys.yaml" "/query/{serverName}/{keyId}": get: summary: Retreive a server key @@ -100,20 +83,16 @@ paths: name: ServerName description: The server names to query additionalProperties: - type: array - name: ServerKey + type: object + title: Query Criteria description: The server keys to query - items: - type: object - title: Query Criteria - description: The query criteria - properties: - minimum_valid_until_ts: - type: integer - format: int64 - description: Minimum Valid Until MS - required: true # TODO: Verify - x-example: 1234567890 + properties: + minimum_valid_until_ts: + type: integer + format: int64 + description: Minimum Valid Until MS + required: true # TODO: Verify + example: 1234567890 responses: 200: description: The keys for the server diff --git a/api/server-server/keys_server.yaml b/api/server-server/keys_server.yaml new file mode 100644 index 000000000..819a2599d --- /dev/null +++ b/api/server-server/keys_server.yaml @@ -0,0 +1,42 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Key Exchange API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/key/v2 +produces: + - application/json +paths: + "/server/{keyId}": + get: + summary: Get the server's key + description: Get the server's key + operationId: getServerKey + parameters: + - in: path + name: keyId + type: string + description: Key ID + required: false + x-example: TODO # No examples in the spec so far + responses: + 200: + description: The server's keys + schema: + $ref: "definitions/keys.yaml" diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 861519910..925076871 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -167,50 +167,8 @@ If a server goes offline intermediate notary servers should continue to return the last response they received from that server so that the signatures of old events sent by that server can still be checked. -==================== =================== ====================================== - Key Type Description -==================== =================== ====================================== -``server_name`` String DNS name of the homeserver. -``verify_keys`` Object Public keys of the homeserver for - verifying digital signatures. -``old_verify_keys`` Object The public keys that the server used - to use and when it stopped using them. -``signatures`` Object Digital signatures for this object - signed using the ``verify_keys``. -``tls_fingerprints`` Array of Objects Hashes of X.509 TLS certificates used - by this server encoded as `Unpadded Base64`_. -``valid_until_ts`` Integer POSIX timestamp when the list of valid - keys should be refreshed. -==================== =================== ====================================== - +{{keys_server_ss_http_api}} -.. code:: json - - { - "old_verify_keys": { - "ed25519:auto1": { - "expired_ts": 922834800000, - "key": "Base+64+Encoded+Old+Verify+Key" - } - }, - "server_name": "example.org", - "signatures": { - "example.org": { - "ed25519:auto2": "Base+64+Encoded+Signature" - } - }, - "tls_fingerprints": [ - { - "sha256": "Base+64+Encoded+SHA-256-Fingerprint" - } - ], - "valid_until_ts": 1052262000000, - "verify_keys": { - "ed25519:auto2": { - "key": "Base+64+Encoded+Signature+Verification+Key" - } - } - } Querying Keys Through Another Server ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -233,7 +191,7 @@ This API can return keys for servers that are offline by using cached responses taken from when the server was online. Keys can be queried from multiple servers to mitigate against DNS spoofing. -{{keys_ss_http_api}} +{{keys_query_ss_http_api}} Version 1 +++++++++ From 3e13ec2841e2d8c0a67a4e4ae595734d5dc3e757 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 11 Jul 2018 13:43:15 -0600 Subject: [PATCH 087/123] Fix required properties in POST /query --- api/server-server/keys_query.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/server-server/keys_query.yaml b/api/server-server/keys_query.yaml index c0811dbc2..1ba0f5ba3 100644 --- a/api/server-server/keys_query.yaml +++ b/api/server-server/keys_query.yaml @@ -77,7 +77,6 @@ paths: server_keys: type: object description: The query criteria - required: true additionalProperties: type: object name: ServerName @@ -91,8 +90,8 @@ paths: type: integer format: int64 description: Minimum Valid Until MS - required: true # TODO: Verify example: 1234567890 + required: ['server_keys'] responses: 200: description: The keys for the server From 374ec00046d0366c88a8e6cfd40c70c9624ed264 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 11 Jul 2018 16:51:24 -0600 Subject: [PATCH 088/123] Convert things that mention "Transaction" to swagger There's two kinds of transactions currently: one with EDUs and one without. The one with EDUs is only used on /send, however the schema is still somewhat worth splitting out for simplicity. The examples are brought apart to make them slightly more reusable for when they get dumped into the relevant sections of the spec (see TODO in server_server_api.rst) Further, the Transactions stuff introduces tuples to the spec. The units.py has been updated to support this. --- api/server-server/definitions/edu.yaml | 37 +++++ .../definitions/full_transaction.yaml | 28 ++++ api/server-server/definitions/pdu.yaml | 146 ++++++++++++++++++ .../definitions/transaction.yaml | 35 +++++ api/server-server/events.yaml | 118 ++++++++++++++ api/server-server/examples/edu.json | 8 + .../examples/full_transaction.json | 6 + api/server-server/examples/pdu.json | 25 +++ api/server-server/examples/transaction.json | 5 + api/server-server/transactions.yaml | 51 ++++++ scripts/templating/matrix_templates/units.py | 25 ++- specification/server_server_api.rst | 55 +------ 12 files changed, 485 insertions(+), 54 deletions(-) create mode 100644 api/server-server/definitions/edu.yaml create mode 100644 api/server-server/definitions/full_transaction.yaml create mode 100644 api/server-server/definitions/pdu.yaml create mode 100644 api/server-server/definitions/transaction.yaml create mode 100644 api/server-server/events.yaml create mode 100644 api/server-server/examples/edu.json create mode 100644 api/server-server/examples/full_transaction.json create mode 100644 api/server-server/examples/pdu.json create mode 100644 api/server-server/examples/transaction.json create mode 100644 api/server-server/transactions.yaml diff --git a/api/server-server/definitions/edu.yaml b/api/server-server/definitions/edu.yaml new file mode 100644 index 000000000..4597ca623 --- /dev/null +++ b/api/server-server/definitions/edu.yaml @@ -0,0 +1,37 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: Address any concerns about this being inaccurate (flagged as such in the EDUs section) + +type: object +title: Ephemeral Data Unit +description: An ephemeral data unit +example: + $ref: "../examples/edu.json" +properties: + edu_type: + type: string + description: The type of ephemeral message + example: "!abc123:matrix.org" + origin: + type: string + description: The server name sending the ephemeral message + example: "matrix.org" + destination: + type: string + description: The server name receiving the ephemeral message + example: "elsewhere.com" + content: + type: object + description: The content of the ephemeral message diff --git a/api/server-server/definitions/full_transaction.yaml b/api/server-server/definitions/full_transaction.yaml new file mode 100644 index 000000000..39250433c --- /dev/null +++ b/api/server-server/definitions/full_transaction.yaml @@ -0,0 +1,28 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +title: Transaction +description: Transaction +example: + $ref: "../examples/full_transaction.json" +allOf: + - $ref: "transaction.yaml" + - type: object + properties: + edus: + type: array + description: List of ephemeral messages. May be omitted if there are no ephemeral messages to be sent. + items: + $ref: "edu.yaml" +required: ['origin', 'origin_server_ts', 'pdus'] \ No newline at end of file diff --git a/api/server-server/definitions/pdu.yaml b/api/server-server/definitions/pdu.yaml new file mode 100644 index 000000000..b3f25ccf1 --- /dev/null +++ b/api/server-server/definitions/pdu.yaml @@ -0,0 +1,146 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +title: Persistent Data Unit +description: A persistent data unit (event) +example: + $ref: "../examples/pdu.json" +properties: + room_id: + type: string + description: Room identifier + example: "!abc123:matrix.org" + sender: + type: string + description: The ID of the user sending the event + example: "@someone:matrix.org" + origin: + type: string + description: The ``server_name`` of the homeserver that created this event + example: "matrix.org" + origin_server_ts: + type: integer + format: int64 + description: Timestamp in milliseconds on origin homeserver when this event was created. + example: 1234567890 + type: + type: string + description: Event type + required: true + example: "m.room.message" + state_key: + type: string + description: |- + If this key is present, the event is a state event, and it will replace previous events + with the same ``type`` and ``state_key`` in the room state. + example: "my_key" + content: + type: object + description: The content of the event + prev_events: + type: array + description: |- + Event IDs and hashes of the most recent events in the room that the homeserver was aware + of when it made this event + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + depth: + type: integer + description: The maximum depth of the ``prev_events``, plus one + example: 12 + auth_events: + type: array + description: Event IDs and hashes for the "auth events" of this event + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + hashes: + type: object + title: Event Hash + description: Hashes of the PDU, following the algorithm specified in `Signing Events`_ + example: { + "sha256": "thishashcoversallfieldsincasethisisredacted" + } + properties: + sha256: + type: string + description: The hash + example: thishashcoversallfieldsincasethisisredacted + required: ['sha256'] + signatures: + type: object + description: |- + Signatures of the redacted PDU, following the algorithm specified in `Signing Events`_ + example: { + "example.com": { + "ed25519:key_version:": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" + } + } + additionalProperties: + type: object + title: Server Signatures + additionalProperties: + type: string + redacts: + type: string + description: For redaction events, the ID of the event being redacted + example: "$def456:matrix.org" + unsigned: + type: object + description: Additional data added by the origin server but not covered by the ``signatures`` +required: + - room_id + - sender + - origin + - origin_server_ts + - type + - content + - prev_events + - depth + - auth_events + - hashes + - signatures diff --git a/api/server-server/definitions/transaction.yaml b/api/server-server/definitions/transaction.yaml new file mode 100644 index 000000000..630d2ba36 --- /dev/null +++ b/api/server-server/definitions/transaction.yaml @@ -0,0 +1,35 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +title: Transaction +description: Transaction +example: + $ref: "../examples/transaction.json" +properties: + origin: + type: string + description: |- + The ``server_name`` of hoemserver sending this transaction + example: "example.org" + origin_server_ts: + type: integer + format: int64 + description: Timestamp in milliseconds on originating homeserver when this transaction started + example: 1234567890 + pdus: + type: array + description: List of persistent updates to rooms + items: + $ref: "pdu.yaml" +required: ['origin', 'origin_server_ts', 'pdus'] \ No newline at end of file diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml new file mode 100644 index 000000000..055883c26 --- /dev/null +++ b/api/server-server/events.yaml @@ -0,0 +1,118 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Events API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/state/{roomId}": + get: + summary: Get all the state of a given room + description: |- + Retrieves a snapshot of the entire current state of the given room. + operationId: getRoomState + parameters: + - in: path + name: roomId + type: string + description: The room ID to get state for + required: true + x-example: "!abc123:matrix.org" + responses: + 200: + description: The room state for the room (kept under ``pdus``) + schema: + $ref: "definitions/transaction.yaml" + "/event/{eventId}": + get: + summary: Get a single event + description: |- + Retrieves a single event + operationId: getEvent + parameters: + - in: path + name: eventId + type: string + description: The event ID to get + required: true + x-example: "$abc123:matrix.org" + responses: + 200: + description: A transaction containing a single PDU which is the event requested + schema: + $ref: "definitions/transaction.yaml" + "/backfill/{roomId}": + get: + summary: Retrieves the events which precede the given event + description: |- + Retreives a sliding-window history of previous PDUs that occurred on the given room. + Starting from the PDU ID(s) given in the ``v`` argument, the PDUs that preceded it + are retrived, up to the total number given by the ``limit``. + operationId: backfillRoom + parameters: + - in: path + name: roomId + type: string + description: The room ID to backfill + required: true + x-example: "!abc123:matrix.org" + - in: query + name: v + type: string # TODO: The description says this is plural - figure out how to specify multiple, and spec it + description: The event ID to backfill from + required: true + x-example: "$abc123:matrix.org" + - in: query + name: limit + type: integer + description: The maximum number of events to retrieve + required: true # TODO: Verify + x-example: 10 + responses: + 200: + description: A transaction containing the PDUs that preceded the given event(s). + schema: + $ref: "definitions/transaction.yaml" + "/pull": + get: + summary: Stream events later than a given point in history + description: |- + Retrieves all of the transactions later than any version given by the ``v`` arguments. + operationId: pull + parameters: + - in: query + name: v + type: string # TODO: The description says this is plural - figure out how to specify multiple, and spec it + description: The event ID to backfill from + required: true + x-example: "$abc123:matrix.org" + - in: query + name: origin + type: string + description: The origin # TODO: What is this actually? + required: true # TODO: Verify + x-example: "matrix.org" + responses: + 200: + # TODO: This could do with a better description + description: A transaction containing multiple PDUs + schema: + $ref: "definitions/transaction.yaml" \ No newline at end of file diff --git a/api/server-server/examples/edu.json b/api/server-server/examples/edu.json new file mode 100644 index 000000000..95a7b55d9 --- /dev/null +++ b/api/server-server/examples/edu.json @@ -0,0 +1,8 @@ +{ + "edu_type": "m.presence", + "origin": "blue", + "destination": "orange", + "content": { + "key": "value" + } +} \ No newline at end of file diff --git a/api/server-server/examples/full_transaction.json b/api/server-server/examples/full_transaction.json new file mode 100644 index 000000000..c453d6ed3 --- /dev/null +++ b/api/server-server/examples/full_transaction.json @@ -0,0 +1,6 @@ +{ + "origin": "matrix.org", + "origin_server_ts": 1234567890, + "pdus": [{"$ref": "pdu.json"}], + "edus": [{"$ref": "edu.json"}] +} \ No newline at end of file diff --git a/api/server-server/examples/pdu.json b/api/server-server/examples/pdu.json new file mode 100644 index 000000000..531f4a374 --- /dev/null +++ b/api/server-server/examples/pdu.json @@ -0,0 +1,25 @@ +{ + "room_id": "!UcYsUzyxTGDxLBEvLy:example.org", + "sender": "@alice:example.com", + "origin": "example.com", + "event_id": "$a4ecee13e2accdadf56c1025:example.com", + "origin_server_ts": 1404838188000, + "type": "m.room.message", + "prev_events": [ + [ + "$af232176:example.org", + {"sha256": "abase64encodedsha256hashshouldbe43byteslong"} + ] + ], + "hashes": { + "sha256": "thishashcoversallfieldsincasethisisredacted" + }, + "signatures": { + "example.com": { + "ed25519:key_version:": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" + } + }, + "content": { + "key": "value" + } +} \ No newline at end of file diff --git a/api/server-server/examples/transaction.json b/api/server-server/examples/transaction.json new file mode 100644 index 000000000..bd8ac3dc7 --- /dev/null +++ b/api/server-server/examples/transaction.json @@ -0,0 +1,5 @@ +{ + "origin": "matrix.org", + "origin_server_ts": 1234567890, + "pdus": [{"$ref": "pdu.json"}] +} \ No newline at end of file diff --git a/api/server-server/transactions.yaml b/api/server-server/transactions.yaml new file mode 100644 index 000000000..389eecc7e --- /dev/null +++ b/api/server-server/transactions.yaml @@ -0,0 +1,51 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Transaction API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/send/{txnId}": + put: + summary: Send a transaction + description: |- + Push messages representing live activity to another server. The destination name + will be set to that of the receiving server itself. Each embedded PDU in the + transaction body will be processed. + operationId: sendTransaction + parameters: + - in: path + name: txnId + type: string + # TODO: "Overrides any ID given in the JSON body" - What does this mean? + description: |- + The transaction ID. Overrides any ID given in the JSON body. + required: true + x-example: TODO # No examples in the spec so far + - in: body + name: body + type: object + schema: + $ref: "definitions/full_transaction.yaml" + responses: + 200: + # TODO: Spec this (and figure out what it is) + description: TODO diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index dac183d1f..7819dbf90 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -292,10 +292,19 @@ def process_data_type(prop, required=False, enforce_title=True): is_object = True elif prop_type == "array": - nested = process_data_type(prop["items"]) - prop_title = "[%s]" % nested["title"] - tables = nested["tables"] - enum_desc = nested["enum_desc"] + items = prop["items"] + if isinstance(items, list): + prop_title = "[" + for i in items: + nested = process_data_type(i) + tables.extend(nested['tables']) + prop_title = "%s%s, " % (prop_title, nested['title']) + prop_title = prop_title[:-2] + "]" + else: + nested = process_data_type(prop["items"]) + prop_title = "[%s]" % nested["title"] + tables = nested["tables"] + enum_desc = nested["enum_desc"] else: prop_title = prop_type @@ -505,7 +514,13 @@ class MatrixUnits(Units): if val_type == "array": items = param.get("items") if items: - val_type = "[%s]" % items.get("type") + if isinstance(items, list): + val_type = "[" + for i in items: + val_type += "%s, " % items.get("type") + val_type = val_type[:-2] + "]" + else: + val_type = "[%s]" % items.get("type") if param.get("enum"): val_type = "enum" diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f2b3204ab..7b66584dc 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -360,6 +360,10 @@ the destination. PDU Fields ~~~~~~~~~~ +.. TODO-spec + + Figure out how to embed swagger definitions in here (or improve the section) + ==================== ================== ======================================= Key Type Description ==================== ================== ======================================= @@ -749,57 +753,10 @@ All these URLs are name-spaced within a prefix of:: /_matrix/federation/v1/... -For active pushing of messages representing live activity "as it happens":: - - PUT .../send// - Body: JSON encoding of a single Transaction - Response: TODO-doc - -The transaction_id path argument will override any ID given in the JSON body. -The destination name will be set to that of the receiving server itself. Each -embedded PDU in the transaction body will be processed. - - -To fetch all the state of a given room:: - - GET .../state// - Response: JSON encoding of a single Transaction containing multiple PDUs - -Retrieves a snapshot of the entire current state of the given room. The -response will contain a single Transaction, inside which will be a list of PDUs -that encode the state. - - -To fetch a particular event:: - - GET .../event// - Response: JSON encoding of a partial Transaction containing the event - -Retrieves a single event. The response will contain a partial Transaction, -having just the ``origin``, ``origin_server_ts`` and ``pdus`` fields; the -event will be encoded as the only PDU in the ``pdus`` list. - - -To backfill events on a given room:: - - GET .../backfill// - Query args: v, limit - Response: JSON encoding of a single Transaction containing multiple PDUs - -Retrieves a sliding-window history of previous PDUs that occurred on the given -room. Starting from the PDU ID(s) given in the "v" argument, the PDUs that -preceded it are retrieved, up to a total number given by the "limit" argument. - - -To stream all the events:: - - GET .../pull/ - Query args: origin, v - Response: JSON encoding of a single Transaction consisting of multiple PDUs -Retrieves all of the transactions later than any version given by the "v" -arguments. +{{transactions_ss_http_api}} +{{events_ss_http_api}} To make a query:: From 114bcf1a2e1d612c15a735612e8a8f0ab9fb3b4f Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 6 Jul 2018 10:28:57 +0100 Subject: [PATCH 089/123] Use $ref, clean up, fix errors, AS is now a C-S module. --- .../application_service.yaml | 192 +---- .../definitions/schema/location.yaml | 30 + .../definitions/schema/location_batch.yaml | 17 + .../definitions/schema/protocol.yaml | 79 ++ .../definitions/schema/protocol_metadata.yaml | 66 ++ .../definitions/schema/user.yaml | 31 + .../definitions/schema/user_batch.yaml | 17 + api/client-server/third_party_lookup.yaml | 209 +---- specification/feature_profiles.rst | 2 + .../application_services.rst} | 2 + specification/proposals.rst | 770 +----------------- specification/targets.yaml | 2 +- 12 files changed, 323 insertions(+), 1094 deletions(-) create mode 100644 api/application-service/definitions/schema/location.yaml create mode 100644 api/application-service/definitions/schema/location_batch.yaml create mode 100644 api/application-service/definitions/schema/protocol.yaml create mode 100644 api/application-service/definitions/schema/protocol_metadata.yaml create mode 100644 api/application-service/definitions/schema/user.yaml create mode 100644 api/application-service/definitions/schema/user_batch.yaml rename specification/{third_party_lookup.rst => modules/application_services.rst} (94%) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 86411ffa2..b68cbe187 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -131,7 +131,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -140,7 +140,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 404: description: |- The application service indicates that this room alias does not exist. @@ -150,7 +150,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - type: object + $ref: ../client-server/definitions/error.yaml "/users/{userId}": get: summary: Query if a user should exist on the application service. @@ -187,7 +187,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -196,7 +196,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 404: description: |- The application service indicates that this user does not exist. @@ -206,7 +206,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - type: object + $ref: ../client-server/definitions/error.yaml "/_matrix/app/unstable/thirdparty/protocol/{protocol}": get: summary: Retrieve metadata about a specific protocol that the application service supports. @@ -219,44 +219,14 @@ paths: - in: path name: protocol type: string - description: |- - The name of the protocol. + description: The protocol ID. required: true x-example: "irc" responses: 200: description: The protocol was found and metadata returned. - examples: - application/json: { - "user_fields": ["network", "nickname"], - "location_fields": ["network", "channel"], - "icon": "mxc://example.org/aBcDeFgH", - "field_types": { - "network": { - "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", - "placeholder": "irc.example.org" - }, - "nickname": { - "regexp": "[^\\s]+", - "placeholder": "username" - }, - "channel": { - "regexp": "#[^\\s]+", - "placeholder": "#foobar" - } - }, - "instances": [ - { - "desc": "Freenode", - "icon": "mxc://example.org/JkLmNoPq", - "fields": { - "network": "freenode.net", - } - } - ] - } schema: - type: object + $ref: definitions/schema/protocol_metadata.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -266,7 +236,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -275,7 +245,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 404: description: No protocol was found with the given path. examples: @@ -283,20 +253,20 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - type: object + $ref: ../client-server/definitions/error.yaml "/_matrix/app/unstable/thirdparty/user/{protocol}": get: - summary: Retrieve the Matrix ID of a corresponding third party user. + summary: Retrieve the Matrix User ID of a corresponding third party user. description: |- - This API is called by the homeserver in order to retrieve a Matrix ID linked - to a user on the external service, given a set of user parameters. + This API is called by the homeserver in order to retrieve a Matrix + User ID linked to a user on the third party network, given a set of + user parameters. operationId: queryUserByProtocol parameters: - in: path name: protocol type: string - description: |- - The name of the protocol. + description: The protocol ID. required: true x-example: irc - in: query @@ -307,35 +277,9 @@ paths: service to help identify the user. responses: 200: - description: The Matrix IDs found with the given parameters. - examples: - application/json: [{ - "userid": "@_gitter_jim:matrix.org", - "protocol": "gitter", - "fields": { - "user": "jim" - } - }] + description: The Matrix User IDs found with the given parameters. schema: - type: array - description: Matched users. - items: - type: object - title: User - schema: - userid: - type: string - description: The Matrix ID of the matched user. - protocol: - type: string - description: The third party protocol. - fields: - type: object - description: The third party network fields used to identify this user. - properties: - user: - type: string - description: An example field, in this case the username for a Gitter user. + $ref: definitions/schema/user_batch.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -345,7 +289,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -354,7 +298,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 404: description: No users were found with the given parameters. examples: @@ -362,27 +306,20 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - type: object + $ref: ../client-server/definitions/error.yaml "/_matrix/app/unstable/thirdparty/location/{protocol}": get: summary: Retreive Matrix-side portal rooms leading to a third party location. description: |- - Requesting this endpoint with a valid protocol name results in a list - of successful mapping results in a JSON array. Each result contains - objects to represent the Matrix room or rooms that represent a portal - to this third party network. Each has the Matrix room alias string, - an identifier for the particular third party network protocol, and an - object containing the network-specific fields that comprise this - identifier. It should attempt to canonicalise the identifier as much - as reasonably possible given the network type. + Retrieve a list of Matrix portal rooms that lead to the matched third party location. operationId: queryLocationByProtocol parameters: - in: path name: protocol type: string - description: The protocol used to communicate to the third party network. + description: The protocol ID. required: true - x-example: "irc" + x-example: irc - in: query name: field1, field2... type: string @@ -392,23 +329,8 @@ paths: responses: 200: description: At least one portal room was found. - examples: - application/json: [{ - "alias": "#freenode_#matrix:matrix.org", - "protocol": "irc", - "fields": { - "network": "freenode", - "channel": "#matrix" - } - }] schema: - type: array - description: |- - Array of portal rooms leading to the requested third party - location. - items: - type: object - title: Portal Room + $ref: definitions/schema/location_batch.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -418,7 +340,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -427,7 +349,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 404: description: No mappings were found with the given parameters. examples: @@ -435,7 +357,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - type: object + $ref: ../client-server/definitions/error.yaml "/_matrix/app/unstable/thirdparty/location": get: summary: Reverse-lookup third party locations given a Matrix room alias. @@ -444,30 +366,16 @@ paths: alias. operationId: queryLocationByAlias parameters: - - in: query - name: alias - type: string - description: The Matrix room alias to look up. + - in: query + name: alias + type: string + description: The Matrix room alias to look up. responses: 200: description: |- - All found third party locations. Same response format as the - forward lookup response. - examples: - application/json: [{ - "alias": "#freenode_#matrix:matrix.org", - "protocol": "irc", - "fields": { - "network": "freenode", - "channel": "#matrix" - } - }] + All found third party locations. schema: - type: array - description: Matched third party locations. - items: - type: object - title: Location + $ref: definitions/schema/location_batch.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -477,7 +385,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -486,7 +394,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 404: description: No mappings were found with the given parameters. examples: @@ -494,36 +402,24 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - type: object + $ref: ../client-server/definitions/error.yaml "/_matrix/app/unstable/thirdparty/user": get: - summary: Reverse-lookup third party users given a Matrix ID. + summary: Reverse-lookup third party users given a Matrix User ID. description: |- - Retreive an array of third party users from a Matrix ID. + Retreive an array of third party users from a Matrix User ID. operationId: queryUserByID parameters: - in: query name: userid type: string - description: The Matrix ID to look up. + description: The Matrix User ID to look up. responses: 200: description: |- An array of third party users. - examples: - application/json: [{ - "userid": "@_gitter_jim:matrix.org", - "protocol": "gitter", - "fields": { - "user": "jim" - } - }] schema: - type: array - description: Matched third party users - items: - type: object - title: User + $ref: definitions/schema/user_batch.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -533,7 +429,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -542,7 +438,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 404: description: No mappings were found with the given parameters. examples: @@ -550,4 +446,4 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - type: object \ No newline at end of file + $ref: ../client-server/definitions/error.yaml \ No newline at end of file diff --git a/api/application-service/definitions/schema/location.yaml b/api/application-service/definitions/schema/location.yaml new file mode 100644 index 000000000..4967ef61f --- /dev/null +++ b/api/application-service/definitions/schema/location.yaml @@ -0,0 +1,30 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +properties: + alias: + description: An alias for a matrix room. + type: string + example: "#freenode_#matrix:matrix.org" + protocol: + description: The protocol ID that the third party location is a part of. + type: string + example: irc + fields: + description: Information used to identify this third party location. + type: object + example: + "network": "freenode" + "channel": "#matrix" +title: Location +type: object \ No newline at end of file diff --git a/api/application-service/definitions/schema/location_batch.yaml b/api/application-service/definitions/schema/location_batch.yaml new file mode 100644 index 000000000..3f6de9df0 --- /dev/null +++ b/api/application-service/definitions/schema/location_batch.yaml @@ -0,0 +1,17 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: array +description: List of matched third party locations. +items: + $ref: location.yaml diff --git a/api/application-service/definitions/schema/protocol.yaml b/api/application-service/definitions/schema/protocol.yaml new file mode 100644 index 000000000..231e8288a --- /dev/null +++ b/api/application-service/definitions/schema/protocol.yaml @@ -0,0 +1,79 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +properties: + user_fields: + description: Fields used to identify a third party user. + type: array + items: + type: string + description: Field used to identify a third party user. + example: ["network", "nickname"] + location_fields: + description: Fields used to identify a third party location. + type: array + items: + type: string + description: Field used to identify a third party location. + example: ["network", "channel"] + icon: + description: An icon representing the third party protocol. + type: string + example: "mxc://example.org/aBcDeFgH" + field_types: + title: Field Types + description: All location or user fields should have an entry here. + type: object + properties: + fieldname: + title: Field Type + description: Definition of valid values for a field. + type: object + properties: + regexp: + description: A regular expression for validation of a field's value. + type: string + placeholder: + description: An placeholder serving as a valid example of the field value. + type: string + example: { + "network": { + "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", + "placeholder": "irc.example.org" + }, + "nickname": { + "regexp": "[^\\s#]+", + "placeholder": "username" + }, + "channel": { + "regexp": "#[^\\s]+", + "placeholder": "#foobar" + } + } + instances: + description: |- + A list of objects representing independent instances of configuration. + For instance multiple networks on IRC if multiple are bridged by the + same bridge. + type: array + items: + type: object + example: { + "desc": "Freenode", + "icon": "mxc://example.org/JkLmNoPq", + "fields": { + "network": "freenode.net", + } + } +title: Protocol +type: object \ No newline at end of file diff --git a/api/application-service/definitions/schema/protocol_metadata.yaml b/api/application-service/definitions/schema/protocol_metadata.yaml new file mode 100644 index 000000000..722640602 --- /dev/null +++ b/api/application-service/definitions/schema/protocol_metadata.yaml @@ -0,0 +1,66 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +description: Dictionary of supported third party protocols. +example: { + "irc": { + "user_fields": ["network", "nickname"], + "location_fields": ["network", "channel"], + "icon": "mxc://example.org/aBcDeFgH", + "field_types": { + "network": { + "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", + "placeholder": "irc.example.org" + }, + "nickname": { + "regexp": "[^\\s]+", + "placeholder": "username" + }, + "channel": { + "regexp": "#[^\\s]+", + "placeholder": "#foobar" + } + }, + "instances": [ + { + "desc": "Freenode", + "icon": "mxc://example.org/JkLmNoPq", + "fields": { + "network": "freenode.net", + } + } + ] + }, + "gitter": { + "user_fields": ["username"], + "location_fields": ["room"], + "field_types": { + "username": { + "regexp": "@[^\\s]+", + "placeholder": "@username" + }, + "room": { + "regexp": "[^\\s]+\\/[^\\s]+", + "placeholder": "matrix-org/matrix-doc" + } + }, + "instances": [ + { + "desc": "Gitter", + "icon": "mxc://example.org/zXyWvUt", + "fields": {} + } + ] + } +} \ No newline at end of file diff --git a/api/application-service/definitions/schema/user.yaml b/api/application-service/definitions/schema/user.yaml new file mode 100644 index 000000000..5f8d0460f --- /dev/null +++ b/api/application-service/definitions/schema/user.yaml @@ -0,0 +1,31 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: Change userid to user_id as a breaking change +properties: + userid: + description: A Matrix User ID represting a third party user. + type: string + example: "@_gitter_jim:matrix.org" + protocol: + description: The protocol ID that the third party location is a part of. + type: string + example: gitter + fields: + description: Information used to identify this third party location. + type: object + example: + "user": "jim" +title: Location +type: object \ No newline at end of file diff --git a/api/application-service/definitions/schema/user_batch.yaml b/api/application-service/definitions/schema/user_batch.yaml new file mode 100644 index 000000000..3653feb44 --- /dev/null +++ b/api/application-service/definitions/schema/user_batch.yaml @@ -0,0 +1,17 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: array +description: List of matched third party users. +items: + $ref: user.yaml diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 343cf2b77..863896030 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -24,6 +24,8 @@ consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/thirdparty/protocols": get: @@ -36,60 +38,8 @@ paths: responses: 200: description: The protocols supported by the homeserver. - examples: - application/json: { - "irc": { - "user_fields": ["network", "nickname"], - "location_fields": ["network", "channel"], - "icon": "mxc://example.org/aBcDeFgH", - "field_types": { - "network": { - "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", - "placeholder": "irc.example.org" - }, - "nickname": { - "regexp": "[^\\s]+", - "placeholder": "username" - }, - "channel": { - "regexp": "#[^\\s]+", - "placeholder": "#foobar" - } - }, - "instances": [ - { - "desc": "Freenode", - "icon": "mxc://example.org/JkLmNoPq", - "fields": { - "network": "freenode.net", - } - } - ] - }, - "gitter": { - "user_fields": ["username"], - "location_fields": ["room"], - "field_types": { - "username": { - "regexp": "@[^\\s]+", - "placeholder": "@username" - }, - "room": { - "regexp": "[^\\s]+\\/[^\\s]+", - "placeholder": "matrix-org/matrix-doc" - } - }, - "instances": [ - { - "desc": "Gitter", - "icon": "mxc://example.org/zXyWvUt", - "fields": {} - } - ] - } - } schema: - type: object + $ref: ../application-service/definitions/schema/protocol_metadata.yaml "/thirdparty/protocol/{protocol}": get: summary: Retrieve metadata about a specific protocol that the homeserver supports. @@ -107,45 +57,16 @@ paths: responses: 200: description: The protocol was found and metadata returned. - examples: - application/json: { - "user_fields": ["network", "nickname"], - "location_fields": ["network", "channel"], - "icon": "mxc://example.org/aBcDeFgH", - "field_types": { - "network": { - "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", - "placeholder": "irc.example.org" - }, - "nickname": { - "regexp": "[^\\s#]+", - "placeholder": "username" - }, - "channel": { - "regexp": "#[^\\s]+", - "placeholder": "#foobar" - } - }, - "instances": [ - { - "desc": "Freenode", - "icon": "mxc://example.org/JkLmNoPq", - "fields": { - "network": "freenode.net", - } - } - ] - } schema: - type: object + $ref: ../application-service/definitions/schema/protocol.yaml 404: - description: The protocol is unknown + description: The protocol is unknown. examples: application/json: { "errcode": "M_NOT_FOUND" } schema: - type: object + $ref: definitions/error.yaml "/thirdparty/location/{protocol}": get: summary: Retreive Matrix-side portals rooms leading to a third party location. @@ -165,9 +86,9 @@ paths: type: string description: The protocol used to communicate to the third party network. required: true - x-example: "irc" + x-example: irc - in: query - name: field1, field2... + name: searchFields type: string description: |- One or more custom fields to help identify the third party @@ -175,36 +96,21 @@ paths: responses: 200: description: At least one portal room was found. - examples: - application/json: [{ - "alias": "#freenode_#matrix:matrix.org", - "protocol": "irc", - "fields": { - "network": "freenode", - "channel": "#matrix" - } - }] schema: - type: array - description: |- - Array of portal rooms leading to the requested third party - location. - items: - type: object - title: Portal Room + $ref: ../application-service/definitions/schema/location_batch.yaml 404: - description: The Matrix room alias was not found + description: No portal rooms were found. examples: application/json: { "errcode": "M_NOT_FOUND" } schema: - type: object + $ref: definitions/error.yaml "/thirdparty/user/{protocol}": get: - summary: Retrieve the Matrix ID of a corresponding third party user. + summary: Retrieve the Matrix User ID of a corresponding third party user. description: |- - Retrieve a Matrix ID linked to a user on the third party service, given + Retrieve a Matrix User ID linked to a user on the third party service, given a set of user parameters. operationId: queryUserByProtocol parameters: @@ -222,43 +128,16 @@ paths: One or more custom fields that are passed to the AS to help identify the user. responses: 200: - description: The Matrix IDs found with the given parameters. - examples: - application/json: [{ - "userid": "@_gitter_jim:matrix.org", - "protocol": "gitter", - "fields": { - "user": "jim" - } - }] - schema: - type: array - description: Matched users. - items: - type: object - title: User - schema: - userid: - type: string - description: The Matrix ID of the matched user. - protocol: - type: string - description: The third party protocol. - fields: - type: object - description: The third party network fields used to identify this user. - properties: - user: - type: string - description: An example field, in this case the username for a Gitter user. + description: The Matrix User IDs found with the given parameters. + $ref: ../application-service/definitions/schema/user_batch.yaml 404: - description: The Matrix ID was not found + description: The Matrix User ID was not found examples: application/json: { "errcode": "M_NOT_FOUND" } schema: - type: object + $ref: definitions/error.yaml "/thirdparty/location": get: summary: Reverse-lookup third party locations given a Matrix room alias. @@ -267,30 +146,17 @@ paths: alias. operationId: queryLocationByAlias parameters: - - in: path - name: alias - type: string - description: The Matrix room alias to look up. + - in: path + name: alias + type: string + description: The Matrix room alias to look up. + required: true responses: 200: description: |- - All found third party locations. Same response format as the - forward lookup response. - examples: - application/json: [{ - "alias": "#freenode_#matrix:matrix.org", - "protocol": "irc", - "fields": { - "network": "freenode", - "channel": "#matrix" - } - }] + All found third party locations. schema: - type: array - description: Matched third party locations. - items: - type: object - title: Location + $ref: ../application-service/definitions/schema/location_batch.yaml 404: description: The Matrix room alias was not found examples: @@ -298,41 +164,30 @@ paths: "errcode": "M_NOT_FOUND" } schema: - type: object + $ref: definitions/error.yaml "/thirdparty/user": get: - summary: Reverse-lookup third party users given a Matrix ID. + summary: Reverse-lookup third party users given a Matrix User ID. description: |- - Retreive an array of third party users from a Matrix ID. + Retreive an array of third party users from a Matrix User ID. operationId: queryUserByID parameters: - in: path name: userid type: string - description: The Matrix ID to look up. + description: The Matrix User ID to look up. + required: true responses: 200: description: |- An array of third party users. - examples: - application/json: [{ - "userid": "@_gitter_jim:matrix.org", - "protocol": "gitter", - "fields": { - "user": "jim" - } - }] schema: - type: array - description: Matched third party users - items: - type: object - title: User + $ref: ../application-service/definitions/schema/user_batch.yaml 404: - description: The Matrix ID was not found + description: The Matrix User ID was not found examples: application/json: { "errcode": "M_NOT_FOUND" } schema: - type: object \ No newline at end of file + $ref: definitions/error.yaml \ No newline at end of file diff --git a/specification/feature_profiles.rst b/specification/feature_profiles.rst index 7fc9696de..97d0de0ea 100644 --- a/specification/feature_profiles.rst +++ b/specification/feature_profiles.rst @@ -42,6 +42,7 @@ Summary `Server Side Search`_ Optional Optional Optional Optional Optional `Server Administration`_ Optional Optional Optional Optional Optional `Event Context`_ Optional Optional Optional Optional Optional + `Application Services`_ Optional Optional Optional Optional Optional ===================================== ========== ========== ========== ========== ========== *Please see each module for more details on what clients need to implement.* @@ -57,6 +58,7 @@ Summary .. _Server Side Search: `module:search`_ .. _Server Administration: `module:admin`_ .. _Event Context: `module:event-context`_ +.. _Application Services: `module:application-services`_ Clients ------- diff --git a/specification/third_party_lookup.rst b/specification/modules/application_services.rst similarity index 94% rename from specification/third_party_lookup.rst rename to specification/modules/application_services.rst index 6325dca0c..52e35dc8d 100644 --- a/specification/third_party_lookup.rst +++ b/specification/modules/application_services.rst @@ -1,6 +1,8 @@ Application Services ==================== +.. _module:application-services: + An application service is a separate program that interacts with a homeserver and provides various bits of functionality that would otherwise not make sense to include directly in the homeserver. This ranges from bots, which can diff --git a/specification/proposals.rst b/specification/proposals.rst index d04edfa84..371850ab3 100644 --- a/specification/proposals.rst +++ b/specification/proposals.rst @@ -1,772 +1,6 @@ Tables of Tracked Proposals --------------------------- -proposal-wip -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +This file is autogenerated by a jenkins build process -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC455 `_ - - Do we want to specify a matrix:// URI scheme for rooms? (SPEC-5) - - 2014-09-15 - - 2018-05-15 - - `455-1 `_ - - `@KitsuneRal`_ - - None - - - * - `MSC586 `_ - - Federation API for canonicalising MXIDs - - 2014-10-27 - - 2018-05-15 - - `586-1 `_ - - `@ara4n`_ - - None - - - * - `MSC489 `_ - - Extensible Profiles. (SPEC-93) - - 2015-01-19 - - 2018-05-15 - - `489-1 `_ - - `@erikjohnston`_ - - None - - - * - `MSC1196 `_ - - Matrix Escape Hatch (Versioned Rooms) - - 2015-10-22 - - 2018-05-15 - - `1196-1 `_ - - `@leonerd`_ - - None - - - * - `MSC1148 `_ - - Support for websockets - - 2015-11-16 - - 2018-06-04 - - `1148-1 `_, `1148-2 `_ - - `@richvdh`_, `@krombel`_ - - None - - - * - `MSC1238 `_ - - Push to Talk - - 2016-04-13 - - 2018-05-15 - - `1238-1 `_ - - `@aviraldg`_ - - None - - `PR#310`_ - * - `MSC1198 `_ - - Threading API - - 2016-05-23 - - 2018-05-15 - - `1198-1 `_ - - `@Half-Shot`_ - - None - - - * - `MSC1207 `_ - - Publishing Room Lists for 3rd party networks - - 2016-10-21 - - 2018-05-31 - - `1207-1 `_ - - `@erikjohnston`_ - - None - - - * - `MSC441 `_ - - Support for Reactions / Aggregations - - 2016-12-25 - - 2018-05-15 - - `441-1 `_ - - `@pik`_ - - `@ara4n`_ - - - * - `MSC1237 `_ - - Improving m.location with GeoJSON and accuracy - - 2017-05-19 - - 2018-05-15 - - `1237-1 `_ - - `@Half-Shot`_ - - None - - `PR#919`_ - * - `MSC971 `_ - - Add groups stuff to spec - - 2017-08-10 - - 2018-05-20 - - `971-1 `_, `971-2 `_, `971-3 `_ - - `@erikjohnston`_ - - None - - - * - `MSC1215 `_ - - Groups as Rooms - - 2017-10-17 - - 2018-05-15 - - `1215-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1217 `_ - - Visibility of groups to group members and their non-members - - 2017-10-30 - - 2018-05-15 - - `1217-1 `_ - - `@lampholder`_ - - None - - - * - `MSC1218 `_ - - Bridging group membership with 3rd party group sources - - 2017-11-15 - - 2018-05-15 - - `1218-1 `_ - - `@lukebarnard1`_ - - None - - - * - `MSC1219 `_ - - Proposal for storing megolm keys serverside - - 2017-11-23 - - 2018-05-15 - - `1219-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1221 `_ - - Improving Presence - - 2017-12-26 - - 2018-05-24 - - `1221-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1222 `_ - - Pushing updates about Groups (Communities) to clients - - 2018-01-02 - - 2018-05-24 - - `1222-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1225 `_ - - Extensible event types & fallback in Matrix - - 2018-02-18 - - 2018-05-15 - - `1225-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1206 `_ - - Proposal for improved bot support - - 2018-03-14 - - 2018-06-08 - - `1206-1 `_ - - `@turt2live`_ - - None - - - * - `MSC1228 `_ - - Removing MXIDs from events - - 2018-04-19 - - 2018-05-15 - - `1228-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1231 `_ - - Handling Consent for Privacy Policy - - 2018-05-02 - - 2018-05-15 - - `1231-1 `_ - - `@neilisfragile`_ - - None - - - * - `MSC1267 `_ - - Interactive Key Verification - - 2018-05-28 - - 2018-05-28 - - `1267-1 `_ - - `@uhoreg`_ - - None - - - * - `MSC1280 `_ - - Mechanisms for communicating erasure requests to bots and federated homeservers - - 2018-06-05 - - 2018-06-05 - - `1280-1 `_ - - `@richvdh`_ - - None - - - * - `MSC688 `_ - - Calculate room names server-side - - 2018-06-10 - - 2018-06-15 - - `688-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1324 `_ - - Custom protocol definitions for application services - - 2018-06-20 - - 2018-06-20 - - `1324-1 `_ - - `@anoadragon453`_ - - None - - - * - `MSC1323 `_ - - AS traffic rate-limiting - - 2018-06-20 - - 2018-07-03 - - `1323-1 `_ - - `@anoadragon453`_ - - None - - - * - `MSC1322 `_ - - Mechanism to communicate 3PID binding updates or deletions to homeservers - - 2018-06-20 - - 2018-06-20 - - `1322-1 `_ - - `@babolivier`_ - - None - - - - - -proposal-ready-for-review -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC1227 `_ - - Proposal for lazy-loading room members to improve initial sync speed and client RAM usage - - 2018-03-05 - - 2018-06-10 - - `1227-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1206 `_ - - Proposal for improved bot support - - 2018-03-14 - - 2018-06-08 - - `1206-1 `_ - - `@turt2live`_ - - None - - - * - `MSC1256 `_ - - Custom emoji and sticker packs in matrix - - 2018-05-23 - - 2018-05-24 - - `1256-1 `_ - - `@turt2live`_ - - None - - - * - `MSC1270 `_ - - Proposal Add /_matrix/media/v1/resolve_url to Client-Server API: download and preview urls in the clients despite CORS - - 2018-05-31 - - 2018-06-19 - - `1270-1 `_ - - `@oivoodoo`_ - - None - - - * - `MSC701 `_ - - Auth for content repo (and enforcing GDPR erasure) - - 2018-06-04 - - 2018-06-07 - - `701-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1304 `_ - - Proposal to simplify the auth rules of m.room.power_level events. - - 2018-06-13 - - 2018-06-14 - - `1304-1 `_ - - `@richvdh`_, `@ara4n`_ - - None - - - * - `MSC1301 `_ - - Proposal for improving authorization for the matrix profile API - - 2018-06-13 - - 2018-06-13 - - `1301-1 `_ - - `@turt2live`_ - - None - - - * - `MSC1309 `_ - - Proposal for an application service management API - - 2018-06-14 - - 2018-06-15 - - `1309-1 `_ - - `@turt2live`_ - - None - - - * - `MSC1308 `_ - - Proposal for speeding up review of simple spec changes - - 2018-06-14 - - 2018-06-24 - - - - `@ara4n`_ - - None - - - * - `MSC1306 `_ - - Proposal to filter out traffic to Appservices based on filters - - 2018-06-14 - - 2018-06-14 - - `1306-1 `_ - - `@Half-Shot`_ - - None - - - - - -proposal-in-review -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC1194 `_ - - A way for HSes to remove bindings from ISes (aka unbind) - - 2018-05-09 - - 2018-06-05 - - `1194-1 `_ - - `@dbkr`_ - - None - - - - - -proposal-passed-review -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC433 `_ - - Support for discovering API endpoints via .well-known URIs (SPEC-121) - - 2015-03-08 - - 2018-07-01 - - `433-1 `_, `433-2 `_ - - `@maxidor`_, `others`_ - - `@uhoreg`_ - - - * - `MSC1226 `_ - - State Reset mitigation proposal - - 2018-02-20 - - 2018-05-15 - - `1226-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1229 `_ - - Mitigating abuse of the event depth parameter over federation - - 2018-04-30 - - 2018-05-15 - - `1229-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1232 `_ - - Media limits API - - 2018-05-04 - - 2018-06-21 - - `1232-1 `_ - - `@Half-Shot`_ - - None - - `PR#1189`_ - * - `MSC1236 `_ - - Matrix Widget API v2 - - 2018-05-13 - - 2018-05-15 - - `1236-1 `_ - - `@rxl881`_ - - None - - - - - -spec-pr-missing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC1200 `_ - - Configuration of E2E encryption in a room - - 2016-06-16 - - 2018-05-31 - - `1200-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1201 `_ - - Device Management API - - 2016-07-14 - - 2018-05-15 - - `1201-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1203 `_ - - 3rd Party Entity lookup API - - 2016-07-21 - - 2018-07-02 - - `1203-1 `_ - - `@leonerd`_ - - None - - - * - `MSC1208 `_ - - Encrypted attachment format - - 2016-10-26 - - 2018-05-15 - - `1208-1 `_ - - `@NegativeMjark`_ - - None - - - * - `MSC739 `_ - - Reporting inappropriate content in Matrix - - 2016-11-21 - - 2018-05-31 - - `739-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1211 `_ - - Megolm session export format - - 2017-01-03 - - 2018-05-15 - - `1211-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1212 `_ - - Device List Update Stream - - 2017-01-18 - - 2018-05-15 - - `1212-1 `_ - - `@richvdh`_ - - None - - - * - `MSC829 `_ - - Need to spec msisdn login API - - 2017-03-08 - - 2018-05-15 - - `829-1 `_ - - `@dbkr`_ - - None - - - * - `MSC855 `_ - - spec m.login.msisdn UI auth type - - 2017-03-24 - - 2018-05-15 - - `855-1 `_ - - `@dbkr`_ - - None - - - * - `MSC910 `_ - - Add new Read Marker API to docs - - 2017-05-08 - - 2018-05-15 - - - - `@lukebarnard1`_ - - None - - - * - `MSC1067 `_ - - Spec @mentions - - 2017-07-14 - - 2018-05-15 - - `1067-1 `_ - - `@lukebarnard1`_ - - None - - - * - `MSC1214 `_ - - Related Groups (i.e. flair) - - 2017-10-03 - - 2018-05-15 - - `1214-1 `_ - - `@lukebarnard1`_ - - None - - - * - `MSC1033 `_ - - Doc @room notifications - - 2017-10-23 - - 2018-05-31 - - - - `@dbkr`_ - - None - - - * - `MSC1183 `_ - - Document key-share requests - - 2018-04-30 - - 2018-05-31 - - `1183-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1230 `_ - - Temporary mitigation for depth parameter abuse - - 2018-05-01 - - 2018-05-15 - - `1230-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1234 `_ - - Rich Replies format - - 2018-05-12 - - 2018-05-18 - - `1234-1 `_ - - `@t3chguy`_ - - None - - - - - -merged -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC1197 `_ - - Ignoring Users - - 2016-05-03 - - 2018-05-18 - - `1197-1 `_ - - `@erikjohnston`_ - - None - - `PR#1142`_ - * - `MSC1199 `_ - - Notifications API - - 2016-05-23 - - 2018-06-25 - - `1199-1 `_ - - `@dbkr`_ - - None - - - * - `MSC1204 `_ - - Access Token Semantics (refresh and macaroons) - aka Auth Sept 2016 Edition - - 2016-09-29 - - 2018-06-25 - - `1204-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1205 `_ - - Proposal for multi-device deletion API - - 2016-10-12 - - 2018-06-25 - - `1205-1 `_ - - `@richvdh`_ - - None - - `PR#1239`_ - * - `MSC953 `_ - - Add /user_directory/search API - - 2017-05-31 - - 2018-05-10 - - `953-1 `_ - - `@erikjohnston`_ - - None - - - * - `MSC1233 `_ - - A proposal for organising spec proposals - - 2018-05-10 - - 2018-06-25 - - `1233-1 `_ - - `@ara4n`_ - - None - - `PR#1240`_ - - - -abandoned -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC531 `_ - - Homeservers as OAuth authorization endpoints (resource owners) (SPEC-206) - - 2015-07-25 - - 2018-05-15 - - `531-1 `_ - - `@Kegsay`_ - - None - - - * - `MSC1202 `_ - - Profile Personae - - 2016-07-15 - - 2018-05-15 - - `1202-1 `_ - - `@erikjohnston`_ - - None - - - * - `MSC1209 `_ - - Server Side Profile API - - 2016-11-01 - - 2018-05-15 - - `1209-1 `_ - - `@erikjohnston`_ - - None - - - * - `MSC1213 `_ - - Set defaults for m.federate - - 2017-04-10 - - 2018-05-18 - - `1213-1 `_ - - `@psaavedra`_ - - None - - - - - -obsolete -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC1223 `_ - - Replies event format - - 2018-02-01 - - 2018-05-15 - - `1223-1 `_ - - `@t3chguy`_ - - None - - - * - `MSC1224 `_ - - Replies - next steps - - 2018-02-03 - - 2018-05-15 - - `1224-1 `_ - - `@t3chguy`_ - - None - - - * - `MSC1235 `_ - - Proposal for Calendar Events - - 2018-02-06 - - 2018-05-15 - - `1235-1 `_ - - `@Half-Shot`_ - - None - - - * - `MSC1220 `_ - - Rich quoting proposal - - 2018-05-10 - - 2018-05-15 - - `1220-1 `_ - - `@t3chguy`_ - - None - - - - - - - -.. _@rxl881: https://github.com/rxl881 -.. _@turt2live: https://github.com/turt2live -.. _@erikjohnston: https://github.com/erikjohnston -.. _@anoadragon453: https://github.com/anoadragon453 -.. _@t3chguy: https://github.com/t3chguy -.. _@Kegsay: https://github.com/Kegsay -.. _@KitsuneRal: https://github.com/KitsuneRal -.. _@leonerd: https://github.com/leonerd -.. _@psaavedra: https://github.com/psaavedra -.. _@ara4n: https://github.com/ara4n -.. _@krombel: https://github.com/krombel -.. _@maxidor: https://github.com/maxidor -.. _@uhoreg: https://github.com/uhoreg -.. _@pik: https://github.com/pik -.. _@neilisfragile: https://github.com/neilisfragile -.. _@babolivier: https://github.com/babolivier -.. _@lukebarnard1: https://github.com/lukebarnard1 -.. _others: https://github.com/thers -.. _@Half-Shot: https://github.com/Half-Shot -.. _@aviraldg: https://github.com/aviraldg -.. _@oivoodoo: https://github.com/oivoodoo -.. _@richvdh: https://github.com/richvdh -.. _@NegativeMjark: https://github.com/NegativeMjark -.. _@lampholder: https://github.com/lampholder -.. _@dbkr: https://github.com/dbkr -.. _PR#1189: https://github.com/matrix-org/matrix-doc/pull/1189 -.. _PR#310: https://github.com/matrix-org/matrix-doc/pull/310 -.. _PR#1142: https://github.com/matrix-org/matrix-doc/pull/1142 -.. _PR#1239: https://github.com/matrix-org/matrix-doc/pull/1239 -.. _PR#1240: https://github.com/matrix-org/matrix-doc/pull/1240 -.. _PR#919: https://github.com/matrix-org/matrix-doc/pull/919 \ No newline at end of file +View the current live version `at https://matrix.org/docs/spec/proposals `_ diff --git a/specification/targets.yaml b/specification/targets.yaml index 96199d291..ba3e32996 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -12,7 +12,6 @@ targets: - { 1: modules.rst } - { 2: feature_profiles.rst } - { 2: "group:modules" } # reference a group of files - - { 1: third_party_lookup.rst } version_label: "%CLIENT_RELEASE_LABEL%" application_service: files: @@ -69,6 +68,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/ignore_users.rst - modules/stickers.rst - modules/report_content.rst + - modules/application_services.rst title_styles: ["=", "-", "~", "+", "^", "`", "@", ":"] From 9bde78ac288b198ed0af9514f0aa51b7f4d68bf4 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 12 Jul 2018 14:41:24 +0100 Subject: [PATCH 090/123] $ref needs to be under a schema: --- api/client-server/third_party_lookup.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 863896030..848d92d1d 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -129,7 +129,8 @@ paths: responses: 200: description: The Matrix User IDs found with the given parameters. - $ref: ../application-service/definitions/schema/user_batch.yaml + schema: + $ref: ../application-service/definitions/schema/user_batch.yaml 404: description: The Matrix User ID was not found examples: From 89998872452c79be9efdf60b6d0936b9069b8855 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 12 Jul 2018 14:44:48 +0100 Subject: [PATCH 091/123] Path -> query params --- api/client-server/third_party_lookup.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 848d92d1d..2fcf790b0 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -147,7 +147,7 @@ paths: alias. operationId: queryLocationByAlias parameters: - - in: path + - in: query name: alias type: string description: The Matrix room alias to look up. @@ -173,7 +173,7 @@ paths: Retreive an array of third party users from a Matrix User ID. operationId: queryUserByID parameters: - - in: path + - in: query name: userid type: string description: The Matrix User ID to look up. From 16125f16f5cc1f1f02f96d20c3c11d7e6858ac06 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 09:28:30 -0600 Subject: [PATCH 092/123] Spec retry_after_ms on ratelimit errors --- api/client-server/definitions/errors/rate_limited.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/client-server/definitions/errors/rate_limited.yaml b/api/client-server/definitions/errors/rate_limited.yaml index 6b4932ee6..aca82ce7b 100644 --- a/api/client-server/definitions/errors/rate_limited.yaml +++ b/api/client-server/definitions/errors/rate_limited.yaml @@ -23,4 +23,10 @@ properties: type: string description: A human-readable error message. example: Too many requests + retry_after_ms: + type: integer + description: |- + The amount of time in milliseconds the client should wait + before trying the request again. + example: 2000 required: ["errcode"] \ No newline at end of file From 84e07a0c10276ac3b4d87586e8a25e74faad9506 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 09:31:50 -0600 Subject: [PATCH 093/123] Convert errors in presence.yaml --- api/client-server/presence.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/client-server/presence.yaml b/api/client-server/presence.yaml index 68b3ee564..91b75c6a8 100644 --- a/api/client-server/presence.yaml +++ b/api/client-server/presence.yaml @@ -123,6 +123,8 @@ paths: description: |- There is no presence state for this user. This user may not exist or isn't exposing presence information to you. + schema: + "$ref": "definitions/errors/error.yaml" 403: description: You are not allowed to see this user's presence status. examples: @@ -131,7 +133,7 @@ paths: "error": "You are not allowed to see their presence" } schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/error.yaml" tags: - Presence "/presence/list/{userId}": From 291a4dfc7612635f77b4d71a89f549111bdb7e49 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 12 Jul 2018 16:52:31 +0100 Subject: [PATCH 094/123] Third party lookup module. Fix wording --- .../application_service.yaml | 10 +++---- .../definitions/{schema => }/location.yaml | 0 .../{schema => }/location_batch.yaml | 0 .../definitions/{schema => }/protocol.yaml | 0 .../{schema => }/protocol_metadata.yaml | 0 .../definitions/{schema => }/user.yaml | 0 .../definitions/{schema => }/user_batch.yaml | 0 api/client-server/third_party_lookup.yaml | 12 ++++---- specification/feature_profiles.rst | 4 +-- .../modules/application_services.rst | 28 ------------------- .../modules/third_party_networks.rst | 20 +++++++++++++ specification/targets.yaml | 2 +- 12 files changed, 34 insertions(+), 42 deletions(-) rename api/application-service/definitions/{schema => }/location.yaml (100%) rename api/application-service/definitions/{schema => }/location_batch.yaml (100%) rename api/application-service/definitions/{schema => }/protocol.yaml (100%) rename api/application-service/definitions/{schema => }/protocol_metadata.yaml (100%) rename api/application-service/definitions/{schema => }/user.yaml (100%) rename api/application-service/definitions/{schema => }/user_batch.yaml (100%) delete mode 100644 specification/modules/application_services.rst create mode 100644 specification/modules/third_party_networks.rst diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index b68cbe187..43a6023b2 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -226,7 +226,7 @@ paths: 200: description: The protocol was found and metadata returned. schema: - $ref: definitions/schema/protocol_metadata.yaml + $ref: definitions/protocol_metadata.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -279,7 +279,7 @@ paths: 200: description: The Matrix User IDs found with the given parameters. schema: - $ref: definitions/schema/user_batch.yaml + $ref: definitions/user_batch.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -330,7 +330,7 @@ paths: 200: description: At least one portal room was found. schema: - $ref: definitions/schema/location_batch.yaml + $ref: definitions/location_batch.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -375,7 +375,7 @@ paths: description: |- All found third party locations. schema: - $ref: definitions/schema/location_batch.yaml + $ref: definitions/location_batch.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -419,7 +419,7 @@ paths: description: |- An array of third party users. schema: - $ref: definitions/schema/user_batch.yaml + $ref: definitions/user_batch.yaml 401: description: |- The homeserver has not supplied credentials to the application service. diff --git a/api/application-service/definitions/schema/location.yaml b/api/application-service/definitions/location.yaml similarity index 100% rename from api/application-service/definitions/schema/location.yaml rename to api/application-service/definitions/location.yaml diff --git a/api/application-service/definitions/schema/location_batch.yaml b/api/application-service/definitions/location_batch.yaml similarity index 100% rename from api/application-service/definitions/schema/location_batch.yaml rename to api/application-service/definitions/location_batch.yaml diff --git a/api/application-service/definitions/schema/protocol.yaml b/api/application-service/definitions/protocol.yaml similarity index 100% rename from api/application-service/definitions/schema/protocol.yaml rename to api/application-service/definitions/protocol.yaml diff --git a/api/application-service/definitions/schema/protocol_metadata.yaml b/api/application-service/definitions/protocol_metadata.yaml similarity index 100% rename from api/application-service/definitions/schema/protocol_metadata.yaml rename to api/application-service/definitions/protocol_metadata.yaml diff --git a/api/application-service/definitions/schema/user.yaml b/api/application-service/definitions/user.yaml similarity index 100% rename from api/application-service/definitions/schema/user.yaml rename to api/application-service/definitions/user.yaml diff --git a/api/application-service/definitions/schema/user_batch.yaml b/api/application-service/definitions/user_batch.yaml similarity index 100% rename from api/application-service/definitions/schema/user_batch.yaml rename to api/application-service/definitions/user_batch.yaml diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 2fcf790b0..efb4310f6 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -39,7 +39,7 @@ paths: 200: description: The protocols supported by the homeserver. schema: - $ref: ../application-service/definitions/schema/protocol_metadata.yaml + $ref: ../application-service/definitions/protocol_metadata.yaml "/thirdparty/protocol/{protocol}": get: summary: Retrieve metadata about a specific protocol that the homeserver supports. @@ -58,7 +58,7 @@ paths: 200: description: The protocol was found and metadata returned. schema: - $ref: ../application-service/definitions/schema/protocol.yaml + $ref: ../application-service/definitions/protocol.yaml 404: description: The protocol is unknown. examples: @@ -97,7 +97,7 @@ paths: 200: description: At least one portal room was found. schema: - $ref: ../application-service/definitions/schema/location_batch.yaml + $ref: ../application-service/definitions/location_batch.yaml 404: description: No portal rooms were found. examples: @@ -130,7 +130,7 @@ paths: 200: description: The Matrix User IDs found with the given parameters. schema: - $ref: ../application-service/definitions/schema/user_batch.yaml + $ref: ../application-service/definitions/user_batch.yaml 404: description: The Matrix User ID was not found examples: @@ -157,7 +157,7 @@ paths: description: |- All found third party locations. schema: - $ref: ../application-service/definitions/schema/location_batch.yaml + $ref: ../application-service/definitions/location_batch.yaml 404: description: The Matrix room alias was not found examples: @@ -183,7 +183,7 @@ paths: description: |- An array of third party users. schema: - $ref: ../application-service/definitions/schema/user_batch.yaml + $ref: ../application-service/definitions/user_batch.yaml 404: description: The Matrix User ID was not found examples: diff --git a/specification/feature_profiles.rst b/specification/feature_profiles.rst index 97d0de0ea..c6b8ef4c0 100644 --- a/specification/feature_profiles.rst +++ b/specification/feature_profiles.rst @@ -42,7 +42,7 @@ Summary `Server Side Search`_ Optional Optional Optional Optional Optional `Server Administration`_ Optional Optional Optional Optional Optional `Event Context`_ Optional Optional Optional Optional Optional - `Application Services`_ Optional Optional Optional Optional Optional + `Third Party Networks`_ Optional Optional Optional Optional Optional ===================================== ========== ========== ========== ========== ========== *Please see each module for more details on what clients need to implement.* @@ -58,7 +58,7 @@ Summary .. _Server Side Search: `module:search`_ .. _Server Administration: `module:admin`_ .. _Event Context: `module:event-context`_ -.. _Application Services: `module:application-services`_ +.. _Third Party Networks: `module:third-party-networks`_ Clients ------- diff --git a/specification/modules/application_services.rst b/specification/modules/application_services.rst deleted file mode 100644 index 52e35dc8d..000000000 --- a/specification/modules/application_services.rst +++ /dev/null @@ -1,28 +0,0 @@ -Application Services -==================== - -.. _module:application-services: - -An application service is a separate program that interacts with a homeserver -and provides various bits of functionality that would otherwise not make -sense to include directly in the homeserver. This ranges from bots, which can -often be interacted with, to bridges, which allow Matrix users to communicate -with users on third party networks. The following describes endpoints that a -Matrix client can use to interact with an application service through the -facilitation of the homeserver. - -Third Party Lookups -------------------- - -Application services can provide access to third party networks via bridging. -This allows Matrix users to communicate with users on other communication -platforms, with messages ferried back and forth by the application service. A -single application service may bridge multiple third party networks, and many -individual locations within those networks. A single third party network -location may be bridged to multiple Matrix rooms. - -In order for a client to join one of these bridged rooms, or communicate -directly with a user on a third party network, the following endpoints can be -used. - -{{third_party_lookup_cs_http_api}} \ No newline at end of file diff --git a/specification/modules/third_party_networks.rst b/specification/modules/third_party_networks.rst new file mode 100644 index 000000000..cd4ce4143 --- /dev/null +++ b/specification/modules/third_party_networks.rst @@ -0,0 +1,20 @@ +Third Party Networks +==================== + +.. _module:third-party-networks: + +Application services can provide access to third party networks via bridging. +This allows Matrix users to communicate with users on other communication +platforms, with messages ferried back and forth by the application service. A +single application service may bridge multiple third party networks, and many +individual locations within those networks. A single third party network +location may be bridged to multiple Matrix rooms. + +Third Party Lookups +------------------- + +A client may wish to provide a rich interface for joining third party +locations and connecting with third party users. Information necessary for +such an interface is provided by third party lookups. + +{{third_party_lookup_cs_http_api}} \ No newline at end of file diff --git a/specification/targets.yaml b/specification/targets.yaml index ba3e32996..b9718bc49 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -68,7 +68,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/ignore_users.rst - modules/stickers.rst - modules/report_content.rst - - modules/application_services.rst + - modules/third_party_networks.rst title_styles: ["=", "-", "~", "+", "^", "`", "@", ":"] From af7460088fa756eb2b3afff2949f5ee281c2e2e0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 10:07:09 -0600 Subject: [PATCH 095/123] Use six for url quoting --- scripts/templating/matrix_templates/units.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 760aeb6b8..12db409f3 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -30,9 +30,8 @@ import re import subprocess import sys import yaml -import requests.utils from functools import reduce -from six.moves.urllib.parse import urlencode +from six.moves.urllib.parse import urlencode, quote matrix_doc_dir=reduce(lambda acc,_: os.path.dirname(acc), range(1, 5), os.path.abspath(__file__)) @@ -525,7 +524,7 @@ class MatrixUnits(Units): if param_loc == "path": path_template = path_template.replace( - "{%s}" % param_name, requests.utils.requote_uri(example) + "{%s}" % param_name, quote(example) ) elif param_loc == "query": if type(example) == list: From 8a4ba8c5cac07011159855735e96948f33787e08 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 10:16:32 -0600 Subject: [PATCH 096/123] Use Popen rather than run() --- scripts/templating/matrix_templates/units.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 12db409f3..cd563346c 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -838,21 +838,21 @@ class MatrixUnits(Units): tc_lines = [] if os.path.isdir(tc_path): logger.info("Generating towncrier changelog for: %s" % name) - p = subprocess.run( + p = subprocess.Popen( ['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'], cwd=tc_path, - check=False, # We'll manually check the exit code stderr=subprocess.PIPE, stdout=subprocess.PIPE, ) + stdout, stderr = p.communicate() if p.returncode != 0: # Something broke - dump as much information as we can logger.error("Towncrier exited with code %s" % p.returncode) - logger.error(p.stdout.decode('UTF-8')) - logger.error(p.stderr.decode('UTF-8')) + logger.error(stdout.decode('UTF-8')) + logger.error(stderr.decode('UTF-8')) raw_log = "" else: - raw_log = p.stdout.decode('UTF-8') + raw_log = stdout.decode('UTF-8') # This is a bit of a hack, but it does mean that the log at least gets *something* # to tell us it broke From 527a5a2fdccd72d38a7b7d3e621b31b9128a23c3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 10:18:46 -0600 Subject: [PATCH 097/123] Require sudo see also: https://github.com/travis-ci/travis-ci/issues/9875 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 27e5cf202..68fc77438 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: go go: - 1.8 -sudo: false +sudo: required # we only need a single git commit git: From 1829589d23f9c23cbcb7f3266408cb5db80f5517 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 12:16:11 -0600 Subject: [PATCH 098/123] Convert 3pn errors to new definition --- .../application_service.yaml | 42 +++++++++---------- api/client-server/third_party_lookup.yaml | 10 ++--- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 43a6023b2..b459e29ff 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -131,7 +131,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -140,7 +140,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 404: description: |- The application service indicates that this room alias does not exist. @@ -150,7 +150,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml "/users/{userId}": get: summary: Query if a user should exist on the application service. @@ -187,7 +187,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -196,7 +196,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 404: description: |- The application service indicates that this user does not exist. @@ -206,7 +206,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml "/_matrix/app/unstable/thirdparty/protocol/{protocol}": get: summary: Retrieve metadata about a specific protocol that the application service supports. @@ -236,7 +236,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -245,7 +245,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 404: description: No protocol was found with the given path. examples: @@ -253,7 +253,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml "/_matrix/app/unstable/thirdparty/user/{protocol}": get: summary: Retrieve the Matrix User ID of a corresponding third party user. @@ -289,7 +289,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -298,7 +298,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 404: description: No users were found with the given parameters. examples: @@ -306,7 +306,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml "/_matrix/app/unstable/thirdparty/location/{protocol}": get: summary: Retreive Matrix-side portal rooms leading to a third party location. @@ -340,7 +340,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -349,7 +349,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 404: description: No mappings were found with the given parameters. examples: @@ -357,7 +357,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml "/_matrix/app/unstable/thirdparty/location": get: summary: Reverse-lookup third party locations given a Matrix room alias. @@ -385,7 +385,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -394,7 +394,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 404: description: No mappings were found with the given parameters. examples: @@ -402,7 +402,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml "/_matrix/app/unstable/thirdparty/user": get: summary: Reverse-lookup third party users given a Matrix User ID. @@ -429,7 +429,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -438,7 +438,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 404: description: No mappings were found with the given parameters. examples: @@ -446,4 +446,4 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - $ref: ../client-server/definitions/error.yaml \ No newline at end of file + $ref: ../client-server/definitions/errors/error.yaml \ No newline at end of file diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index efb4310f6..55d4c70a2 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -66,7 +66,7 @@ paths: "errcode": "M_NOT_FOUND" } schema: - $ref: definitions/error.yaml + $ref: definitions/errors/error.yaml "/thirdparty/location/{protocol}": get: summary: Retreive Matrix-side portals rooms leading to a third party location. @@ -105,7 +105,7 @@ paths: "errcode": "M_NOT_FOUND" } schema: - $ref: definitions/error.yaml + $ref: definitions/errors/error.yaml "/thirdparty/user/{protocol}": get: summary: Retrieve the Matrix User ID of a corresponding third party user. @@ -138,7 +138,7 @@ paths: "errcode": "M_NOT_FOUND" } schema: - $ref: definitions/error.yaml + $ref: definitions/errors/error.yaml "/thirdparty/location": get: summary: Reverse-lookup third party locations given a Matrix room alias. @@ -165,7 +165,7 @@ paths: "errcode": "M_NOT_FOUND" } schema: - $ref: definitions/error.yaml + $ref: definitions/errors/error.yaml "/thirdparty/user": get: summary: Reverse-lookup third party users given a Matrix User ID. @@ -191,4 +191,4 @@ paths: "errcode": "M_NOT_FOUND" } schema: - $ref: definitions/error.yaml \ No newline at end of file + $ref: definitions/errors/error.yaml \ No newline at end of file From 1e500fd00563d52006807886cb223be107482d27 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 13:18:27 -0600 Subject: [PATCH 099/123] Add newsfragment for third party network endpoints Specifying multiple entries for the same PR would be covered under: https://github.com/hawkowl/towncrier/pull/119 --- changelogs/client_server/newsfragments/1353.new | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1353.new diff --git a/changelogs/client_server/newsfragments/1353.new b/changelogs/client_server/newsfragments/1353.new new file mode 100644 index 000000000..0af0c5206 --- /dev/null +++ b/changelogs/client_server/newsfragments/1353.new @@ -0,0 +1 @@ +``GET /thirdparty/*`` Endpoints From 51faf8ed2e4a63d4cfd6d23183698ed169956cc0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 13:46:42 -0600 Subject: [PATCH 100/123] Convert the general query API to swagger Includes "moving" the directory query API into the same area. This will make more sense once the other query types are specified. --- api/server-server/directory.yaml | 3 +- api/server-server/query_general.yaml | 44 ++++++++++++++++++++++++++++ specification/server_server_api.rst | 11 +------ 3 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 api/server-server/query_general.yaml diff --git a/api/server-server/directory.yaml b/api/server-server/directory.yaml index 516b1bf81..94f115b91 100644 --- a/api/server-server/directory.yaml +++ b/api/server-server/directory.yaml @@ -1,4 +1,5 @@ # Copyright 2017 Kamax.io +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +15,7 @@ swagger: '2.0' info: - title: "Matrix Federation Query Directory API" + title: "Matrix Federation Query API" version: "1.0.0" host: localhost:8448 schemes: diff --git a/api/server-server/query_general.yaml b/api/server-server/query_general.yaml new file mode 100644 index 000000000..97e3406ea --- /dev/null +++ b/api/server-server/query_general.yaml @@ -0,0 +1,44 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Query API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/query/{queryType}": + get: + summary: Query for information + description: |- + Performs a single query request on the receiving homeserver. The query string + arguments are dependent on which type of query is being made. Known query types + are specified as their own endpoints as an extension to this definition. + operationId: queryInfo + parameters: + - in: path + name: queryType + type: string + description: The type of query to make + required: true + x-example: profile + responses: + 200: + description: |- + The query response. The schema varies depending on the query being made. diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f2b3204ab..66c06c8b8 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -801,16 +801,7 @@ Retrieves all of the transactions later than any version given by the "v" arguments. -To make a query:: - - GET .../query/ - Query args: as specified by the individual query types - Response: JSON encoding of a response object - -Performs a single query request on the receiving homeserver. The Query Type -part of the path specifies the kind of query being made, and its query -arguments have a meaning specific to that kind of query. The response is a -JSON-encoded object whose meaning also depends on the kind of query. +{{query_general_ss_http_api}} To join a room:: From e5da1e17412c99437d2af50b9b62c150c1c40662 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 21:34:52 -0600 Subject: [PATCH 101/123] Improve upon the transaction schema and examples Reduces data duplication and makes the objects more reusable for other areas of the spec --- .../definitions/full_transaction.yaml | 28 --- api/server-server/definitions/pdu.yaml | 160 ++++-------------- .../definitions/unsigned_pdu.yaml | 119 +++++++++++++ .../examples/full_transaction.json | 6 - api/server-server/examples/pdu.json | 16 +- api/server-server/examples/unsigned_pdu.json | 17 ++ api/server-server/transactions.yaml | 15 +- scripts/templating/matrix_templates/units.py | 13 +- 8 files changed, 196 insertions(+), 178 deletions(-) delete mode 100644 api/server-server/definitions/full_transaction.yaml create mode 100644 api/server-server/definitions/unsigned_pdu.yaml delete mode 100644 api/server-server/examples/full_transaction.json create mode 100644 api/server-server/examples/unsigned_pdu.json diff --git a/api/server-server/definitions/full_transaction.yaml b/api/server-server/definitions/full_transaction.yaml deleted file mode 100644 index 39250433c..000000000 --- a/api/server-server/definitions/full_transaction.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2018 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -type: object -title: Transaction -description: Transaction -example: - $ref: "../examples/full_transaction.json" -allOf: - - $ref: "transaction.yaml" - - type: object - properties: - edus: - type: array - description: List of ephemeral messages. May be omitted if there are no ephemeral messages to be sent. - items: - $ref: "edu.yaml" -required: ['origin', 'origin_server_ts', 'pdus'] \ No newline at end of file diff --git a/api/server-server/definitions/pdu.yaml b/api/server-server/definitions/pdu.yaml index b3f25ccf1..2b6328be1 100644 --- a/api/server-server/definitions/pdu.yaml +++ b/api/server-server/definitions/pdu.yaml @@ -16,131 +16,37 @@ title: Persistent Data Unit description: A persistent data unit (event) example: $ref: "../examples/pdu.json" -properties: - room_id: - type: string - description: Room identifier - example: "!abc123:matrix.org" - sender: - type: string - description: The ID of the user sending the event - example: "@someone:matrix.org" - origin: - type: string - description: The ``server_name`` of the homeserver that created this event - example: "matrix.org" - origin_server_ts: - type: integer - format: int64 - description: Timestamp in milliseconds on origin homeserver when this event was created. - example: 1234567890 - type: - type: string - description: Event type - required: true - example: "m.room.message" - state_key: - type: string - description: |- - If this key is present, the event is a state event, and it will replace previous events - with the same ``type`` and ``state_key`` in the room state. - example: "my_key" - content: - type: object - description: The content of the event - prev_events: - type: array - description: |- - Event IDs and hashes of the most recent events in the room that the homeserver was aware - of when it made this event - items: - type: array - maxItems: 2 - minItems: 2 - items: - - type: string - title: Event ID - example: "$abc123:matrix.org" - - type: object - title: Event Hash - example: { - "sha256": "abase64encodedsha256hashshouldbe43byteslong" - } - properties: - sha256: - type: string - description: The event hash - example: abase64encodedsha256hashshouldbe43byteslong - required: ['sha256'] - depth: - type: integer - description: The maximum depth of the ``prev_events``, plus one - example: 12 - auth_events: - type: array - description: Event IDs and hashes for the "auth events" of this event - items: - type: array - maxItems: 2 - minItems: 2 - items: - - type: string - title: Event ID - example: "$abc123:matrix.org" - - type: object - title: Event Hash - example: { - "sha256": "abase64encodedsha256hashshouldbe43byteslong" - } - properties: - sha256: - type: string - description: The event hash - example: abase64encodedsha256hashshouldbe43byteslong - required: ['sha256'] - hashes: - type: object - title: Event Hash - description: Hashes of the PDU, following the algorithm specified in `Signing Events`_ - example: { - "sha256": "thishashcoversallfieldsincasethisisredacted" - } +allOf: + - $ref: "unsigned_pdu.yaml" + - type: object properties: - sha256: - type: string - description: The hash - example: thishashcoversallfieldsincasethisisredacted - required: ['sha256'] - signatures: - type: object - description: |- - Signatures of the redacted PDU, following the algorithm specified in `Signing Events`_ - example: { - "example.com": { - "ed25519:key_version:": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" - } - } - additionalProperties: - type: object - title: Server Signatures - additionalProperties: - type: string - redacts: - type: string - description: For redaction events, the ID of the event being redacted - example: "$def456:matrix.org" - unsigned: - type: object - description: Additional data added by the origin server but not covered by the ``signatures`` -required: - - room_id - - sender - - origin - - origin_server_ts - - type - - content - - prev_events - - depth - - auth_events - - hashes - - signatures + hashes: + type: object + title: Event Hash + description: Hashes of the PDU, following the algorithm specified in `Signing Events`_ + example: { + "sha256": "thishashcoversallfieldsincasethisisredacted" + } + properties: + sha256: + type: string + description: The hash + example: thishashcoversallfieldsincasethisisredacted + required: ['sha256'] + signatures: + type: object + description: |- + Signatures of the redacted PDU, following the algorithm specified in `Signing Events`_ + example: { + "example.com": { + "ed25519:key_version:": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" + } + } + additionalProperties: + type: object + title: Server Signatures + additionalProperties: + type: string + required: + - hashes + - signatures diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml new file mode 100644 index 000000000..173bddd4e --- /dev/null +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -0,0 +1,119 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +title: Unsigned Persistent Data Unit +description: An unsigned persistent data unit (event) +example: + $ref: "../examples/unsigned_pdu.json" +properties: + room_id: + type: string + description: Room identifier + example: "!abc123:matrix.org" + sender: + type: string + description: The ID of the user sending the event + example: "@someone:matrix.org" + origin: + type: string + description: The ``server_name`` of the homeserver that created this event + example: "matrix.org" + origin_server_ts: + type: integer + format: int64 + description: Timestamp in milliseconds on origin homeserver when this event was created. + example: 1234567890 + type: + type: string + description: Event type + required: true + example: "m.room.message" + state_key: + type: string + description: |- + If this key is present, the event is a state event, and it will replace previous events + with the same ``type`` and ``state_key`` in the room state. + example: "my_key" + content: + type: object + description: The content of the event + example: {"key": "value"} + prev_events: + type: array + description: |- + Event IDs and hashes of the most recent events in the room that the homeserver was aware + of when it made this event + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + depth: + type: integer + description: The maximum depth of the ``prev_events``, plus one + example: 12 + auth_events: + type: array + description: Event IDs and hashes for the "auth events" of this event + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + redacts: + type: string + description: For redaction events, the ID of the event being redacted + example: "$def456:matrix.org" + unsigned: + type: object + description: Additional data added by the origin server but not covered by the ``signatures`` + example: {"key": "value"} +required: + - room_id + - sender + - origin + - origin_server_ts + - type + - content + - prev_events + - depth + - auth_events diff --git a/api/server-server/examples/full_transaction.json b/api/server-server/examples/full_transaction.json deleted file mode 100644 index c453d6ed3..000000000 --- a/api/server-server/examples/full_transaction.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "origin": "matrix.org", - "origin_server_ts": 1234567890, - "pdus": [{"$ref": "pdu.json"}], - "edus": [{"$ref": "edu.json"}] -} \ No newline at end of file diff --git a/api/server-server/examples/pdu.json b/api/server-server/examples/pdu.json index 531f4a374..81981b23c 100644 --- a/api/server-server/examples/pdu.json +++ b/api/server-server/examples/pdu.json @@ -1,16 +1,5 @@ { - "room_id": "!UcYsUzyxTGDxLBEvLy:example.org", - "sender": "@alice:example.com", - "origin": "example.com", - "event_id": "$a4ecee13e2accdadf56c1025:example.com", - "origin_server_ts": 1404838188000, - "type": "m.room.message", - "prev_events": [ - [ - "$af232176:example.org", - {"sha256": "abase64encodedsha256hashshouldbe43byteslong"} - ] - ], + "$ref": "unsigned_pdu.json", "hashes": { "sha256": "thishashcoversallfieldsincasethisisredacted" }, @@ -18,8 +7,5 @@ "example.com": { "ed25519:key_version:": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" } - }, - "content": { - "key": "value" } } \ No newline at end of file diff --git a/api/server-server/examples/unsigned_pdu.json b/api/server-server/examples/unsigned_pdu.json new file mode 100644 index 000000000..6ed4b79d6 --- /dev/null +++ b/api/server-server/examples/unsigned_pdu.json @@ -0,0 +1,17 @@ +{ + "room_id": "!UcYsUzyxTGDxLBEvLy:example.org", + "sender": "@alice:example.com", + "origin": "example.com", + "event_id": "$a4ecee13e2accdadf56c1025:example.com", + "origin_server_ts": 1404838188000, + "type": "m.room.message", + "prev_events": [ + [ + "$af232176:example.org", + {"sha256": "abase64encodedsha256hashshouldbe43byteslong"} + ] + ], + "content": { + "key": "value" + } +} \ No newline at end of file diff --git a/api/server-server/transactions.yaml b/api/server-server/transactions.yaml index 389eecc7e..13ba6826b 100644 --- a/api/server-server/transactions.yaml +++ b/api/server-server/transactions.yaml @@ -43,8 +43,21 @@ paths: - in: body name: body type: object + required: true schema: - $ref: "definitions/full_transaction.yaml" + allOf: + - $ref: "definitions/transaction.yaml" + - type: object + properties: + edus: + type: array + description: List of ephemeral messages. May be omitted if there are no ephemeral messages to be sent. + items: + $ref: "definitions/edu.yaml" + example: { + "$ref": "examples/transaction.json", + "edus": [{"$ref": "edu.json"}] # Relative to the examples directory + } responses: 200: # TODO: Spec this (and figure out what it is) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 7819dbf90..8037bb61d 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -138,6 +138,10 @@ def inherit_parents(obj): Recurse through the 'allOf' declarations in the object """ logger.debug("inherit_parents %r" % obj) + + if isinstance(obj, list): + return [inherit_parents(obj[i]) for i in range(0, len(obj))] + parents = obj.get("allOf", []) if not parents: return obj @@ -379,6 +383,10 @@ def get_tables_for_response(schema): def get_example_for_schema(schema): """Returns a python object representing a suitable example for this object""" schema = inherit_parents(schema) + + if isinstance(schema, list): + return [get_example_for_schema(v) for v in schema] + if 'example' in schema: example = schema['example'] return example @@ -398,7 +406,10 @@ def get_example_for_schema(schema): if proptype == 'array': if 'items' not in schema: raise Exception('"array" property has neither items nor example') - return [get_example_for_schema(schema['items'])] + result = get_example_for_schema(schema['items']) + if isinstance(result, list): + return result + return [result] if proptype == 'integer': return 0 From baf19cc741071bf6e5b5c3235f56a9cd99488ae8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 13 Jul 2018 10:05:14 -0600 Subject: [PATCH 102/123] Add newsfragment for 1379 --- changelogs/client_server/newsfragments/1379.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1379.clarification diff --git a/changelogs/client_server/newsfragments/1379.clarification b/changelogs/client_server/newsfragments/1379.clarification new file mode 100644 index 000000000..122b39004 --- /dev/null +++ b/changelogs/client_server/newsfragments/1379.clarification @@ -0,0 +1 @@ +Document guest access in ``/createRoom`` presets From cf93c88337c15b491f9ae069d5f08ffd69c9fdc2 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 13 Jul 2018 15:44:33 -0600 Subject: [PATCH 103/123] Convert third party invite exchange to swagger --- api/server-server/third_party_invite.yaml | 190 ++++++++++++++++++++++ specification/server_server_api.rst | 33 +--- 2 files changed, 192 insertions(+), 31 deletions(-) create mode 100644 api/server-server/third_party_invite.yaml diff --git a/api/server-server/third_party_invite.yaml b/api/server-server/third_party_invite.yaml new file mode 100644 index 000000000..30d8fe9d2 --- /dev/null +++ b/api/server-server/third_party_invite.yaml @@ -0,0 +1,190 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Third Party Invites API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/exchange_third_party_invite/{roomId}": + put: + summary: Request a server to auth a third party invite event + description: |- + The receiving server will verify the partial ``m.room.member`` event + given in the request body. If valid, the receiving server will issue + an invite as per the `Inviting Users`_ section before returning a + response to this request. + operationId: exchangeThirdPartyInvite + parameters: + - in: path + name: roomId + type: string + description: The room ID to exchange a third party invite in + required: true + x-example: "!abc123:matrix.org" + - in: body + name: body + type: object + description: A partial ``m.room.member`` event + required: true + schema: + type: object + properties: + type: + type: string + description: The event type. Must be ``m.room.member`` + example: "m.room.member" + room_id: + type: string + description: |- + The room ID the event is for. Must match the ID given in + the path. + example: "!abc123:matrix.org" + sender: + type: string + description: |- + The user ID of the user who sent the original ``m.room.third_party_invite`` + event. + example: "@joe:matrix.org" + state_key: + type: string + description: The user ID of the invited user + example: "@someone:example.org" + content: + type: object + description: The event content + title: Event Content + properties: + membership: + type: string + description: The membership state. Must be ``invite`` + example: invite + third_party_invite: + type: object + description: The third party invite + properties: + display_name: + type: string + description: |- + A name which can be displayed to represent the user instead of their + third party identifier. + example: "alice" + signed: + type: object + description: |- + A block of content which has been signed, which servers can use to + verify the event. + properties: + signatures: + type: object + description: The server signatures for this event. + additionalProperties: + type: object + title: Server Signatures + additionalProperties: + type: string + example: { + "magic.forest": { + "ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg" + } + } + mxid: + type: string + description: The invited matrix user ID + example: "@alice:localhost" + token: + type: string + description: The token used to verify the event + example: abc123 + required: ['signatures', 'mxid', 'token'] + example: { + "mxid": "@alice:localhost", + "token": "abc123", + "signatures": { + "magic.forest": { + "ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg" + } + } + } + required: ['display_name', 'signed'] + example: { + "display_name": "alice", + "signed": { + "mxid": "@alice:localhost", + "token": "abc123", + "signatures": { + "magic.forest": { + "ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg" + } + } + } + } + required: ['membership', 'third_party_invite'] + example: { + "membership": "invite", + "third_party_invite": { + "display_name": "alice", + "signed": { + "mxid": "@alice:localhost", + "token": "abc123", + "signatures": { + "magic.forest": { + "ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg" + } + } + } + } + } + required: + - type + - room_id + - sender + - state_key + - content + example: { + "type": "m.room.member", + "room_id": "!abc123:matrix.org", + "sender": "@joe:matrix.org", + "state_key": "@someone:example.org", + "content": { + "membership": "invite", + "third_party_invite": { + "display_name": "alice", + "signed": { + "mxid": "@alice:localhost", + "token": "abc123", + "signatures": { + "magic.forest": { + "ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg" + } + } + } + } + } + } + responses: + 200: + description: The invite has been issued successfully. + examples: + application/json: {} + schema: + type: object + description: An empty object + example: {} diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 66c06c8b8..43524aca7 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -1104,38 +1104,9 @@ If the invited homeserver is in the room the invite came from, it can auth the event and send it. However, if the invited homeserver isn't in the room the invite came from, it -will need to request the room's homeserver to auth the event:: +will need to request the room's homeserver to auth the event. - PUT .../exchange_third_party_invite/{roomId} - -Where ``roomId`` is the ID of the room the invite is for. - -The required fields in the JSON body are: - -==================== ======= ================================================== - Key Type Description -==================== ======= ================================================== -``type`` String The event type. Must be ``m.room.member``. -``room_id`` String The ID of the room the event is for. Must be the - same as the ID specified in the path. -``sender`` String The Matrix ID of the user who sent the original - ``m.room.third_party_invite``. -``state_key`` String The Matrix ID of the invited user. -``content`` Object The content of the event. -==================== ======= ================================================== - -Where the ``content`` key contains the content for the ``m.room.member`` event -as described in the `Client-Server API`_. Its ``membership`` key must be -``invite`` and its content must include the ``third_party_invite`` object. - -The inviting homeserver will then be able to authenticate the event. It will send -a fully authenticated event to the invited homeserver as described in the `Inviting -to a room`_ section above. - -Once the invited homeserver responds with the event to which it appended its -signature, the inviting homeserver will respond with ``200 OK`` and an empty body -(``{}``) to the initial request on ``/exchange_third_party_invite/{roomId}`` and -send the now verified ``m.room.member`` invite event to the room's members. +{{third_party_invite_ss_http_api}} Verifying the invite ++++++++++++++++++++ From 272f3d843245eefe6b0874193af0e848e613a7ea Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 13 Jul 2018 15:49:49 -0600 Subject: [PATCH 104/123] Fix documentation link --- api/server-server/third_party_invite.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/third_party_invite.yaml b/api/server-server/third_party_invite.yaml index 30d8fe9d2..42bdae23e 100644 --- a/api/server-server/third_party_invite.yaml +++ b/api/server-server/third_party_invite.yaml @@ -29,7 +29,7 @@ paths: description: |- The receiving server will verify the partial ``m.room.member`` event given in the request body. If valid, the receiving server will issue - an invite as per the `Inviting Users`_ section before returning a + an invite as per the `Inviting to a room`_ section before returning a response to this request. operationId: exchangeThirdPartyInvite parameters: From cfac4a1ab38ed05fc677c999b8ab379e1ff97eca Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 17 Jul 2018 08:48:31 -0600 Subject: [PATCH 105/123] Use towncrier 18.6.0 final --- scripts/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 1f92c3dc9..2a7d7ff85 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -7,5 +7,5 @@ Jinja2 >= 2.9.6 jsonschema >= 2.6.0 PyYAML >= 3.12 requests >= 2.18.4 -towncrier == 18.6.0rc1 +towncrier == 18.6.0 six >= 1.11.0 \ No newline at end of file From 98c607bff5e59d9d1db4a5af34fb197ac996d428 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 17 Jul 2018 10:27:01 -0600 Subject: [PATCH 106/123] Advertise python 3.4 support and use that in Travis CI We also don't need to update virtualenv, or use python3.5 specific packages. --- .travis.yml | 5 ++--- README.rst | 4 ++-- scripts/test-and-build.sh | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 68fc77438..9e9363aff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,8 @@ git: depth: 1 install: -- sudo apt-get update -- sudo apt-get install python3.5 python3.5-dev -- sudo pip install --upgrade virtualenv + - sudo apt-get update + - sudo apt-get install python3 python3-dev script: - ./scripts/test-and-build.sh diff --git a/README.rst b/README.rst index c0949d9e6..b8847bfb6 100644 --- a/README.rst +++ b/README.rst @@ -41,9 +41,9 @@ specs and event schemas in this repository. Preparation ----------- -To use the scripts, it is best to create a Python 3.5+ virtualenv as follows:: +To use the scripts, it is best to create a Python 3.4+ virtualenv as follows:: - virtualenv env + virtualenv -p python3 env env/bin/pip install -r scripts/requirements.txt (Benjamin Synders has contributed a script for `Nix`_ users, which can be diff --git a/scripts/test-and-build.sh b/scripts/test-and-build.sh index 397b620ed..710b03ddf 100755 --- a/scripts/test-and-build.sh +++ b/scripts/test-and-build.sh @@ -4,7 +4,7 @@ set -ex cd `dirname $0`/.. -virtualenv -p python3.5 env +virtualenv -p python3 env . env/bin/activate # Print out the python versions for debugging purposes From 9e6f2b2fd01a3f1ecd7d9e56169c2d52690b0f49 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 18 Jul 2018 08:29:13 -0600 Subject: [PATCH 107/123] Add newsfragment --- changelogs/client_server/newsfragments/1373.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1373.clarification diff --git a/changelogs/client_server/newsfragments/1373.clarification b/changelogs/client_server/newsfragments/1373.clarification new file mode 100644 index 000000000..21e18966b --- /dev/null +++ b/changelogs/client_server/newsfragments/1373.clarification @@ -0,0 +1 @@ +Describe the rate limit error response schema From bafdcf364011c5e8e1337b23762eea71f8256aab Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 18 Jul 2018 10:37:56 -0600 Subject: [PATCH 108/123] Full stops, spelling, and operation IDs. --- api/server-server/definitions/keys.yaml | 18 ++++++------ .../definitions/keys_query_response.yaml | 2 +- api/server-server/keys_query.yaml | 28 +++++++++---------- api/server-server/keys_server.yaml | 4 +-- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/api/server-server/definitions/keys.yaml b/api/server-server/definitions/keys.yaml index 5d1c39def..b08f24650 100644 --- a/api/server-server/definitions/keys.yaml +++ b/api/server-server/definitions/keys.yaml @@ -19,12 +19,12 @@ example: properties: server_name: type: string - description: DNS name of the homeserver + description: DNS name of the homeserver. required: true # TODO: Verify example: "example.org" verify_keys: type: object - description: Public keys of the homeserver for verifying digital signatures + description: Public keys of the homeserver for verifying digital signatures. required: true # TODO: Verify additionalProperties: type: object @@ -42,7 +42,7 @@ properties: example: "Base+64+Encoded+Signature+Verification+Key" old_verify_keys: type: object - description: The public keys that the server used to use and when it stopped using them + description: The public keys that the server used to use and when it stopped using them. additionalProperties: type: object title: Old Verify Key @@ -56,17 +56,17 @@ properties: expired_ts: type: integer format: int64 - description: The expiration time + description: The expiration time. required: true example: 922834800000 key: type: string - description: The key + description: The key. required: true example: "Base+64+Encoded+Signature+Verification+Key" signatures: type: object - description: Digital signatures for this object signed using the ``verify_keys`` + description: Digital signatures for this object signed using the ``verify_keys``. additionalProperties: type: object title: Signed Server @@ -80,17 +80,17 @@ properties: name: Encoded Signature Verification Key tls_fingerprints: type: array - description: Hashes of X.509 TLS certificates used by this server encoded as `Unpadded Base64`_ + description: Hashes of X.509 TLS certificates used by this server encoded as `Unpadded Base64`_. items: type: object title: TLS Fingerprint properties: sha256: type: string - description: The encoded fingerprint + description: The encoded fingerprint. example: Base+64+Encoded+SHA-256-Fingerprint valid_until_ts: type: integer format: int64 - description: POSIX timestamp when the list of valid keys should be refreshed + description: POSIX timestamp when the list of valid keys should be refreshed. example: 1052262000000 diff --git a/api/server-server/definitions/keys_query_response.yaml b/api/server-server/definitions/keys_query_response.yaml index 8223f9ec1..bf2d238f8 100644 --- a/api/server-server/definitions/keys_query_response.yaml +++ b/api/server-server/definitions/keys_query_response.yaml @@ -22,6 +22,6 @@ properties: server_keys: type: array title: Server Keys - description: The server keys + description: The server keys. items: $ref: "keys.yaml" diff --git a/api/server-server/keys_query.yaml b/api/server-server/keys_query.yaml index 1ba0f5ba3..f41cb35b2 100644 --- a/api/server-server/keys_query.yaml +++ b/api/server-server/keys_query.yaml @@ -25,27 +25,27 @@ produces: paths: "/query/{serverName}/{keyId}": get: - summary: Retreive a server key - description: Retreive a server key - operationId: getQueryKeys + summary: Retrieve a server key. + description: Retrieve a server key. + operationId: perspectivesKeyQuery parameters: - in: path name: serverName type: string - description: Server name + description: Server name. required: true x-example: matrix.org - in: path name: keyId type: string - description: Key ID + description: Key ID. required: true x-example: TODO # No examples in spec so far - in: query name: minimum_valid_until_ts type: integer format: int64 - description: Minimum Valid Until Milliseconds + description: Minimum Valid Until Milliseconds. required: true # TODO: Verify x-example: 1234567890 responses: @@ -55,9 +55,9 @@ paths: $ref: "definitions/keys_query_response.yaml" "/query": post: - summary: Retreive a server key - description: Retreive a server key - operationId: postQueryKeys + summary: Retrieve a server key + description: Retrieve a server key. + operationId: bulkPerspectivesKeyQuery parameters: - in: body name: body @@ -76,24 +76,24 @@ paths: properties: server_keys: type: object - description: The query criteria + description: The query criteria. additionalProperties: type: object name: ServerName - description: The server names to query + description: The server names to query. additionalProperties: type: object title: Query Criteria - description: The server keys to query + description: The server keys to query. properties: minimum_valid_until_ts: type: integer format: int64 - description: Minimum Valid Until MS + description: Minimum Valid Until MS. example: 1234567890 required: ['server_keys'] responses: 200: - description: The keys for the server + description: The keys for the server. schema: $ref: "definitions/keys_query_response.yaml" diff --git a/api/server-server/keys_server.yaml b/api/server-server/keys_server.yaml index 819a2599d..46beeebba 100644 --- a/api/server-server/keys_server.yaml +++ b/api/server-server/keys_server.yaml @@ -26,7 +26,7 @@ paths: "/server/{keyId}": get: summary: Get the server's key - description: Get the server's key + description: Get the server's key. operationId: getServerKey parameters: - in: path @@ -37,6 +37,6 @@ paths: x-example: TODO # No examples in the spec so far responses: 200: - description: The server's keys + description: The server's keys. schema: $ref: "definitions/keys.yaml" From 89384c96c18310d084fa3c2695c51109780b9f53 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 18 Jul 2018 11:06:23 -0600 Subject: [PATCH 109/123] Full stops, language. --- api/server-server/definitions/edu.yaml | 10 ++++----- api/server-server/definitions/pdu.yaml | 8 +++---- .../definitions/transaction.yaml | 6 ++--- .../definitions/unsigned_pdu.yaml | 22 +++++++++---------- api/server-server/events.yaml | 21 +++++++++--------- 5 files changed, 34 insertions(+), 33 deletions(-) diff --git a/api/server-server/definitions/edu.yaml b/api/server-server/definitions/edu.yaml index 4597ca623..af12752c2 100644 --- a/api/server-server/definitions/edu.yaml +++ b/api/server-server/definitions/edu.yaml @@ -22,16 +22,16 @@ example: properties: edu_type: type: string - description: The type of ephemeral message - example: "!abc123:matrix.org" + description: The type of ephemeral message. + example: "m.presence" origin: type: string - description: The server name sending the ephemeral message + description: The server name sending the ephemeral message. example: "matrix.org" destination: type: string - description: The server name receiving the ephemeral message + description: The server name receiving the ephemeral message. example: "elsewhere.com" content: type: object - description: The content of the ephemeral message + description: The content of the ephemeral message. diff --git a/api/server-server/definitions/pdu.yaml b/api/server-server/definitions/pdu.yaml index 2b6328be1..a5da57c04 100644 --- a/api/server-server/definitions/pdu.yaml +++ b/api/server-server/definitions/pdu.yaml @@ -23,20 +23,20 @@ allOf: hashes: type: object title: Event Hash - description: Hashes of the PDU, following the algorithm specified in `Signing Events`_ + description: Hashes of the PDU, following the algorithm specified in `Signing Events`_. example: { "sha256": "thishashcoversallfieldsincasethisisredacted" } properties: sha256: type: string - description: The hash - example: thishashcoversallfieldsincasethisisredacted + description: The hash. + example: thishashcoversallfieldsincasthisisredacted required: ['sha256'] signatures: type: object description: |- - Signatures of the redacted PDU, following the algorithm specified in `Signing Events`_ + Signatures of the redacted PDU, following the algorithm specified in `Signing Events`_. example: { "example.com": { "ed25519:key_version:": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" diff --git a/api/server-server/definitions/transaction.yaml b/api/server-server/definitions/transaction.yaml index 630d2ba36..e4c1f559e 100644 --- a/api/server-server/definitions/transaction.yaml +++ b/api/server-server/definitions/transaction.yaml @@ -20,16 +20,16 @@ properties: origin: type: string description: |- - The ``server_name`` of hoemserver sending this transaction + The ``server_name`` of the hoemserver sending this transaction. example: "example.org" origin_server_ts: type: integer format: int64 - description: Timestamp in milliseconds on originating homeserver when this transaction started + description: Timestamp in milliseconds on originating homeserver when this transaction started. example: 1234567890 pdus: type: array - description: List of persistent updates to rooms + description: List of persistent updates to rooms. items: $ref: "pdu.yaml" required: ['origin', 'origin_server_ts', 'pdus'] \ No newline at end of file diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index 173bddd4e..ca0a11835 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -19,15 +19,15 @@ example: properties: room_id: type: string - description: Room identifier + description: Room identifier. example: "!abc123:matrix.org" sender: type: string - description: The ID of the user sending the event + description: The ID of the user sending the event. example: "@someone:matrix.org" origin: type: string - description: The ``server_name`` of the homeserver that created this event + description: The ``server_name`` of the homeserver that created this event. example: "matrix.org" origin_server_ts: type: integer @@ -47,13 +47,13 @@ properties: example: "my_key" content: type: object - description: The content of the event + description: The content of the event. example: {"key": "value"} prev_events: type: array description: |- Event IDs and hashes of the most recent events in the room that the homeserver was aware - of when it made this event + of when it made this event. items: type: array maxItems: 2 @@ -70,16 +70,16 @@ properties: properties: sha256: type: string - description: The event hash + description: The event hash. example: abase64encodedsha256hashshouldbe43byteslong required: ['sha256'] depth: type: integer - description: The maximum depth of the ``prev_events``, plus one + description: The maximum depth of the ``prev_events``, plus one. example: 12 auth_events: type: array - description: Event IDs and hashes for the "auth events" of this event + description: Event IDs and hashes for the "auth events" of this event. items: type: array maxItems: 2 @@ -96,16 +96,16 @@ properties: properties: sha256: type: string - description: The event hash + description: The event hash. example: abase64encodedsha256hashshouldbe43byteslong required: ['sha256'] redacts: type: string - description: For redaction events, the ID of the event being redacted + description: For redaction events, the ID of the event being redacted. example: "$def456:matrix.org" unsigned: type: object - description: Additional data added by the origin server but not covered by the ``signatures`` + description: Additional data added by the origin server but not covered by the ``signatures``. example: {"key": "value"} required: - room_id diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index 055883c26..08498c357 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -33,37 +33,37 @@ paths: - in: path name: roomId type: string - description: The room ID to get state for + description: The room ID to get state for. required: true x-example: "!abc123:matrix.org" responses: 200: - description: The room state for the room (kept under ``pdus``) + description: The room state for the room (kept under ``pdus``). schema: $ref: "definitions/transaction.yaml" "/event/{eventId}": get: summary: Get a single event description: |- - Retrieves a single event + Retrieves a single event. operationId: getEvent parameters: - in: path name: eventId type: string - description: The event ID to get + description: The event ID to get. required: true x-example: "$abc123:matrix.org" responses: 200: - description: A transaction containing a single PDU which is the event requested + description: A transaction containing a single PDU which is the event requested. schema: $ref: "definitions/transaction.yaml" "/backfill/{roomId}": get: summary: Retrieves the events which precede the given event description: |- - Retreives a sliding-window history of previous PDUs that occurred on the given room. + Retreives a sliding-window history of previous PDUs that occurred in the given room. Starting from the PDU ID(s) given in the ``v`` argument, the PDUs that preceded it are retrived, up to the total number given by the ``limit``. operationId: backfillRoom @@ -71,19 +71,19 @@ paths: - in: path name: roomId type: string - description: The room ID to backfill + description: The room ID to backfill. required: true x-example: "!abc123:matrix.org" - in: query name: v type: string # TODO: The description says this is plural - figure out how to specify multiple, and spec it - description: The event ID to backfill from + description: The event ID to backfill from. required: true x-example: "$abc123:matrix.org" - in: query name: limit type: integer - description: The maximum number of events to retrieve + description: The maximum number of events to retrieve. required: true # TODO: Verify x-example: 10 responses: @@ -91,6 +91,7 @@ paths: description: A transaction containing the PDUs that preceded the given event(s). schema: $ref: "definitions/transaction.yaml" + # TODO: It's possible that this endpoint doesn't exist anymore - verify "/pull": get: summary: Stream events later than a given point in history @@ -101,7 +102,7 @@ paths: - in: query name: v type: string # TODO: The description says this is plural - figure out how to specify multiple, and spec it - description: The event ID to backfill from + description: The event ID to backfill from. required: true x-example: "$abc123:matrix.org" - in: query From 2dae0c9c49542854005291a71d542b9c06fedb59 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 19 Jul 2018 08:54:04 -0600 Subject: [PATCH 110/123] use python3 in dump-swagger.py --- scripts/dump-swagger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dump-swagger.py b/scripts/dump-swagger.py index 7994324f6..e02c554eb 100755 --- a/scripts/dump-swagger.py +++ b/scripts/dump-swagger.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # dump-swagger reads all of the swagger API docs used in spec generation and # outputs a JSON file which merges them all, for use as input to a swagger UI From 54d6d58fff385981bf8fb60abee083cfd74334a7 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Fri, 20 Jul 2018 14:15:47 +0100 Subject: [PATCH 111/123] naively move content from intro.rst to index.rst --- specification/index.rst | 384 ++++++++++++++++++++++++++++++++++- specification/intro.rst | 400 ------------------------------------- specification/targets.yaml | 3 - 3 files changed, 382 insertions(+), 405 deletions(-) delete mode 100644 specification/intro.rst diff --git a/specification/index.rst b/specification/index.rst index f9a3fef34..cb8b92414 100644 --- a/specification/index.rst +++ b/specification/index.rst @@ -30,18 +30,392 @@ communication ecosystem. To propose a change to the Matrix Spec, see the explanations at `Proposals for Spec Changes to Matrix `_. +.. contents:: Table of Contents +.. sectnum:: + Matrix APIs ----------- The specification consists of the following parts: -`Introduction to Matrix `_ provides a full introduction to Matrix and the spec. - {{apis}} The `Appendices `_ contain supplemental information not specific to one of the above APIs. +Introduction to the Matrix APIs +------------------------------- +.. WARNING:: + The Matrix specification is still evolving: the APIs are not yet frozen + and this document is in places a work in progress or stale. We have made every + effort to clearly flag areas which are still being finalised. + We're publishing it at this point because it's complete enough to be more than + useful and provide a canonical reference to how Matrix is evolving. Our end + goal is to mirror WHATWG's `Living Standard + `_. + +Matrix is a set of open APIs for open-federated Instant Messaging (IM), Voice +over IP (VoIP) and Internet of Things (IoT) communication, designed to create +and support a new global real-time communication ecosystem. The intention is to +provide an open decentralised pubsub layer for the internet for securely +persisting and publishing/subscribing JSON objects. This specification is the +ongoing result of standardising the APIs used by the various components of the +Matrix ecosystem to communicate with one another. + +The principles that Matrix attempts to follow are: + +- Pragmatic Web-friendly APIs (i.e. JSON over REST) +- Keep It Simple & Stupid + + + provide a simple architecture with minimal third-party dependencies. + +- Fully open: + + + Fully open federation - anyone should be able to participate in the global + Matrix network + + Fully open standard - publicly documented standard with no IP or patent + licensing encumbrances + + Fully open source reference implementation - liberally-licensed example + implementations with no IP or patent licensing encumbrances + +- Empowering the end-user + + + The user should be able to choose the server and clients they use + + The user should be control how private their communication is + + The user should know precisely where their data is stored + +- Fully decentralised - no single points of control over conversations or the + network as a whole +- Learning from history to avoid repeating it + + + Trying to take the best aspects of XMPP, SIP, IRC, SMTP, IMAP and NNTP + whilst trying to avoid their failings + + +The functionality that Matrix provides includes: + +- Creation and management of fully distributed chat rooms with no + single points of control or failure +- Eventually-consistent cryptographically secure synchronisation of room + state across a global open network of federated servers and services +- Sending and receiving extensible messages in a room with (optional) + end-to-end encryption +- Extensible user management (inviting, joining, leaving, kicking, banning) + mediated by a power-level based user privilege system. +- Extensible room state management (room naming, aliasing, topics, bans) +- Extensible user profile management (avatars, display names, etc) +- Managing user accounts (registration, login, logout) +- Use of 3rd Party IDs (3PIDs) such as email addresses, phone numbers, + Facebook accounts to authenticate, identify and discover users on Matrix. +- Trusted federation of Identity servers for: + + + Publishing user public keys for PKI + + Mapping of 3PIDs to Matrix IDs + + +The end goal of Matrix is to be a ubiquitous messaging layer for synchronising +arbitrary data between sets of people, devices and services - be that for +instant messages, VoIP call setups, or any other objects that need to be +reliably and persistently pushed from A to B in an interoperable and federated +manner. + + +Spec Change Proposals +~~~~~~~~~~~~~~~~~~~~~ +To propose a change to the Matrix Spec, see the explanations at `Proposals +for Spec Changes to Matrix `_. + + +Architecture +------------ + +Matrix defines APIs for synchronising extensible JSON objects known as +"events" between compatible clients, servers and services. Clients are +typically messaging/VoIP applications or IoT devices/hubs and communicate by +synchronising communication history with their "homeserver" using the +"Client-Server API". Each homeserver stores the communication history and +account information for all of its clients, and shares data with the wider +Matrix ecosystem by synchronising communication history with other homeservers +and their clients. + +Clients typically communicate with each other by emitting events in the +context of a virtual "room". Room data is replicated across *all of the +homeservers* whose users are participating in a given room. As such, *no +single homeserver has control or ownership over a given room*. Homeservers +model communication history as a partially ordered graph of events known as +the room's "event graph", which is synchronised with eventual consistency +between the participating servers using the "Server-Server API". This process +of synchronising shared conversation history between homeservers run by +different parties is called "Federation". Matrix optimises for the +Availability and Partitioned properties of CAP theorem at +the expense of Consistency. + +For example, for client A to send a message to client B, client A performs an +HTTP PUT of the required JSON event on its homeserver (HS) using the +client-server API. A's HS appends this event to its copy of the room's event +graph, signing the message in the context of the graph for integrity. A's HS +then replicates the message to B's HS by performing an HTTP PUT using the +server-server API. B's HS authenticates the request, validates the event's +signature, authorises the event's contents and then adds it to its copy of the +room's event graph. Client B then receives the message from his homeserver via +a long-lived GET request. + +:: + + How data flows between clients + ============================== + + { Matrix client A } { Matrix client B } + ^ | ^ | + | events | Client-Server API | events | + | V | V + +------------------+ +------------------+ + | |---------( HTTPS )--------->| | + | homeserver | | homeserver | + | |<--------( HTTPS )----------| | + +------------------+ Server-Server API +------------------+ + History Synchronisation + (Federation) + + +Users +~~~~~ + +Each client is associated with a user account, which is identified in Matrix +using a unique "user ID". This ID is namespaced to the homeserver which +allocated the account and has the form:: + + @localpart:domain + +See the `appendices `_ for full details of +the structure of user IDs. + +Devices +~~~~~~~ + +The Matrix specification has a particular meaning for the term "device". As a +user, I might have several devices: a desktop client, some web browsers, an +Android device, an iPhone, etc. They broadly relate to a real device in the +physical world, but you might have several browsers on a physical device, or +several Matrix client applications on a mobile device, each of which would be +its own device. + +Devices are used primarily to manage the keys used for end-to-end encryption +(each device gets its own copy of the decryption keys), but they also help +users manage their access - for instance, by revoking access to particular +devices. + +When a user first uses a client, it registers itself as a new device. The +longevity of devices might depend on the type of client. A web client will +probably drop all of its state on logout, and create a new device every time +you log in, to ensure that cryptography keys are not leaked to a new user. In +a mobile client, it might be acceptable to reuse the device if a login session +expires, provided the user is the same. + +Devices are identified by a ``device_id``, which is unique within the scope of +a given user. + +A user may assign a human-readable display name to a device, to help them +manage their devices. + +Events +~~~~~~ + +All data exchanged over Matrix is expressed as an "event". Typically each client +action (e.g. sending a message) correlates with exactly one event. Each event +has a ``type`` which is used to differentiate different kinds of data. ``type`` +values MUST be uniquely globally namespaced following Java's `package naming +conventions`_, e.g. +``com.example.myapp.event``. The special top-level namespace ``m.`` is reserved +for events defined in the Matrix specification - for instance ``m.room.message`` +is the event type for instant messages. Events are usually sent in the context +of a "Room". + +.. _package naming conventions: https://en.wikipedia.org/wiki/Java_package#Package_naming_conventions + +Event Graphs +~~~~~~~~~~~~ + +.. _sect:event-graph: + +Events exchanged in the context of a room are stored in a directed acyclic graph +(DAG) called an "event graph". The partial ordering of this graph gives the +chronological ordering of events within the room. Each event in the graph has a +list of zero or more "parent" events, which refer to any preceding events +which have no chronological successor from the perspective of the homeserver +which created the event. + +Typically an event has a single parent: the most recent message in the room at +the point it was sent. However, homeservers may legitimately race with each +other when sending messages, resulting in a single event having multiple +successors. The next event added to the graph thus will have multiple parents. +Every event graph has a single root event with no parent. + +To order and ease chronological comparison between the events within the graph, +homeservers maintain a ``depth`` metadata field on each event. An event's +``depth`` is a positive integer that is strictly greater than the depths of any +of its parents. The root event should have a depth of 1. Thus if one event is +before another, then it must have a strictly smaller depth. + +Room structure +~~~~~~~~~~~~~~ + +A room is a conceptual place where users can send and receive events. Events are +sent to a room, and all participants in that room with sufficient access will +receive the event. Rooms are uniquely identified internally via "Room IDs", +which have the form:: + + !opaque_id:domain + +There is exactly one room ID for each room. Whilst the room ID does contain a +domain, it is simply for globally namespacing room IDs. The room does NOT +reside on the domain specified. + +See the `appendices `_ for full details of +the structure of a room ID. + +The following conceptual diagram shows an +``m.room.message`` event being sent to the room ``!qporfwt:matrix.org``:: + + { @alice:matrix.org } { @bob:domain.com } + | ^ + | | + [HTTP POST] [HTTP GET] + Room ID: !qporfwt:matrix.org Room ID: !qporfwt:matrix.org + Event type: m.room.message Event type: m.room.message + Content: { JSON object } Content: { JSON object } + | | + V | + +------------------+ +------------------+ + | homeserver | | homeserver | + | matrix.org | | domain.com | + +------------------+ +------------------+ + | ^ + | [HTTP PUT] | + | Room ID: !qporfwt:matrix.org | + | Event type: m.room.message | + | Content: { JSON object } | + `-------> Pointer to the preceding message ------` + PKI signature from matrix.org + Transaction-layer metadata + PKI Authorization header + + ................................... + | Shared Data | + | State: | + | Room ID: !qporfwt:matrix.org | + | Servers: matrix.org, domain.com | + | Members: | + | - @alice:matrix.org | + | - @bob:domain.com | + | Messages: | + | - @alice:matrix.org | + | Content: { JSON object } | + |...................................| + +Federation maintains *shared data structures* per-room between multiple home +servers. The data is split into ``message events`` and ``state events``. + +Message events: + These describe transient 'once-off' activity in a room such as an + instant messages, VoIP call setups, file transfers, etc. They generally + describe communication activity. + +State events: + These describe updates to a given piece of persistent information + ('state') related to a room, such as the room's name, topic, membership, + participating servers, etc. State is modelled as a lookup table of key/value + pairs per room, with each key being a tuple of ``state_key`` and ``event type``. + Each state event updates the value of a given key. + +The state of the room at a given point is calculated by considering all events +preceding and including a given event in the graph. Where events describe the +same state, a merge conflict algorithm is applied. The state resolution +algorithm is transitive and does not depend on server state, as it must +consistently select the same event irrespective of the server or the order the +events were received in. Events are signed by the originating server (the +signature includes the parent relations, type, depth and payload hash) and are +pushed over federation to the participating servers in a room, currently using +full mesh topology. Servers may also request backfill of events over federation +from the other servers participating in a room. + + +Room Aliases +++++++++++++ + +Each room can also have multiple "Room Aliases", which look like:: + + #room_alias:domain + +See the `appendices `_ for full details of +the structure of a room alias. + +A room alias "points" to a room ID and is the human-readable label by which +rooms are publicised and discovered. The room ID the alias is pointing to can +be obtained by visiting the domain specified. Note that the mapping from a room +alias to a room ID is not fixed, and may change over time to point to a +different room ID. For this reason, Clients SHOULD resolve the room alias to a +room ID once and then use that ID on subsequent requests. + +When resolving a room alias the server will also respond with a list of servers +that are in the room that can be used to join via. + +:: + + HTTP GET + #matrix:domain.com !aaabaa:matrix.org + | ^ + | | + _______V____________________|____ + | domain.com | + | Mappings: | + | #matrix >> !aaabaa:matrix.org | + | #golf >> !wfeiofh:sport.com | + | #bike >> !4rguxf:matrix.org | + |________________________________| + +Identity +~~~~~~~~ + +Users in Matrix are identified via their Matrix user ID. However, +existing 3rd party ID namespaces can also be used in order to identify Matrix +users. A Matrix "Identity" describes both the user ID and any other existing IDs +from third party namespaces *linked* to their account. +Matrix users can *link* third-party IDs (3PIDs) such as email addresses, social +network accounts and phone numbers to their user ID. Linking 3PIDs creates a +mapping from a 3PID to a user ID. This mapping can then be used by Matrix +users in order to discover the user IDs of their contacts. +In order to ensure that the mapping from 3PID to user ID is genuine, a globally +federated cluster of trusted "Identity Servers" (IS) are used to verify the 3PID +and persist and replicate the mappings. + +Usage of an IS is not required in order for a client application to be part of +the Matrix ecosystem. However, without one clients will not be able to look up +user IDs using 3PIDs. + + +Profiles +~~~~~~~~ + +Users may publish arbitrary key/value data associated with their account - such +as a human readable display name, a profile photo URL, contact information +(email address, phone numbers, website URLs etc). + +.. TODO + Actually specify the different types of data - e.g. what format are display + names allowed to be? + +Private User Data +~~~~~~~~~~~~~~~~~ + +Users may also store arbitrary private key/value data in their account - such as +client preferences, or server configuration settings which lack any other +dedicated API. The API is symmetrical to managing Profile data. + +.. TODO + Would it really be overengineered to use the same API for both profile & + private user data, but with different ACLs? + Specification Versions ---------------------- @@ -57,3 +431,9 @@ The specification for each API is versioned in the form ``rX.Y.Z``. * A change to ``Z`` represents a change which is backwards-compatible on both sides. Typically this implies a clarification to the specification, rather than a change which must be implemented. + +License +------- + +The Matrix specification is licensed under the `Apache License, Version 2.0 +`_. diff --git a/specification/intro.rst b/specification/intro.rst deleted file mode 100644 index ad5452484..000000000 --- a/specification/intro.rst +++ /dev/null @@ -1,400 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -.. contents:: Table of Contents -.. sectnum:: - -.. Note that this file is specifically unversioned because we don't want to -.. have to add Yet Another version number, and the commentary on what specs we -.. have should hopefully not get complex enough that we need to worry about -.. versioning it. - -Introduction ------------- -.. WARNING:: - The Matrix specification is still evolving: the APIs are not yet frozen - and this document is in places a work in progress or stale. We have made every - effort to clearly flag areas which are still being finalised. - We're publishing it at this point because it's complete enough to be more than - useful and provide a canonical reference to how Matrix is evolving. Our end - goal is to mirror WHATWG's `Living Standard - `_. - -Matrix is a set of open APIs for open-federated Instant Messaging (IM), Voice -over IP (VoIP) and Internet of Things (IoT) communication, designed to create -and support a new global real-time communication ecosystem. The intention is to -provide an open decentralised pubsub layer for the internet for securely -persisting and publishing/subscribing JSON objects. This specification is the -ongoing result of standardising the APIs used by the various components of the -Matrix ecosystem to communicate with one another. - -The principles that Matrix attempts to follow are: - -- Pragmatic Web-friendly APIs (i.e. JSON over REST) -- Keep It Simple & Stupid - - + provide a simple architecture with minimal third-party dependencies. - -- Fully open: - - + Fully open federation - anyone should be able to participate in the global - Matrix network - + Fully open standard - publicly documented standard with no IP or patent - licensing encumbrances - + Fully open source reference implementation - liberally-licensed example - implementations with no IP or patent licensing encumbrances - -- Empowering the end-user - - + The user should be able to choose the server and clients they use - + The user should be control how private their communication is - + The user should know precisely where their data is stored - -- Fully decentralised - no single points of control over conversations or the - network as a whole -- Learning from history to avoid repeating it - - + Trying to take the best aspects of XMPP, SIP, IRC, SMTP, IMAP and NNTP - whilst trying to avoid their failings - - -The functionality that Matrix provides includes: - -- Creation and management of fully distributed chat rooms with no - single points of control or failure -- Eventually-consistent cryptographically secure synchronisation of room - state across a global open network of federated servers and services -- Sending and receiving extensible messages in a room with (optional) - end-to-end encryption -- Extensible user management (inviting, joining, leaving, kicking, banning) - mediated by a power-level based user privilege system. -- Extensible room state management (room naming, aliasing, topics, bans) -- Extensible user profile management (avatars, display names, etc) -- Managing user accounts (registration, login, logout) -- Use of 3rd Party IDs (3PIDs) such as email addresses, phone numbers, - Facebook accounts to authenticate, identify and discover users on Matrix. -- Trusted federation of Identity servers for: - - + Publishing user public keys for PKI - + Mapping of 3PIDs to Matrix IDs - - -The end goal of Matrix is to be a ubiquitous messaging layer for synchronising -arbitrary data between sets of people, devices and services - be that for -instant messages, VoIP call setups, or any other objects that need to be -reliably and persistently pushed from A to B in an interoperable and federated -manner. - - -Spec Change Proposals -~~~~~~~~~~~~~~~~~~~~~ -To propose a change to the Matrix Spec, see the explanations at `Proposals -for Spec Changes to Matrix `_. - - -Architecture ------------- - -Matrix defines APIs for synchronising extensible JSON objects known as -"events" between compatible clients, servers and services. Clients are -typically messaging/VoIP applications or IoT devices/hubs and communicate by -synchronising communication history with their "homeserver" using the -"Client-Server API". Each homeserver stores the communication history and -account information for all of its clients, and shares data with the wider -Matrix ecosystem by synchronising communication history with other homeservers -and their clients. - -Clients typically communicate with each other by emitting events in the -context of a virtual "room". Room data is replicated across *all of the -homeservers* whose users are participating in a given room. As such, *no -single homeserver has control or ownership over a given room*. Homeservers -model communication history as a partially ordered graph of events known as -the room's "event graph", which is synchronised with eventual consistency -between the participating servers using the "Server-Server API". This process -of synchronising shared conversation history between homeservers run by -different parties is called "Federation". Matrix optimises for the -Availability and Partitioned properties of CAP theorem at -the expense of Consistency. - -For example, for client A to send a message to client B, client A performs an -HTTP PUT of the required JSON event on its homeserver (HS) using the -client-server API. A's HS appends this event to its copy of the room's event -graph, signing the message in the context of the graph for integrity. A's HS -then replicates the message to B's HS by performing an HTTP PUT using the -server-server API. B's HS authenticates the request, validates the event's -signature, authorises the event's contents and then adds it to its copy of the -room's event graph. Client B then receives the message from his homeserver via -a long-lived GET request. - -:: - - How data flows between clients - ============================== - - { Matrix client A } { Matrix client B } - ^ | ^ | - | events | Client-Server API | events | - | V | V - +------------------+ +------------------+ - | |---------( HTTPS )--------->| | - | homeserver | | homeserver | - | |<--------( HTTPS )----------| | - +------------------+ Server-Server API +------------------+ - History Synchronisation - (Federation) - - -Users -~~~~~ - -Each client is associated with a user account, which is identified in Matrix -using a unique "user ID". This ID is namespaced to the homeserver which -allocated the account and has the form:: - - @localpart:domain - -See the `appendices `_ for full details of -the structure of user IDs. - -Devices -~~~~~~~ - -The Matrix specification has a particular meaning for the term "device". As a -user, I might have several devices: a desktop client, some web browsers, an -Android device, an iPhone, etc. They broadly relate to a real device in the -physical world, but you might have several browsers on a physical device, or -several Matrix client applications on a mobile device, each of which would be -its own device. - -Devices are used primarily to manage the keys used for end-to-end encryption -(each device gets its own copy of the decryption keys), but they also help -users manage their access - for instance, by revoking access to particular -devices. - -When a user first uses a client, it registers itself as a new device. The -longevity of devices might depend on the type of client. A web client will -probably drop all of its state on logout, and create a new device every time -you log in, to ensure that cryptography keys are not leaked to a new user. In -a mobile client, it might be acceptable to reuse the device if a login session -expires, provided the user is the same. - -Devices are identified by a ``device_id``, which is unique within the scope of -a given user. - -A user may assign a human-readable display name to a device, to help them -manage their devices. - -Events -~~~~~~ - -All data exchanged over Matrix is expressed as an "event". Typically each client -action (e.g. sending a message) correlates with exactly one event. Each event -has a ``type`` which is used to differentiate different kinds of data. ``type`` -values MUST be uniquely globally namespaced following Java's `package naming -conventions`_, e.g. -``com.example.myapp.event``. The special top-level namespace ``m.`` is reserved -for events defined in the Matrix specification - for instance ``m.room.message`` -is the event type for instant messages. Events are usually sent in the context -of a "Room". - -.. _package naming conventions: https://en.wikipedia.org/wiki/Java_package#Package_naming_conventions - -Event Graphs -~~~~~~~~~~~~ - -.. _sect:event-graph: - -Events exchanged in the context of a room are stored in a directed acyclic graph -(DAG) called an "event graph". The partial ordering of this graph gives the -chronological ordering of events within the room. Each event in the graph has a -list of zero or more "parent" events, which refer to any preceding events -which have no chronological successor from the perspective of the homeserver -which created the event. - -Typically an event has a single parent: the most recent message in the room at -the point it was sent. However, homeservers may legitimately race with each -other when sending messages, resulting in a single event having multiple -successors. The next event added to the graph thus will have multiple parents. -Every event graph has a single root event with no parent. - -To order and ease chronological comparison between the events within the graph, -homeservers maintain a ``depth`` metadata field on each event. An event's -``depth`` is a positive integer that is strictly greater than the depths of any -of its parents. The root event should have a depth of 1. Thus if one event is -before another, then it must have a strictly smaller depth. - -Room structure -~~~~~~~~~~~~~~ - -A room is a conceptual place where users can send and receive events. Events are -sent to a room, and all participants in that room with sufficient access will -receive the event. Rooms are uniquely identified internally via "Room IDs", -which have the form:: - - !opaque_id:domain - -There is exactly one room ID for each room. Whilst the room ID does contain a -domain, it is simply for globally namespacing room IDs. The room does NOT -reside on the domain specified. - -See the `appendices `_ for full details of -the structure of a room ID. - -The following conceptual diagram shows an -``m.room.message`` event being sent to the room ``!qporfwt:matrix.org``:: - - { @alice:matrix.org } { @bob:domain.com } - | ^ - | | - [HTTP POST] [HTTP GET] - Room ID: !qporfwt:matrix.org Room ID: !qporfwt:matrix.org - Event type: m.room.message Event type: m.room.message - Content: { JSON object } Content: { JSON object } - | | - V | - +------------------+ +------------------+ - | homeserver | | homeserver | - | matrix.org | | domain.com | - +------------------+ +------------------+ - | ^ - | [HTTP PUT] | - | Room ID: !qporfwt:matrix.org | - | Event type: m.room.message | - | Content: { JSON object } | - `-------> Pointer to the preceding message ------` - PKI signature from matrix.org - Transaction-layer metadata - PKI Authorization header - - ................................... - | Shared Data | - | State: | - | Room ID: !qporfwt:matrix.org | - | Servers: matrix.org, domain.com | - | Members: | - | - @alice:matrix.org | - | - @bob:domain.com | - | Messages: | - | - @alice:matrix.org | - | Content: { JSON object } | - |...................................| - -Federation maintains *shared data structures* per-room between multiple home -servers. The data is split into ``message events`` and ``state events``. - -Message events: - These describe transient 'once-off' activity in a room such as an - instant messages, VoIP call setups, file transfers, etc. They generally - describe communication activity. - -State events: - These describe updates to a given piece of persistent information - ('state') related to a room, such as the room's name, topic, membership, - participating servers, etc. State is modelled as a lookup table of key/value - pairs per room, with each key being a tuple of ``state_key`` and ``event type``. - Each state event updates the value of a given key. - -The state of the room at a given point is calculated by considering all events -preceding and including a given event in the graph. Where events describe the -same state, a merge conflict algorithm is applied. The state resolution -algorithm is transitive and does not depend on server state, as it must -consistently select the same event irrespective of the server or the order the -events were received in. Events are signed by the originating server (the -signature includes the parent relations, type, depth and payload hash) and are -pushed over federation to the participating servers in a room, currently using -full mesh topology. Servers may also request backfill of events over federation -from the other servers participating in a room. - - -Room Aliases -++++++++++++ - -Each room can also have multiple "Room Aliases", which look like:: - - #room_alias:domain - -See the `appendices `_ for full details of -the structure of a room alias. - -A room alias "points" to a room ID and is the human-readable label by which -rooms are publicised and discovered. The room ID the alias is pointing to can -be obtained by visiting the domain specified. Note that the mapping from a room -alias to a room ID is not fixed, and may change over time to point to a -different room ID. For this reason, Clients SHOULD resolve the room alias to a -room ID once and then use that ID on subsequent requests. - -When resolving a room alias the server will also respond with a list of servers -that are in the room that can be used to join via. - -:: - - HTTP GET - #matrix:domain.com !aaabaa:matrix.org - | ^ - | | - _______V____________________|____ - | domain.com | - | Mappings: | - | #matrix >> !aaabaa:matrix.org | - | #golf >> !wfeiofh:sport.com | - | #bike >> !4rguxf:matrix.org | - |________________________________| - -Identity -~~~~~~~~ - -Users in Matrix are identified via their Matrix user ID. However, -existing 3rd party ID namespaces can also be used in order to identify Matrix -users. A Matrix "Identity" describes both the user ID and any other existing IDs -from third party namespaces *linked* to their account. -Matrix users can *link* third-party IDs (3PIDs) such as email addresses, social -network accounts and phone numbers to their user ID. Linking 3PIDs creates a -mapping from a 3PID to a user ID. This mapping can then be used by Matrix -users in order to discover the user IDs of their contacts. -In order to ensure that the mapping from 3PID to user ID is genuine, a globally -federated cluster of trusted "Identity Servers" (IS) are used to verify the 3PID -and persist and replicate the mappings. - -Usage of an IS is not required in order for a client application to be part of -the Matrix ecosystem. However, without one clients will not be able to look up -user IDs using 3PIDs. - - -Profiles -~~~~~~~~ - -Users may publish arbitrary key/value data associated with their account - such -as a human readable display name, a profile photo URL, contact information -(email address, phone numbers, website URLs etc). - -.. TODO - Actually specify the different types of data - e.g. what format are display - names allowed to be? - -Private User Data -~~~~~~~~~~~~~~~~~ - -Users may also store arbitrary private key/value data in their account - such as -client preferences, or server configuration settings which lack any other -dedicated API. The API is symmetrical to managing Profile data. - -.. TODO - Would it really be overengineered to use the same API for both profile & - private user data, but with different ACLs? - -License -------- - -The Matrix specification is licensed under the `Apache License, Version 2.0 -`_. diff --git a/specification/targets.yaml b/specification/targets.yaml index b9718bc49..53957e0af 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -2,9 +2,6 @@ targets: index: files: - index.rst - intro: - files: - - intro.rst client_server: files: - client_server_api.rst From e7c79f2bd4a8e6d1bca5c457833e7f7350dd9cbf Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Fri, 20 Jul 2018 15:30:58 +0100 Subject: [PATCH 112/123] avoid .rst link conflict warnings --- specification/index.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/index.rst b/specification/index.rst index cb8b92414..36c899580 100644 --- a/specification/index.rst +++ b/specification/index.rst @@ -187,7 +187,7 @@ allocated the account and has the form:: @localpart:domain -See the `appendices `_ for full details of +See `'Identifier Grammar' the appendices `_ for full details of the structure of user IDs. Devices @@ -271,7 +271,7 @@ There is exactly one room ID for each room. Whilst the room ID does contain a domain, it is simply for globally namespacing room IDs. The room does NOT reside on the domain specified. -See the `appendices `_ for full details of +See `'Identifier Grammar' in the appendices `_ for full details of the structure of a room ID. The following conceptual diagram shows an @@ -347,7 +347,7 @@ Each room can also have multiple "Room Aliases", which look like:: #room_alias:domain -See the `appendices `_ for full details of +See `'Identifier Grammar' in the appendices `_ for full details of the structure of a room alias. A room alias "points" to a room ID and is the human-readable label by which From 59bdcb5615e5716d5b184f725d72ddaf8c02ea3c Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 19 Jul 2018 21:46:57 +0100 Subject: [PATCH 113/123] Clone and configure the swagger UI as a circle CI artifact Signed-off-by: Stuart Mumford --- .circleci/config.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c11eb70dc..3a0b60325 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,12 +4,26 @@ gendoc: &gendoc source /env/bin/activate scripts/gendoc.py -gendoc: &genswagger +genswagger: &genswagger name: Generate the swagger command: | source /env/bin/activate scripts/dump-swagger.py +buildswaggerui: &buildswaggerui + name: Build Swagger UI + command: | + ls scripts/ + mkdir -p api/client-server + git clone https://github.com/matrix-org/swagger-ui swagger-ui + cp -r swagger-ui/dist/* api/client-server/ + mkdir -p api/client-server/json + cp scripts/swagger/api-docs.json api/client-server/json/ + wget https://raw.githubusercontent.com/matrix-org/matrix.org/master/content/swagger.css -O api/client-server/swagger.css + wget https://raw.githubusercontent.com/matrix-org/matrix.org/master/scripts/swagger-ui.patch + patch api/client-server/index.html swagger-ui.patch + + version: 2 jobs: build-docs: @@ -30,11 +44,12 @@ jobs: steps: - checkout - run: *genswagger + - run: *buildswaggerui - store_artifacts: - path: scripts/swagger/api-docs.json + path: api/client-server/ - run: name: "Swagger UI is available at:" - command: SWAGGER_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/scripts/swagger/api-docs.json"; echo "https://matrix.org/docs/api/client-server/?url="$SWAGGER_URL + command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/api/client-server/index.html"; echo $DOCS_URL workflows: version: 2 From 03b74476f12d65ab5fb1d36ae9b25f8bd11d50d5 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 23 Jul 2018 20:20:52 +0100 Subject: [PATCH 114/123] Add config for Giles Signed-off-by: Stuart Mumford --- pyproject.toml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..61f77ec60 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[ tool.giles ] + + [ tool.giles.circleci_artifacts.docs ] + url = "gen/index.html" + message = "Click details to preview the HTML documentation." + + [ tool.giles.circleci_artifacts.random ] + url = "client-server/index.html" + message = "Click to preview the swagger build." From 7c71b697e7b0ac6ccb8227e3557199422bc18a47 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Tue, 24 Jul 2018 11:07:11 +0100 Subject: [PATCH 115/123] Fix typo in Giles config Signed-off-by: Stuart Mumford --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 61f77ec60..b53982b87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,6 @@ url = "gen/index.html" message = "Click details to preview the HTML documentation." - [ tool.giles.circleci_artifacts.random ] + [ tool.giles.circleci_artifacts.swagger ] url = "client-server/index.html" message = "Click to preview the swagger build." From 07aeaadef914f68b0e6bca274c8d656da91d52cf Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Tue, 24 Jul 2018 11:41:03 +0100 Subject: [PATCH 116/123] update links which used to point to docs/spec/intro --- specification/client_server_api.rst | 2 +- specification/modules/cas_login.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 8dde43029..1bb989ff6 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -1417,7 +1417,7 @@ have to wait in milliseconds before they can try again. .. References .. _`macaroon`: http://research.google.com/pubs/pub41892.html -.. _`devices`: ../intro.html#devices +.. _`devices`: ../index.html#devices .. Links through the external API docs are below .. ============================================= diff --git a/specification/modules/cas_login.rst b/specification/modules/cas_login.rst index b651431a3..5de980575 100644 --- a/specification/modules/cas_login.rst +++ b/specification/modules/cas_login.rst @@ -98,9 +98,9 @@ check for certain user attributes in the response. Any required attributes should be configured by the server administrator. Once the ticket has been validated, the server MUST map the CAS ``user_id`` -to a valid `Matrix user identifier <../intro.html#user-identifiers>`_. The +to a valid `Matrix user identifier <../index.html#user-identifiers>`_. The guidance in `Mapping from other character sets -<../intro.html#mapping-from-other-character-sets>`_ may be useful. +<../index.html#mapping-from-other-character-sets>`_ may be useful. If the generated user identifier represents a new user, it should be registered as a new user. From 21c4af70a95b3bb7d7f8767e91487a5df5475a17 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 09:13:15 -0600 Subject: [PATCH 117/123] Fix schema definitions (spelling, attributes) --- api/server-server/definitions/edu.yaml | 5 ++++- api/server-server/definitions/transaction.yaml | 2 +- api/server-server/definitions/unsigned_pdu.yaml | 1 - api/server-server/events.yaml | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/api/server-server/definitions/edu.yaml b/api/server-server/definitions/edu.yaml index af12752c2..c89573feb 100644 --- a/api/server-server/definitions/edu.yaml +++ b/api/server-server/definitions/edu.yaml @@ -24,14 +24,17 @@ properties: type: string description: The type of ephemeral message. example: "m.presence" + required: true origin: type: string description: The server name sending the ephemeral message. - example: "matrix.org" + example: "matrix.org" + required: true destination: type: string description: The server name receiving the ephemeral message. example: "elsewhere.com" + required: true content: type: object description: The content of the ephemeral message. diff --git a/api/server-server/definitions/transaction.yaml b/api/server-server/definitions/transaction.yaml index e4c1f559e..930ddec1a 100644 --- a/api/server-server/definitions/transaction.yaml +++ b/api/server-server/definitions/transaction.yaml @@ -20,7 +20,7 @@ properties: origin: type: string description: |- - The ``server_name`` of the hoemserver sending this transaction. + The ``server_name`` of the homeserver sending this transaction. example: "example.org" origin_server_ts: type: integer diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index ca0a11835..011b27363 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -37,7 +37,6 @@ properties: type: type: string description: Event type - required: true example: "m.room.message" state_key: type: string diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index 08498c357..96b2a8251 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -65,7 +65,7 @@ paths: description: |- Retreives a sliding-window history of previous PDUs that occurred in the given room. Starting from the PDU ID(s) given in the ``v`` argument, the PDUs that preceded it - are retrived, up to the total number given by the ``limit``. + are retrieved, up to the total number given by the ``limit``. operationId: backfillRoom parameters: - in: path @@ -84,7 +84,7 @@ paths: name: limit type: integer description: The maximum number of events to retrieve. - required: true # TODO: Verify + required: true x-example: 10 responses: 200: From 88beaf4ad8f5a27bbdd5a3c30d0fd002ab431954 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 09:13:46 -0600 Subject: [PATCH 118/123] Completely remove /pull It's not been in use for years - we can safely let it disappear --- api/server-server/events.yaml | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index 96b2a8251..d540085d2 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -91,29 +91,3 @@ paths: description: A transaction containing the PDUs that preceded the given event(s). schema: $ref: "definitions/transaction.yaml" - # TODO: It's possible that this endpoint doesn't exist anymore - verify - "/pull": - get: - summary: Stream events later than a given point in history - description: |- - Retrieves all of the transactions later than any version given by the ``v`` arguments. - operationId: pull - parameters: - - in: query - name: v - type: string # TODO: The description says this is plural - figure out how to specify multiple, and spec it - description: The event ID to backfill from. - required: true - x-example: "$abc123:matrix.org" - - in: query - name: origin - type: string - description: The origin # TODO: What is this actually? - required: true # TODO: Verify - x-example: "matrix.org" - responses: - 200: - # TODO: This could do with a better description - description: A transaction containing multiple PDUs - schema: - $ref: "definitions/transaction.yaml" \ No newline at end of file From a4e5a461f9b5886b5183f28072916c0a0cbba3c1 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 09:14:31 -0600 Subject: [PATCH 119/123] Improve handing of lists appearing in items --- scripts/templating/matrix_templates/units.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index fc2b91817..c99b7c387 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -140,9 +140,6 @@ def inherit_parents(obj): """ logger.debug("inherit_parents %r" % obj) - if isinstance(obj, list): - return [inherit_parents(obj[i]) for i in range(0, len(obj))] - parents = obj.get("allOf", []) if not parents: return obj @@ -298,6 +295,8 @@ def process_data_type(prop, required=False, enforce_title=True): elif prop_type == "array": items = prop["items"] + # Items can be a list of schemas or a schema itself + # http://json-schema.org/latest/json-schema-validation.html#rfc.section.6.4 if isinstance(items, list): prop_title = "[" for i in items: @@ -385,9 +384,6 @@ def get_example_for_schema(schema): """Returns a python object representing a suitable example for this object""" schema = inherit_parents(schema) - if isinstance(schema, list): - return [get_example_for_schema(v) for v in schema] - if 'example' in schema: example = schema['example'] return example @@ -407,10 +403,13 @@ def get_example_for_schema(schema): if proptype == 'array': if 'items' not in schema: raise Exception('"array" property has neither items nor example') - result = get_example_for_schema(schema['items']) + items = schema['items'] + if isinstance(items, list): + return [get_example_for_schema(i) for i in items] + result = get_example_for_schema(items) if isinstance(result, list): - return result - return [result] + return [result] + return result if proptype == 'integer': return 0 From 042772aaf58141c9b5a433134c88b0817e8af648 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 10:13:23 -0600 Subject: [PATCH 120/123] Make nested titles better --- scripts/templating/matrix_templates/units.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index c99b7c387..e6493f2ea 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -298,12 +298,12 @@ def process_data_type(prop, required=False, enforce_title=True): # Items can be a list of schemas or a schema itself # http://json-schema.org/latest/json-schema-validation.html#rfc.section.6.4 if isinstance(items, list): - prop_title = "[" + nested_titles = [] for i in items: nested = process_data_type(i) tables.extend(nested['tables']) - prop_title = "%s%s, " % (prop_title, nested['title']) - prop_title = prop_title[:-2] + "]" + nested_titles.extend([nested['title']]) + prop_title = "[%s]" % (", ".join(nested_titles), ) else: nested = process_data_type(prop["items"]) prop_title = "[%s]" % nested["title"] From 5ade15534d128519a6af77777e76aa79e466fa56 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 10:17:39 -0600 Subject: [PATCH 121/123] Remove extra branch in example generation --- scripts/templating/matrix_templates/units.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index e6493f2ea..15c987fd4 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -406,10 +406,7 @@ def get_example_for_schema(schema): items = schema['items'] if isinstance(items, list): return [get_example_for_schema(i) for i in items] - result = get_example_for_schema(items) - if isinstance(result, list): - return [result] - return result + return get_example_for_schema(items) if proptype == 'integer': return 0 From 9fa838d3e8d51103ac990ed4808274441149e00f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 10:17:57 -0600 Subject: [PATCH 122/123] Improve type naming --- scripts/templating/matrix_templates/units.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 15c987fd4..a4f682bb2 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -302,7 +302,7 @@ def process_data_type(prop, required=False, enforce_title=True): for i in items: nested = process_data_type(i) tables.extend(nested['tables']) - nested_titles.extend([nested['title']]) + nested_titles.append(nested['title']) prop_title = "[%s]" % (", ".join(nested_titles), ) else: nested = process_data_type(prop["items"]) @@ -523,10 +523,8 @@ class MatrixUnits(Units): items = param.get("items") if items: if isinstance(items, list): - val_type = "[" - for i in items: - val_type += "%s, " % items.get("type") - val_type = val_type[:-2] + "]" + types = ", ".join(i.get("type") for i in items) + val_type = "[%s]" % (types,) else: val_type = "[%s]" % items.get("type") From acf9632afcd0fb3861c203a1fb346ffaaffa98d8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 10:25:24 -0600 Subject: [PATCH 123/123] Enlist the examples --- scripts/templating/matrix_templates/units.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index a4f682bb2..5b39a2b3c 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -406,7 +406,7 @@ def get_example_for_schema(schema): items = schema['items'] if isinstance(items, list): return [get_example_for_schema(i) for i in items] - return get_example_for_schema(items) + return [get_example_for_schema(items)] if proptype == 'integer': return 0