From fa96d8629bcce201b1ef32a8b969e0f0ae60343f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 24 Aug 2018 11:36:04 -0600 Subject: [PATCH 1/6] Prepare the appservice spec for an r0 release This puts the scaffolding in place for an r0 release to happen, such as the changelog and version variables. --- changelogs/application_service.rst | 0 .../newsfragments/.gitignore | 1 + changelogs/application_service/pyproject.toml | 30 +++++++++++++++++++ scripts/gendoc.py | 6 ++++ .../templating/matrix_templates/sections.py | 4 +++ scripts/templating/matrix_templates/units.py | 5 ++-- specification/application_service_api.rst | 12 ++++++-- specification/targets.yaml | 2 +- 8 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 changelogs/application_service.rst create mode 100644 changelogs/application_service/newsfragments/.gitignore create mode 100644 changelogs/application_service/pyproject.toml diff --git a/changelogs/application_service.rst b/changelogs/application_service.rst new file mode 100644 index 00000000..e69de29b diff --git a/changelogs/application_service/newsfragments/.gitignore b/changelogs/application_service/newsfragments/.gitignore new file mode 100644 index 00000000..b722e9e1 --- /dev/null +++ b/changelogs/application_service/newsfragments/.gitignore @@ -0,0 +1 @@ +!.gitignore \ No newline at end of file diff --git a/changelogs/application_service/pyproject.toml b/changelogs/application_service/pyproject.toml new file mode 100644 index 00000000..44d430e8 --- /dev/null +++ b/changelogs/application_service/pyproject.toml @@ -0,0 +1,30 @@ +[tool.towncrier] + filename = "../application_service.rst" + directory = "newsfragments" + issue_format = "`#{issue} `_" + title_format = "{version}" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 16c40af5..8bd8537f 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -518,6 +518,10 @@ if __name__ == '__main__': "--server_release", "-s", action="store", default="unstable", help="The server-server release tag to generate, e.g. r1.2" ) + parser.add_argument( + "--appservice_release", "-a", action="store", default="unstable", + help="The appservice release tag to generate, e.g. r1.2" + ) parser.add_argument( "--list_targets", action="store_true", help="Do not update the specification. Instead print a list of targets.", @@ -542,6 +546,8 @@ if __name__ == '__main__': "%CLIENT_MAJOR_VERSION%": "r0", "%SERVER_RELEASE_LABEL%": args.server_release, "%SERVER_MAJOR_VERSION%": extract_major(args.server_release), + "%APPSERVICE_MAJOR_VERSION%": "unstable", + "%APPSERVICE_RELEASE_LABEL%": args.appservice_release, } exit (main(args.target or ["all"], args.dest, args.nodelete, substitutions)) diff --git a/scripts/templating/matrix_templates/sections.py b/scripts/templating/matrix_templates/sections.py index 1a93c723..b3c7d402 100644 --- a/scripts/templating/matrix_templates/sections.py +++ b/scripts/templating/matrix_templates/sections.py @@ -32,6 +32,10 @@ class MatrixSections(Sections): changelogs = self.units.get("changelogs") return changelogs["client_server"] + def render_application_service_changelog(self): + changelogs = self.units.get("changelogs") + return changelogs["application_service"] + def _render_events(self, filterFn, sortFn): template = self.env.get_template("events.tmpl") examples = self.units.get("event_examples") diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 90a87cd4..9423dfc8 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -754,6 +754,7 @@ class MatrixUnits(Units): def load_apis(self, substitutions): cs_ver = substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable") fed_ver = substitutions.get("%SERVER_RELEASE_LABEL%", "unstable") + as_ver = substitutions.get("%APPSERVICE_RELEASE_LABEL%", "unstable") # we abuse the typetable to return this info to the templates return TypeTable(rows=[ @@ -766,8 +767,8 @@ class MatrixUnits(Units): fed_ver, "Federation between servers", ), TypeTableRow( - "`Application Service API `_", - "unstable", + "`Application Service API `_", + as_ver, "Privileged server plugins", ), TypeTableRow( "`Identity Service API `_", diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 51280341..97a2c04a 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -30,13 +30,21 @@ irrespective of the underlying homeserver implementation. .. contents:: Table of Contents .. sectnum:: -Specification version ---------------------- +Changelog +--------- + + +.. topic:: Version: unstable +{{application_service_changelog}} This version of the specification is generated from `matrix-doc `_ as of Git commit `{{git_version}} `_. +For the full historical changelog, see +https://github.com/matrix-org/matrix-doc/blob/master/changelogs/application_service.rst + + Application Services -------------------- Application services are passive and can only observe events from a given diff --git a/specification/targets.yaml b/specification/targets.yaml index acf4b6ac..42a50e35 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -13,7 +13,7 @@ targets: application_service: files: - application_service_api.rst - version_label: unstable + version_label: "%APPSERVICE_RELEASE_LABEL%" server_server: files: - server_server_api.rst From f01cfa2c7bdd1d43109ac3acfc7b016b661e29db Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 24 Aug 2018 11:39:39 -0600 Subject: [PATCH 2/6] Add security definitions to hs->as endpoints --- .../definitions/security.yaml | 18 ++++++++++++++++++ api/application-service/protocols.yaml | 12 ++++++++++++ api/application-service/query_room.yaml | 4 ++++ api/application-service/query_user.yaml | 4 ++++ api/application-service/transactions.yaml | 4 ++++ 5 files changed, 42 insertions(+) create mode 100644 api/application-service/definitions/security.yaml diff --git a/api/application-service/definitions/security.yaml b/api/application-service/definitions/security.yaml new file mode 100644 index 00000000..bcfc69c0 --- /dev/null +++ b/api/application-service/definitions/security.yaml @@ -0,0 +1,18 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +homeserverAccessToken: + type: apiKey + description: The ``hs_token`` provided by the application service's registration. + name: access_token + in: query diff --git a/api/application-service/protocols.yaml b/api/application-service/protocols.yaml index e6489cc5..d58710a4 100644 --- a/api/application-service/protocols.yaml +++ b/api/application-service/protocols.yaml @@ -24,6 +24,8 @@ consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/_matrix/app/unstable/thirdparty/protocol/{protocol}": get: @@ -33,6 +35,8 @@ paths: with specific information about the various third party networks that an application service supports. operationId: getProtocolMetadata + security: + - homeserverAccessToken: [] parameters: - in: path name: protocol @@ -80,6 +84,8 @@ paths: User ID linked to a user on the third party network, given a set of user parameters. operationId: queryUserByProtocol + security: + - homeserverAccessToken: [] parameters: - in: path name: protocol @@ -131,6 +137,8 @@ paths: description: |- Retrieve a list of Matrix portal rooms that lead to the matched third party location. operationId: queryLocationByProtocol + security: + - homeserverAccessToken: [] parameters: - in: path name: protocol @@ -183,6 +191,8 @@ paths: Retrieve an array of third party network locations from a Matrix room alias. operationId: queryLocationByAlias + security: + - homeserverAccessToken: [] parameters: - in: query name: alias @@ -227,6 +237,8 @@ paths: description: |- Retrieve an array of third party users from a Matrix User ID. operationId: queryUserByID + security: + - homeserverAccessToken: [] parameters: - in: query name: userid diff --git a/api/application-service/query_room.yaml b/api/application-service/query_room.yaml index b885cb86..f388affe 100644 --- a/api/application-service/query_room.yaml +++ b/api/application-service/query_room.yaml @@ -25,6 +25,8 @@ consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/rooms/{roomAlias}": get: @@ -36,6 +38,8 @@ paths: homeserver will send this request when it receives a request to join a room alias within the application service's namespace. operationId: queryRoomByAlias + security: + - homeserverAccessToken: [] parameters: - in: path name: roomAlias diff --git a/api/application-service/query_user.yaml b/api/application-service/query_user.yaml index 0431b5e4..5cb8d900 100644 --- a/api/application-service/query_user.yaml +++ b/api/application-service/query_user.yaml @@ -25,6 +25,8 @@ consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/users/{userId}": get: @@ -36,6 +38,8 @@ paths: send this request when it receives an event for an unknown user ID in the application service's namespace, such as a room invite. operationId: queryUserById + security: + - homeserverAccessToken: [] parameters: - in: path name: userId diff --git a/api/application-service/transactions.yaml b/api/application-service/transactions.yaml index 8735cc8f..53291116 100644 --- a/api/application-service/transactions.yaml +++ b/api/application-service/transactions.yaml @@ -23,6 +23,8 @@ schemes: basePath: "/" produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/transactions/{txnId}": put: @@ -35,6 +37,8 @@ paths: from message events via the presence of a ``state_key``, rather than via the event type. operationId: sendTransaction + security: + - homeserverAccessToken: [] parameters: - in: path name: txnId From 07153c22a941a8de243031b2ef661c4e6e4b6866 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 24 Aug 2018 16:54:53 -0600 Subject: [PATCH 3/6] Misc. cleanup of the appservice spec --- specification/application_service_api.rst | 24 +++++++++++------------ 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 97a2c04a..f3205302 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -48,12 +48,12 @@ https://github.com/matrix-org/matrix-doc/blob/master/changelogs/application_serv Application Services -------------------- Application services are passive and can only observe events from a given -homeserver (HS). They can inject events into rooms they are participating in. +homeserver. They can inject events into rooms they are participating in. They cannot prevent events from being sent, nor can they modify the content of the event being sent. In order to observe events from a homeserver, the homeserver needs to be configured to pass certain types of traffic to the application service. This is achieved by manually configuring the homeserver -with information about the application service (AS). +with information about the application service. Registration ~~~~~~~~~~~~ @@ -187,24 +187,24 @@ events. Each list of events includes a transaction ID, which works as follows: Typical HS ---> AS : Homeserver sends events with transaction ID T. - <--- : AS sends back 200 OK. + <--- : Application Service sends back 200 OK. AS ACK Lost HS ---> AS : Homeserver sends events with transaction ID T. <-/- : AS 200 OK is lost. HS ---> AS : Homeserver retries with the same transaction ID of T. - <--- : AS sends back 200 OK. If the AS had processed these events - already, it can NO-OP this request (and it knows if it is the same - events based on the transaction ID). + <--- : Application Service sends back 200 OK. If the AS had processed these + events already, it can NO-OP this request (and it knows if it is the + same events based on the transaction ID). The events sent to the application service should be linearised, as if they were from the event stream. The homeserver MUST maintain a queue of transactions to -send to the AS. If the application service cannot be reached, the homeserver -SHOULD backoff exponentially until the application service is reachable again. +send to the application service. If the application service cannot be reached, the +homeserver SHOULD backoff exponentially until the application service is reachable again. As application services cannot *modify* the events in any way, these requests can be made without blocking other aspects of the homeserver. Homeservers MUST NOT alter (e.g. add more) events they were going to send within that transaction ID -on retries, as the AS may have already processed the events. +on retries, as the application service may have already processed the events. {{transactions_as_http_api}} @@ -313,7 +313,7 @@ Notes: :: - PUT /_matrix/client/r0/rooms/!somewhere:domain.com/send/m.room.message/txnId?ts=1534535223283 + PUT /_matrix/client/r0/rooms/!somewhere:domain.com/send/m.room.message/txnId?ts=1534535223283 Authorization: Bearer YourApplicationServiceTokenHere Content: The event to send, as per the Client-Server API. @@ -334,7 +334,7 @@ users needs API changes in order to: - Have a 'passwordless' user. This involves bypassing the registration flows entirely. This is achieved by -including the AS token on a ``/register`` request, along with a login type of +including the ``as_token`` on a ``/register`` request, along with a login type of ``m.login.application_service`` to set the desired user ID without a password. :: @@ -374,8 +374,6 @@ additional parameters on the ``/publicRooms`` client-server endpoint. Event fields ~~~~~~~~~~~~ -.. TODO-TravisR: Fix this section to be a general "3rd party networks" section - We recommend that any events that originated from a remote network should include an ``external_url`` field in their content to provide a way for Matrix clients to link into the 'native' client from which the event originated. From 6f80db5ddf1a9851cdc3b8f9e167a3e805db28c1 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 24 Aug 2018 17:21:12 -0600 Subject: [PATCH 4/6] Include the "other versions" section of the changelog --- specification/application_service_api.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index f3205302..4b42707d 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -44,6 +44,13 @@ This version of the specification is generated from For the full historical changelog, see https://github.com/matrix-org/matrix-doc/blob/master/changelogs/application_service.rst +Other versions of this specification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following other versions are also available, in reverse chronological order: + +- `HEAD `_: Includes all changes since the latest versioned release. + Application Services -------------------- From b2092922e85ecf633092498dfa840c912d34c7f8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 29 Aug 2018 21:51:26 -0600 Subject: [PATCH 5/6] Don't define a variable for the appservice major version --- scripts/gendoc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 7b900562..00febdf2 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -550,7 +550,6 @@ if __name__ == '__main__': "%CLIENT_MAJOR_VERSION%": "r0", "%SERVER_RELEASE_LABEL%": args.server_release, "%SERVER_MAJOR_VERSION%": extract_major(args.server_release), - "%APPSERVICE_MAJOR_VERSION%": "unstable", "%APPSERVICE_RELEASE_LABEL%": args.appservice_release, "%PUSH_GATEWAY_RELEASE_LABEL%": args.push_gateway_release, } From 05b017276c81b2a0ce0edb23b6c5c8990b86557e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 30 Aug 2018 10:23:12 -0600 Subject: [PATCH 6/6] Don't accidentally limit application services to one homeserver --- specification/application_service_api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 09ecd4ff..45ee75a9 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -54,8 +54,8 @@ The following other versions are also available, in reverse chronological order: Application Services -------------------- -Application services are passive and can only observe events from a given -homeserver. They can inject events into rooms they are participating in. +Application services are passive and can only observe events from homeserver. +They can inject events into rooms they are participating in. They cannot prevent events from being sent, nor can they modify the content of the event being sent. In order to observe events from a homeserver, the homeserver needs to be configured to pass certain types of traffic to the