From 68c774689aa67231421e6d669a6d240bb94de3ac Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 14 Jan 2015 15:34:07 +0000 Subject: [PATCH 01/25] Add stub AS HTTP API doc --- drafts/as-http-api.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 drafts/as-http-api.rst diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst new file mode 100644 index 00000000..e69de29b From ad7d675d3d269de2be9cd7d1b4f7dd1b7772d3cd Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 14 Jan 2015 16:40:50 +0000 Subject: [PATCH 02/25] Draft Registration and User Query APIs --- drafts/as-http-api.rst | 140 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index e69de29b..41afc213 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -0,0 +1,140 @@ +Application Services HTTP API +============================= + +.. contents:: Table of Contents +.. sectnum:: + +Application Service -> Home Server +---------------------------------- +This contains home server APIs which are used by the application service. + +Registration API ``[Draft]`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +This API registers the application service with its host homeserver to offer its services. + +Inputs: + - Credentials (e.g. some kind of string token) + - Namespace[users] + - Namespace[room aliases] + - URL base to receive inbound comms +Output: + - The credentials the HS will use to query the AS with in return. (e.g. some kind of string token) + - The complete namespace prefix for each namespace requested. +Side effects: + - The HS will start delivering events to the URL base specified if this 200s. +API called when: + - The application service wants to register with a brand new home server. +Notes: + - Namespaces are represented in JSON, with a well-defined conversion to IDs. This prevents + parsing errors and allows the HS to enforce their own namespacing semantics. They look like:: + users: { + irc: ["freenode", "rizon"] + } + which represents 2 namespaces: ``@.irc.freenode.*`` and ``@.irc.rizon.*``. The leaf nodes + must be an array. Intermediate nodes must be JSON objects with the key as the desired string + segment. A more complicated example:: + rooms: { + irc: { + freenode: ["matrix"], + rizon: ["matrixorg"] + } + } + which represents 2 namespaces: ``#.irc.freenode.matrix.*`` and ``#.irc.rizon.matrixorg.*``. + - By specifying namespaces like this, you allow home servers to namespace application services + sensibly, rather than every IRC AS trying to nab all ``@.irc.*`` users. In order for home + servers to do this, they need to tell the application service the actual namespaces allocated + for them. This is returned in the JSON response, with the same structure as the original request, + but with the strings now representing the namespace prefix allocated, e.g:: + users: { + irc: [".applicationservice_146.irc.freenode", ".applicationservice_146.irc.rizon"] + } + The sigil prefix ``@`` is omitted since it is clear from the ``users`` key that these namespace + prefixes are for users. + - This makes the request/response JSON structure deliciously symmetric. +:: + + POST /register + + Request format + { + url: "https://my.application.service.com/matrix/", + as_token: "some_AS_token", + namespaces: { + users: { + irc: ["freenode", "rizon"] + }, + rooms: { + irc: { + freenode: ["matrix"], + rizon: ["matrixorg"] + } + } + } + } + + + Returns: + 200 : Registration accepted. + 400 : Namespaces do not conform to regex + 401 : Credentials need to be supplied. + 403 : AS credentials rejected. + + + 200 OK response format + + { + hs_token: "foobar", + namespaces: { + users: { + irc: [".applicationservice_146.irc.freenode", ".applicationservice_146.irc.rizon"] + }, + rooms: { + irc: { + freenode: [".irc.freenode.matrix"], + rizon: [".applicationservice_146.this.can.be.any.prefix.you.like"] + } + } + } + } + + +Home Server -> Application Service +---------------------------------- +This contains application service APIs which are used by the home server. + +User Query ``[Draft]`` +~~~~~~~~~~~~~~~~~~~~~~ +This API is called by the HS to query the existence of a user on the Application Service's namespace. + +Inputs: + - User ID + - HS Credentials +Output: + - Profile info +Side effects: + - User is created on the HS if this response 200s. +API called when: + - HS receives an event for an unknown user ID in the AS's namespace. +Notes: + - The created user will have their profile info set based on the output. + +:: + + GET /users/$user_id?access_token=$hs_token + + Returns: + 200 : User is recognised. + 404 : User not found. + 401 : Credentials need to be supplied. + 403 : HS credentials rejected. + + + 200 OK response format + + { + profile: { + display_name: "Foo" + avatar_url: "mxc://foo/bar" + } + } + From d240e1f856584c598bfa7ef4c14f19b8b9f72535 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 14 Jan 2015 17:01:33 +0000 Subject: [PATCH 03/25] Add TODO APIs --- drafts/as-http-api.rst | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 41afc213..4915ab6e 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -96,6 +96,10 @@ Notes: } } } + +Unregister API ``[TODO]`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + Home Server -> Application Service @@ -137,4 +141,35 @@ Notes: avatar_url: "mxc://foo/bar" } } + +Room Query ``[TODO]`` +~~~~~~~~~~~~~~~~~~~~~ +This API is called by the HS to query the existence of a room on the Application Service's namespace. + +Pushing ``[TODO]`` +~~~~~~~~~~~~~~~~~~ +This API is called by the HS when the HS wants to push an event (or batch of events) to the AS. + + - Retry semantics + - Ordering + + +Client -> Application Service +----------------------------- +This contains application service APIs which are used by the client. + +Linking ``[TODO]`` +~~~~~~~~~~~~~~~~~~ +Clients may want to link their matrix user ID to their 3PID (e.g. IRC nick). This +API allows the AS to do this, so messages sent from the AS are sent as the client. + +- Probably OAuth2 + +Client-Server v2 API Extensions +------------------------------- + + - Identity assertion (rather than access token inference) + - timestamp massaging (for inserting messages between other messages) + - alias mastery over the ASes namespace + - user ID mastery over the ASes namespace From cdeb0faf853c40dd587507ca534708c98d5aadb5 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 14 Jan 2015 17:17:18 +0000 Subject: [PATCH 04/25] Remove block quotes of shame --- drafts/as-http-api.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 4915ab6e..826a774d 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -150,8 +150,8 @@ Pushing ``[TODO]`` ~~~~~~~~~~~~~~~~~~ This API is called by the HS when the HS wants to push an event (or batch of events) to the AS. - - Retry semantics - - Ordering +- Retry semantics +- Ordering @@ -169,7 +169,7 @@ API allows the AS to do this, so messages sent from the AS are sent as the clien Client-Server v2 API Extensions ------------------------------- - - Identity assertion (rather than access token inference) - - timestamp massaging (for inserting messages between other messages) - - alias mastery over the ASes namespace - - user ID mastery over the ASes namespace +- Identity assertion (rather than access token inference) +- timestamp massaging (for inserting messages between other messages) +- alias mastery over the ASes namespace +- user ID mastery over the ASes namespace From d8cd4a45d4a04cc9de02f7c36bb860bec42720db Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 15 Jan 2015 09:50:38 +0000 Subject: [PATCH 05/25] Modify Register API As per https://github.com/matrix-org/matrix-doc/pull/5#issuecomment-70059200 --- drafts/as-http-api.rst | 65 ++++++++++++------------------------------ 1 file changed, 19 insertions(+), 46 deletions(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 826a774d..d7c749f9 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -10,6 +10,9 @@ This contains home server APIs which are used by the application service. Registration API ``[Draft]`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. NOTE:: + - Do we really have to use regex for this? Can't we do this a nicer way? + This API registers the application service with its host homeserver to offer its services. Inputs: @@ -19,38 +22,18 @@ Inputs: - URL base to receive inbound comms Output: - The credentials the HS will use to query the AS with in return. (e.g. some kind of string token) - - The complete namespace prefix for each namespace requested. Side effects: - The HS will start delivering events to the URL base specified if this 200s. API called when: - The application service wants to register with a brand new home server. Notes: - - Namespaces are represented in JSON, with a well-defined conversion to IDs. This prevents - parsing errors and allows the HS to enforce their own namespacing semantics. They look like:: - users: { - irc: ["freenode", "rizon"] - } - which represents 2 namespaces: ``@.irc.freenode.*`` and ``@.irc.rizon.*``. The leaf nodes - must be an array. Intermediate nodes must be JSON objects with the key as the desired string - segment. A more complicated example:: - rooms: { - irc: { - freenode: ["matrix"], - rizon: ["matrixorg"] - } - } - which represents 2 namespaces: ``#.irc.freenode.matrix.*`` and ``#.irc.rizon.matrixorg.*``. - - By specifying namespaces like this, you allow home servers to namespace application services - sensibly, rather than every IRC AS trying to nab all ``@.irc.*`` users. In order for home - servers to do this, they need to tell the application service the actual namespaces allocated - for them. This is returned in the JSON response, with the same structure as the original request, - but with the strings now representing the namespace prefix allocated, e.g:: - users: { - irc: [".applicationservice_146.irc.freenode", ".applicationservice_146.irc.rizon"] - } + - Namespaces are represented by POSIX extended regular expressions in JSON. + They look like:: + users: [ + "irc\.freenode\.net/.*", + ] The sigil prefix ``@`` is omitted since it is clear from the ``users`` key that these namespace prefixes are for users. - - This makes the request/response JSON structure deliciously symmetric. :: POST /register @@ -60,15 +43,12 @@ Notes: url: "https://my.application.service.com/matrix/", as_token: "some_AS_token", namespaces: { - users: { - irc: ["freenode", "rizon"] - }, - rooms: { - irc: { - freenode: ["matrix"], - rizon: ["matrixorg"] - } - } + users: [ + "irc\.freenode\.net/.*" + ], + rooms: [ + "irc\.freenode\.net/.*" + ] } } @@ -83,18 +63,7 @@ Notes: 200 OK response format { - hs_token: "foobar", - namespaces: { - users: { - irc: [".applicationservice_146.irc.freenode", ".applicationservice_146.irc.rizon"] - }, - rooms: { - irc: { - freenode: [".irc.freenode.matrix"], - rizon: [".applicationservice_146.this.can.be.any.prefix.you.like"] - } - } - } + hs_token: "foobar" } Unregister API ``[TODO]`` @@ -108,6 +77,10 @@ This contains application service APIs which are used by the home server. User Query ``[Draft]`` ~~~~~~~~~~~~~~~~~~~~~~ +.. NOTE:: + - This API may be called a lot by the HS (e.g. incoming events for unknown user IDs, profile + requests, etc. Is this okay? + This API is called by the HS to query the existence of a user on the Application Service's namespace. Inputs: From 9401d9a62e8a792f646d5f89a55d1d90d75e8f07 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 15 Jan 2015 13:48:50 +0000 Subject: [PATCH 06/25] Flesh out the required CS extensions --- drafts/as-http-api.rst | 72 +++++++++++++++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 12 deletions(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index d7c749f9..a66628c3 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -46,7 +46,7 @@ Notes: users: [ "irc\.freenode\.net/.*" ], - rooms: [ + aliases: [ "irc\.freenode\.net/.*" ] } @@ -63,12 +63,30 @@ Notes: 200 OK response format { - hs_token: "foobar" + hs_token: "string" } -Unregister API ``[TODO]`` +Unregister API ``[Draft]`` ~~~~~~~~~~~~~~~~~~~~~~~~~ +This API unregisters a previously registered AS from the home server. +Inputs: + - AS token +Output: + - None. +Side effects: + - The HS will stop delivering events to the URL base specified for this AS if this 200s. +API called when: + - The application service wants to stop receiving all events from the HS. + +:: + + POST /unregister + + Request format + { + as_token: "string" + } Home Server -> Application Service @@ -77,9 +95,6 @@ This contains application service APIs which are used by the home server. User Query ``[Draft]`` ~~~~~~~~~~~~~~~~~~~~~~ -.. NOTE:: - - This API may be called a lot by the HS (e.g. incoming events for unknown user IDs, profile - requests, etc. Is this okay? This API is called by the HS to query the existence of a user on the Application Service's namespace. @@ -110,8 +125,8 @@ Notes: { profile: { - display_name: "Foo" - avatar_url: "mxc://foo/bar" + display_name: "string" + avatar_url: "string(uri)" } } @@ -142,7 +157,40 @@ API allows the AS to do this, so messages sent from the AS are sent as the clien Client-Server v2 API Extensions ------------------------------- -- Identity assertion (rather than access token inference) -- timestamp massaging (for inserting messages between other messages) -- alias mastery over the ASes namespace -- user ID mastery over the ASes namespace +Identity assertion +~~~~~~~~~~~~~~~~~~ +The client-server API infers the user ID from the ``access_token`` provided in every +request. It would be an annoying amount of book-keeping to maintain tokens for every +virtual user. It would be preferable if the application service could use the CS API +with its own ``as_token`` instead, and specify the virtual user they wish to be +acting on behalf of. For real users, this would require additional permissions (see +"C-AS Linking"). + +Inputs: + - Application service token (``as_token``) + Either: + - User ID in the AS namespace to act as. + Or: + - OAuth2 token of real user (which may end up being an access token) +Notes: + - This will apply on all aspects of the CS API, except for Account Management. + + +Timestamp massaging +~~~~~~~~~~~~~~~~~~~ +The application service may want to inject events at a certain time (reflecting +the time on the network they are tracking e.g. irc, xmpp). Application services +need to be able to adjust the ``origin_server_ts`` value to do this. + +Inputs: + - Application service token (``as_token``) + - Desired timestamp +Notes: + - This will only apply when sending events. + +Server admin style permissions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The home server needs to give the application service *full control* over its +namespace, both for users and for room aliases. This means that the AS should +be able to create/edit/delete any room alias in its namespace, as well as +create/delete any user in its namespace. From d4ea43d2b9126f3004cf948c470010528cd7a655 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 15 Jan 2015 13:56:03 +0000 Subject: [PATCH 07/25] Outline what the CS extensions look like in HTTP --- drafts/as-http-api.rst | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index a66628c3..666d996c 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -175,6 +175,19 @@ Inputs: Notes: - This will apply on all aspects of the CS API, except for Account Management. +:: + + /path?as_token=$token&user_id=$userid + + Query Parameters: + as_token: The application service token + user_id: The desired user ID to act as. + + /path?as_token=$token&user_token=$token + + Query Parameters: + as_token: The application service token + user_token: The token granted to the AS by the real user Timestamp massaging ~~~~~~~~~~~~~~~~~~~ @@ -187,10 +200,19 @@ Inputs: - Desired timestamp Notes: - This will only apply when sending events. + +:: + + /path?as_token=$token&ts=$timestamp + + Query Parameters added to the send event APIs only: + as_token: The application service token + ts: The desired timestamp Server admin style permissions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The home server needs to give the application service *full control* over its namespace, both for users and for room aliases. This means that the AS should be able to create/edit/delete any room alias in its namespace, as well as -create/delete any user in its namespace. +create/delete any user in its namespace. This does not require any additional +public APIs. From 7d56113b4fa1ff3f5d13fa76a3acd005b9ae66da Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 15 Jan 2015 14:27:20 +0000 Subject: [PATCH 08/25] Clarification and tweaks Room ID namespace clarifications and s/as_token/access_token/g for client-server extensions. --- drafts/as-http-api.rst | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 666d996c..5a1ca364 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -33,7 +33,7 @@ Notes: "irc\.freenode\.net/.*", ] The sigil prefix ``@`` is omitted since it is clear from the ``users`` key that these namespace - prefixes are for users. + prefixes are for users. Likewise, ``#`` for ``aliases`` and ``!`` for ``rooms``. :: POST /register @@ -48,6 +48,9 @@ Notes: ], aliases: [ "irc\.freenode\.net/.*" + ], + rooms: [ + "irc\.freenode\.net/.*" ] } } @@ -167,23 +170,26 @@ acting on behalf of. For real users, this would require additional permissions ( "C-AS Linking"). Inputs: - - Application service token (``as_token``) + - Application service token (``access_token``) Either: - User ID in the AS namespace to act as. Or: - OAuth2 token of real user (which may end up being an access token) Notes: - This will apply on all aspects of the CS API, except for Account Management. + - The ``as_token`` is inserted into ``access_token`` which is usually where the client + token is. This is done on purpose to allow application services to reuse client + SDKs. :: - /path?as_token=$token&user_id=$userid + /path?access_token=$token&user_id=$userid Query Parameters: as_token: The application service token user_id: The desired user ID to act as. - /path?as_token=$token&user_token=$token + /path?access_token=$token&user_token=$token Query Parameters: as_token: The application service token From e0e9beebc0923f0fa07c4719f0cf00bfc5e78a11 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 15 Jan 2015 16:48:39 +0000 Subject: [PATCH 09/25] Add draft Linking API Also re-add sigils to regex. --- drafts/as-http-api.rst | 63 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 5a1ca364..8995ffd8 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -30,10 +30,8 @@ Notes: - Namespaces are represented by POSIX extended regular expressions in JSON. They look like:: users: [ - "irc\.freenode\.net/.*", + "@irc\.freenode\.net/.*", ] - The sigil prefix ``@`` is omitted since it is clear from the ``users`` key that these namespace - prefixes are for users. Likewise, ``#`` for ``aliases`` and ``!`` for ``rooms``. :: POST /register @@ -44,13 +42,13 @@ Notes: as_token: "some_AS_token", namespaces: { users: [ - "irc\.freenode\.net/.*" + "@irc\.freenode\.net/.*" ], aliases: [ - "irc\.freenode\.net/.*" + "#irc\.freenode\.net/.*" ], rooms: [ - "irc\.freenode\.net/.*" + "!irc\.freenode\.net/.*" ] } } @@ -146,16 +144,59 @@ This API is called by the HS when the HS wants to push an event (or batch of eve -Client -> Application Service ------------------------------ -This contains application service APIs which are used by the client. +Client -> Server -> Application Service +--------------------------------------- +This contains home server APIs which are used by the client, to communicate with +the application service. -Linking ``[TODO]`` +Linking ``[Draft]`` ~~~~~~~~~~~~~~~~~~ +.. NOTE:: + - How does the application service know that the matrix user really is the virtual + user they claim to be? If we force an IS lookup, then this would resolve on its + own as anyone who wants to talk to the virtual user will do a lookup before trying + the application service... + Clients may want to link their matrix user ID to their 3PID (e.g. IRC nick). This API allows the AS to do this, so messages sent from the AS are sent as the client. -- Probably OAuth2 +This is not achieved using OAuth2 or similar because the trust relationships are +different. The application service already has a huge amount of trust from the +home server, unlike a random third-party web app. As a result, the data flow is +different because the third-party (the application service) is trusted by the +authorisation entity (the home server). Furthermore, it is desirable to not have +the clients communicate directly with the application service in order to +decrease the complexity of AS design. + +To grant permission for an application service to masquerade as a user: + +Inputs: + - Credentials of user (e.g. ``access_token``) + - The user ID within an application service's namespace to claim. + - Restrictions if any (e.g. only for rooms X,Y,Z. Only for 10 hours. etc) +Output: + - None. +Side effects: + - The home server will generate an ``access_token`` and push it to the + application service along with both user IDs if this response 200s. +Notes: + - Repeated calls to this API should invalidate any existing token for this + user ID / application service combo and provision a new token which is then + pushed. + - The generated access token MUST honour the restrictions laid out by the + client. + +To revoke permission for an application service to masquerade as a user: + +Inputs: + - Credentials of user (e.g. ``access_token``) + - The user ID within an application service's namespace to revoke. If blank, + revokes all user IDs linked to this matrix user ID. +Output: + - None. +Side effects: + - The home server invalidate all access tokens for this user ID / AS combo + and push this invalidation to the application service if this response 200s. Client-Server v2 API Extensions ------------------------------- From 0313397e42591c89f47b65ee35eb40258481f750 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 15 Jan 2015 16:51:20 +0000 Subject: [PATCH 10/25] Clarification on IDs Specifically using the term "virtual user ID" in more places to mean the namespace of AS user IDs. --- drafts/as-http-api.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 8995ffd8..2fb9aed2 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -157,8 +157,9 @@ Linking ``[Draft]`` own as anyone who wants to talk to the virtual user will do a lookup before trying the application service... -Clients may want to link their matrix user ID to their 3PID (e.g. IRC nick). This -API allows the AS to do this, so messages sent from the AS are sent as the client. +Clients may want to link their matrix user ID to their virtual user ID. This +API allows the AS to do this, so messages sent from the AS are sent as the client's +user ID, instead of the virtual user ID. This is not achieved using OAuth2 or similar because the trust relationships are different. The application service already has a huge amount of trust from the @@ -191,7 +192,7 @@ To revoke permission for an application service to masquerade as a user: Inputs: - Credentials of user (e.g. ``access_token``) - The user ID within an application service's namespace to revoke. If blank, - revokes all user IDs linked to this matrix user ID. + revokes all virtual user IDs linked to this matrix user ID. Output: - None. Side effects: From 1b3e27a05a8666e02e031972a7bf0336195d5599 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 15 Jan 2015 17:26:36 +0000 Subject: [PATCH 11/25] Add Linking HTTP API Also add note on masquerading problem. --- drafts/as-http-api.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 2fb9aed2..727a5ff9 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -156,6 +156,8 @@ Linking ``[Draft]`` user they claim to be? If we force an IS lookup, then this would resolve on its own as anyone who wants to talk to the virtual user will do a lookup before trying the application service... + - In other words, what is preventing ``@bob:matrix.org`` masquerading as + ``@.irc.freenode.alice:matrix.org``? Clients may want to link their matrix user ID to their virtual user ID. This API allows the AS to do this, so messages sent from the AS are sent as the client's @@ -186,6 +188,21 @@ Notes: pushed. - The generated access token MUST honour the restrictions laid out by the client. + +:: + + PUT /appservices/$virtual_user_id?access_token=$token + + Request format + { + restrictions: { + expires_in: 3600, + rooms: [ + "!fl3rwfehw:matrix.org", + "!fwet2yugs:matrix.org" + ] + } + } To revoke permission for an application service to masquerade as a user: @@ -198,6 +215,11 @@ Output: Side effects: - The home server invalidate all access tokens for this user ID / AS combo and push this invalidation to the application service if this response 200s. + +:: + + DELETE /appservices/$virtual_user_id?access_token=$token + Client-Server v2 API Extensions ------------------------------- From f33b722e4b006f8ffaee89f38648df0c2b9b2dde Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 16 Jan 2015 08:28:55 +0000 Subject: [PATCH 12/25] More clarifications s/as_token/access_token/ for CS extensions. Clarify when room queries are made. --- drafts/as-http-api.rst | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 727a5ff9..59d5a3e4 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -131,9 +131,11 @@ Notes: } } -Room Query ``[TODO]`` -~~~~~~~~~~~~~~~~~~~~~ -This API is called by the HS to query the existence of a room on the Application Service's namespace. +Room Alias Query ``[TODO]`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +This API is called by the HS to query the existence of a room alias on the Application +Service's namespace. + Pushing ``[TODO]`` ~~~~~~~~~~~~~~~~~~ @@ -250,13 +252,13 @@ Notes: /path?access_token=$token&user_id=$userid Query Parameters: - as_token: The application service token + access_token: The application service token user_id: The desired user ID to act as. /path?access_token=$token&user_token=$token Query Parameters: - as_token: The application service token + access_token: The application service token user_token: The token granted to the AS by the real user Timestamp massaging @@ -273,10 +275,10 @@ Notes: :: - /path?as_token=$token&ts=$timestamp + /path?access_token=$token&ts=$timestamp Query Parameters added to the send event APIs only: - as_token: The application service token + access_token: The application service token ts: The desired timestamp Server admin style permissions From 1d45dc08ca07cecb7f9c618591753c4f6540fe06 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 16 Jan 2015 10:56:42 +0000 Subject: [PATCH 13/25] Add room query API. Remove linking API. Also add section on ID conventions which need thinking about. --- drafts/as-http-api.rst | 171 +++++++++++++++++++---------------------- 1 file changed, 81 insertions(+), 90 deletions(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 59d5a3e4..fd0b302e 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -13,7 +13,8 @@ Registration API ``[Draft]`` .. NOTE:: - Do we really have to use regex for this? Can't we do this a nicer way? -This API registers the application service with its host homeserver to offer its services. +This API registers the application service with its host homeserver to offer its +services. Inputs: - Credentials (e.g. some kind of string token) @@ -21,7 +22,8 @@ Inputs: - Namespace[room aliases] - URL base to receive inbound comms Output: - - The credentials the HS will use to query the AS with in return. (e.g. some kind of string token) + - The credentials the HS will use to query the AS with in return. (e.g. some + kind of string token) Side effects: - The HS will start delivering events to the URL base specified if this 200s. API called when: @@ -76,7 +78,8 @@ Inputs: Output: - None. Side effects: - - The HS will stop delivering events to the URL base specified for this AS if this 200s. + - The HS will stop delivering events to the URL base specified for this AS if + this 200s. API called when: - The application service wants to stop receiving all events from the HS. @@ -97,7 +100,8 @@ This contains application service APIs which are used by the home server. User Query ``[Draft]`` ~~~~~~~~~~~~~~~~~~~~~~ -This API is called by the HS to query the existence of a user on the Application Service's namespace. +This API is called by the HS to query the existence of a user on the Application +Service's namespace. Inputs: - User ID @@ -131,96 +135,71 @@ Notes: } } -Room Alias Query ``[TODO]`` +Room Alias Query ``[Draft]`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This API is called by the HS to query the existence of a room alias on the Application -Service's namespace. - - -Pushing ``[TODO]`` -~~~~~~~~~~~~~~~~~~ -This API is called by the HS when the HS wants to push an event (or batch of events) to the AS. - -- Retry semantics -- Ordering - - - -Client -> Server -> Application Service ---------------------------------------- -This contains home server APIs which are used by the client, to communicate with -the application service. - -Linking ``[Draft]`` -~~~~~~~~~~~~~~~~~~ -.. NOTE:: - - How does the application service know that the matrix user really is the virtual - user they claim to be? If we force an IS lookup, then this would resolve on its - own as anyone who wants to talk to the virtual user will do a lookup before trying - the application service... - - In other words, what is preventing ``@bob:matrix.org`` masquerading as - ``@.irc.freenode.alice:matrix.org``? - -Clients may want to link their matrix user ID to their virtual user ID. This -API allows the AS to do this, so messages sent from the AS are sent as the client's -user ID, instead of the virtual user ID. - -This is not achieved using OAuth2 or similar because the trust relationships are -different. The application service already has a huge amount of trust from the -home server, unlike a random third-party web app. As a result, the data flow is -different because the third-party (the application service) is trusted by the -authorisation entity (the home server). Furthermore, it is desirable to not have -the clients communicate directly with the application service in order to -decrease the complexity of AS design. - -To grant permission for an application service to masquerade as a user: +This API is called by the HS to query the existence of a room alias on the +Application Service's namespace. Inputs: - - Credentials of user (e.g. ``access_token``) - - The user ID within an application service's namespace to claim. - - Restrictions if any (e.g. only for rooms X,Y,Z. Only for 10 hours. etc) + - Room alias + - HS Credentials Output: - - None. + - The current state events for the room if any. + - The message events for the room if any. Side effects: - - The home server will generate an ``access_token`` and push it to the - application service along with both user IDs if this response 200s. + - A new room will be created with the alias input if this response 200s. +API called when: + - HS receives an event to join a room alias in the AS's namespace. Notes: - - Repeated calls to this API should invalidate any existing token for this - user ID / application service combo and provision a new token which is then - pushed. - - The generated access token MUST honour the restrictions laid out by the - client. - + - This can be thought of as an ``initialSync`` but for a 3P networked room, + which is lazily loaded when a matrix user tries to join the room. + :: - PUT /appservices/$virtual_user_id?access_token=$token + GET /rooms/$room_alias?access_token=$hs_token - Request format - { - restrictions: { - expires_in: 3600, - rooms: [ - "!fl3rwfehw:matrix.org", - "!fwet2yugs:matrix.org" + Returns: + 200 : Room is recognised. + 404 : Room not found. + 401 : Credentials need to be supplied. + 403 : HS credentials rejected. + + + 200 OK response format + + { + events: [ + { + content: { + ... + }, + user_id: "string", + origin_server_ts: integer, // massaged timestamps + type: "string" + }, + ... + ], + state: [ + { + content: { + ... + }, + user_id: "string(virtual user id)", + origin_server_ts: integer, + state_key: "string", + type: "string" // e.g. m.room.name + }, + ... ] } - } -To revoke permission for an application service to masquerade as a user: - -Inputs: - - Credentials of user (e.g. ``access_token``) - - The user ID within an application service's namespace to revoke. If blank, - revokes all virtual user IDs linked to this matrix user ID. -Output: - - None. -Side effects: - - The home server invalidate all access tokens for this user ID / AS combo - and push this invalidation to the application service if this response 200s. - -:: +Pushing ``[TODO]`` +~~~~~~~~~~~~~~~~~~ +This API is called by the HS when the HS wants to push an event (or batch of +events) to the AS. - DELETE /appservices/$virtual_user_id?access_token=$token +- Retry semantics +- Ordering Client-Server v2 API Extensions @@ -228,12 +207,12 @@ Client-Server v2 API Extensions Identity assertion ~~~~~~~~~~~~~~~~~~ -The client-server API infers the user ID from the ``access_token`` provided in every -request. It would be an annoying amount of book-keeping to maintain tokens for every -virtual user. It would be preferable if the application service could use the CS API -with its own ``as_token`` instead, and specify the virtual user they wish to be -acting on behalf of. For real users, this would require additional permissions (see -"C-AS Linking"). +The client-server API infers the user ID from the ``access_token`` provided in +every request. It would be an annoying amount of book-keeping to maintain tokens +for every virtual user. It would be preferable if the application service could +use the CS API with its own ``as_token`` instead, and specify the virtual user +they wish to be acting on behalf of. For real users, this would require +additional permissions (see "C-AS Linking"). Inputs: - Application service token (``access_token``) @@ -243,9 +222,9 @@ Inputs: - OAuth2 token of real user (which may end up being an access token) Notes: - This will apply on all aspects of the CS API, except for Account Management. - - The ``as_token`` is inserted into ``access_token`` which is usually where the client - token is. This is done on purpose to allow application services to reuse client - SDKs. + - The ``as_token`` is inserted into ``access_token`` which is usually where the + client token is. This is done on purpose to allow application services to + reuse client SDKs. :: @@ -288,3 +267,15 @@ namespace, both for users and for room aliases. This means that the AS should be able to create/edit/delete any room alias in its namespace, as well as create/delete any user in its namespace. This does not require any additional public APIs. + + +ID conventions +-------------- +This concerns the well-defined conventions for mapping 3P network IDs to matrix +IDs, which we expect clients to be able to do by themselves. + +- What do user IDs look like? Butchered URIs? Can all 3P network IDs be + reasonably expressed as URIs? (e.g. tel, email, irc, xmpp, ...) +- What do room aliases look like? Some cases are clear (e.g. IRC) but others + are a lot more fiddly (e.g. email? You don't want to share a room with + everyone who has ever sent an email to ``bob@gmail.com``)... \ No newline at end of file From bd6fc4b5bfbf1b7e7c9867f1c7f1b48f4b3f4821 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 16 Jan 2015 14:29:53 +0000 Subject: [PATCH 14/25] Add draft Pushing API. Add Examples section. Work through an IRC example; raise issues in the notes section. Add retry notes on HS>AS API calls. --- drafts/as-http-api.rst | 227 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 216 insertions(+), 11 deletions(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index fd0b302e..dd2dbf80 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -31,9 +31,11 @@ API called when: Notes: - Namespaces are represented by POSIX extended regular expressions in JSON. They look like:: + users: [ "@irc\.freenode\.net/.*", ] + :: POST /register @@ -70,7 +72,7 @@ Notes: } Unregister API ``[Draft]`` -~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~ This API unregisters a previously registered AS from the home server. Inputs: @@ -111,10 +113,16 @@ Output: Side effects: - User is created on the HS if this response 200s. API called when: - - HS receives an event for an unknown user ID in the AS's namespace. + - HS receives an event for an unknown user ID in the AS's namespace, e.g. an + invite event to a room. Notes: - The created user will have their profile info set based on the output. - +Retry notes: + - The home server cannot respond to the client's request until the response to + this API is obtained from the AS. + - Recommended that home servers try a few times then time out, returning a + 408 Request Timeout to the client. + :: GET /users/$user_id?access_token=$hs_token @@ -136,7 +144,7 @@ Notes: } Room Alias Query ``[Draft]`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This API is called by the HS to query the existence of a room alias on the Application Service's namespace. @@ -153,6 +161,11 @@ API called when: Notes: - This can be thought of as an ``initialSync`` but for a 3P networked room, which is lazily loaded when a matrix user tries to join the room. +Retry notes: + - The home server cannot respond to the client's request until the response to + this API is obtained from the AS. + - Recommended that home servers try a few times then time out, returning a + 408 Request Timeout to the client. :: @@ -193,14 +206,61 @@ Notes: ] } -Pushing ``[TODO]`` -~~~~~~~~~~~~~~~~~~ +Pushing ``[Draft]`` +~~~~~~~~~~~~~~~~~~~ This API is called by the HS when the HS wants to push an event (or batch of events) to the AS. -- Retry semantics -- Ordering +Inputs: + - HS Credentials + - Event(s) to give to the AS + - HS-generated transaction ID +Output: + - None. + +Data flows: + +:: + + Typical + HS ---> AS : Home server sends events with transaction ID T. + <--- : AS sends back 200 OK. + + AS ACK Lost + HS ---> AS : Home server sends events with transaction ID T. + <-/- : AS 200 OK is lost. + HS ---> AS : Home server retries with the same transaction ID of T. + <--- : AS sends back 200 OK. If the AS had processed these events + already, it can NO-OP this request (and it knows if it is the same + events based on the transacton ID). + + +Retry notes: + - If the HS fails to pass on the events to the AS, it must retry the request. + - Since ASes by definition cannot alter the traffic being passed to it (unlike + say, a Policy Server), these requests can be done in parallel to general HS + processing; the HS doesn't need to block whilst doing this. + - Home servers should use exponential backoff as their retry algorithm. + - Home servers MUST NOT alter (e.g. add more) events they were going to + send within that transaction ID on retries, as the AS may have already + processed the events. + +Ordering notes: + - The events sent to the AS should be linearised, as they are from the event + stream. + - The home server will need to maintain a queue of transactions to send to + the AS. +:: + + PUT /transactions/$transaction_id?access_token=$hs_token + + Request format + { + events: [ + ... + ] + } Client-Server v2 API Extensions ------------------------------- @@ -216,6 +276,7 @@ additional permissions (see "C-AS Linking"). Inputs: - Application service token (``access_token``) + Either: - User ID in the AS namespace to act as. Or: @@ -269,8 +330,8 @@ create/delete any user in its namespace. This does not require any additional public APIs. -ID conventions --------------- +ID conventions ``[TODO]`` +------------------------- This concerns the well-defined conventions for mapping 3P network IDs to matrix IDs, which we expect clients to be able to do by themselves. @@ -278,4 +339,148 @@ IDs, which we expect clients to be able to do by themselves. reasonably expressed as URIs? (e.g. tel, email, irc, xmpp, ...) - What do room aliases look like? Some cases are clear (e.g. IRC) but others are a lot more fiddly (e.g. email? You don't want to share a room with - everyone who has ever sent an email to ``bob@gmail.com``)... \ No newline at end of file + everyone who has ever sent an email to ``bob@gmail.com``)... + +Examples +-------- +.. NOTE:: + - User/Alias namespaces are subject to change depending on ID conventions. + - Should home servers by default generate fixed room IDs which match the room + alias? Otherwise, you need to tell the AS that room alias X matches room ID + Y so when the home server pushes events with room ID Y the AS knows which + room that is. + +IRC +~~~ +Pre-conditions: + - Server admin stores the AS token "T_a" on the home server. + - Home server has a token "T_h". + - Home server has the domain "hsdomain.com" + +1. Application service registration +:: + + AS -> HS: Registers itself with the home server + POST /register + { + url: "https://someapp.com/matrix", + as_token: "T_a", + namespaces: { + users: [ + "@irc\.freenode\.net/.*" + ], + aliases: [ + "#irc\.freenode\.net/.*" + ], + rooms: [ + "!irc\.freenode\.net/.*" + ] + } + } + + Returns 200 OK: + { + hs_token: "T_h" + } + +2. IRC user "Bob" says "hello?" on "#matrix" at timestamp 1421416883133: +:: + + - AS stores message as potential scrollback. + - Nothing happens as no Matrix users are in the room. + +3. Matrix user "@alice:hsdomain.com" wants to join "#matrix": +:: + + User -> HS: Request to join "#irc.freenode.net/#matrix:hsdomain.com" + + HS -> AS: Room Query "#irc.freenode.net/#matrix:hsdomain.com" + GET /rooms/%23irc.freenode.net%2F%23matrix%3Ahsdomain.com?access_token=T_h + Returns 200 OK: + { + events: [ + { + content: { + body: "hello?", + msgtype: "m.text" + } + origin_server_ts: 1421416883133, + user_id: "@irc.freenode.net/Bob:hsdomain.com" + type: "m.room.message" + } + ], + state: [ + { + content: { + name: "#matrix" + } + origin_server_ts: 1421416883133, // default this to the first msg? + user_id: "@irc.freenode.net/Bob:hsdomain.com", // see above + state_key: "", + type: "m.room.name" + } + ] + } + + - HS provisions new room with *FIXED* room ID (see notes section) + "!irc.freenode.net/#matrix:hsdomain.com" with normal state events + (e.g. m.room.create). join_rules can be overridden by the AS if supplied in + "state". + - HS injects messages into room. Finds unknown user ID + "@irc.freenode.net/Bob:hsdomain.com" in AS namespace, so queries AS. + + HS -> AS: User Query "@irc.freenode.net/Bob:hsdomain.com" + GET /users/%40irc.freenode.net%2FBob%3Ahsdomain.com?access_token=T_h + Returns 200 OK: + { + profile: { + display_name: "Bob" + } + } + + - HS provisions new user with display name "Bob". + - HS sends room information back to client. + +4. @alice:hsdomain.com says "hi!" in this room: +:: + + User -> HS: Send message "hi!" in room !irc.freenode.net/#matrix:hsdomain.com + + - HS sends message. + - HS sees the room ID is in the AS namespace and pushes it to the AS. + + HS -> AS: Push event + PUT /transactions/1?access_token=T_h + { + events: [ + { + content: { + body: "hi!", + msgtype: "m.text" + }, + origin_server_ts: , + user_id: "@alice:hsdomain.com", + room_id: "!irc.freenode.net/#matrix:hsdomain.com", + type: "m.room.message" + } + ] + } + + - AS passes this through to IRC. + + +5. IRC user "Bob" says "what's up?" on "#matrix" at timestamp 1421418084816: +:: + + IRC -> AS: "what's up?" + AS -> HS: Send message via CS API extension + PUT /rooms/%21irc.freenode.net%2F%23matrix%3Ahsdomain.com/send/m.room.message + ?access_token=T_a + &user_id=%40irc.freenode.net%2FBob%3Ahsdomain.com + &ts=1421418084816 + { + body: "what's up?" + msgtype: "m.text" + } + + - HS modifies the user_id and origin_server_ts on the event and sends it. \ No newline at end of file From 828d6992e563acf78d4a595f446ba1b518844ea4 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 16 Jan 2015 14:35:20 +0000 Subject: [PATCH 15/25] Small thoughts about the IRC example. --- drafts/as-http-api.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index dd2dbf80..1bd247f8 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -348,7 +348,10 @@ Examples - Should home servers by default generate fixed room IDs which match the room alias? Otherwise, you need to tell the AS that room alias X matches room ID Y so when the home server pushes events with room ID Y the AS knows which - room that is. + room that is. Alternatively, do we expect ASes to query the HS via the room + alias APIs just like clients do? This doesn't get us the reverse mapping + though which is what we really want when events are pushed to the AS. Maybe + a special room_id_to_alias event poke is pushed to the AS? IRC ~~~ From 603361bfd68674dae57624122eb4adb5ee6bd689 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 19 Jan 2015 10:20:39 +0000 Subject: [PATCH 16/25] Update user and room query APIs Expect the AS to create these entities, rather than the HS. --- drafts/as-http-api.rst | 126 ++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 85 deletions(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 1bd247f8..931c8a07 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -109,14 +109,21 @@ Inputs: - User ID - HS Credentials Output: - - Profile info + - Whether the user exists. Side effects: - - User is created on the HS if this response 200s. + - User is created on the HS by the AS via CS APIs during the processing of this request. API called when: - HS receives an event for an unknown user ID in the AS's namespace, e.g. an invite event to a room. Notes: - - The created user will have their profile info set based on the output. + - When the AS receives this request, if the user exists, it must create the user via + the CS API. + - It can also set arbitrary information about the user (e.g. display name, join rooms, etc) + using the CS API. + - When this setup is complete, the AS should respond to the HS request. This means the AS + blocks the HS until the user is created. + - This is deemed more flexible than alternative methods (e.g. returning a JSON blob with the + user's display name and get the HS to provision the user). Retry notes: - The home server cannot respond to the client's request until the response to this API is obtained from the AS. @@ -136,12 +143,7 @@ Retry notes: 200 OK response format - { - profile: { - display_name: "string" - avatar_url: "string(uri)" - } - } + {} Room Alias Query ``[Draft]`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -155,12 +157,20 @@ Output: - The current state events for the room if any. - The message events for the room if any. Side effects: - - A new room will be created with the alias input if this response 200s. + - Room is created on the HS by the AS via CS APIs during the processing of + this request. API called when: - HS receives an event to join a room alias in the AS's namespace. Notes: - - This can be thought of as an ``initialSync`` but for a 3P networked room, - which is lazily loaded when a matrix user tries to join the room. + - When the AS receives this request, if the room exists, it must create the room via + the CS API. + - It can also set arbitrary information about the room (e.g. name, topic, etc) + using the CS API. + - When this setup is complete, the AS should respond to the HS request. This means the AS + blocks the HS until the room is created and configured. + - This is deemed more flexible than alternative methods (e.g. returning an initial sync + style JSON blob and get the HS to provision the room). It also means that the AS knows + the room ID -> alias mapping. Retry notes: - The home server cannot respond to the client's request until the response to this API is obtained from the AS. @@ -180,31 +190,7 @@ Retry notes: 200 OK response format - { - events: [ - { - content: { - ... - }, - user_id: "string", - origin_server_ts: integer, // massaged timestamps - type: "string" - }, - ... - ], - state: [ - { - content: { - ... - }, - user_id: "string(virtual user id)", - origin_server_ts: integer, - state_key: "string", - type: "string" // e.g. m.room.name - }, - ... - ] - } + {} Pushing ``[Draft]`` ~~~~~~~~~~~~~~~~~~~ @@ -345,13 +331,6 @@ Examples -------- .. NOTE:: - User/Alias namespaces are subject to change depending on ID conventions. - - Should home servers by default generate fixed room IDs which match the room - alias? Otherwise, you need to tell the AS that room alias X matches room ID - Y so when the home server pushes events with room ID Y the AS knows which - room that is. Alternatively, do we expect ASes to query the HS via the room - alias APIs just like clients do? This doesn't get us the reverse mapping - though which is what we really want when events are pushed to the AS. Maybe - a special room_id_to_alias event poke is pushed to the AS? IRC ~~~ @@ -399,49 +378,26 @@ Pre-conditions: HS -> AS: Room Query "#irc.freenode.net/#matrix:hsdomain.com" GET /rooms/%23irc.freenode.net%2F%23matrix%3Ahsdomain.com?access_token=T_h - Returns 200 OK: - { - events: [ - { - content: { - body: "hello?", - msgtype: "m.text" - } - origin_server_ts: 1421416883133, - user_id: "@irc.freenode.net/Bob:hsdomain.com" - type: "m.room.message" - } - ], - state: [ + [Starts blocking] + AS -> HS: Creates room. Gets room ID "!aasaasasa:matrix.org". + AS -> HS: Sets room name to "#matrix". + AS -> HS: Sends message as ""@irc.freenode.net/Bob:hsdomain.com" + PUT /rooms/%21irc.freenode.net%2F%23matrix%3Ahsdomain.com/send/m.room.message + ?access_token=T_a + &user_id=%40irc.freenode.net%2FBob%3Ahsdomain.com + &ts=1421416883133 { - content: { - name: "#matrix" - } - origin_server_ts: 1421416883133, // default this to the first msg? - user_id: "@irc.freenode.net/Bob:hsdomain.com", // see above - state_key: "", - type: "m.room.name" + body: "hello?" + msgtype: "m.text" } - ] - } - - - HS provisions new room with *FIXED* room ID (see notes section) - "!irc.freenode.net/#matrix:hsdomain.com" with normal state events - (e.g. m.room.create). join_rules can be overridden by the AS if supplied in - "state". - - HS injects messages into room. Finds unknown user ID - "@irc.freenode.net/Bob:hsdomain.com" in AS namespace, so queries AS. - - HS -> AS: User Query "@irc.freenode.net/Bob:hsdomain.com" - GET /users/%40irc.freenode.net%2FBob%3Ahsdomain.com?access_token=T_h - Returns 200 OK: - { - profile: { - display_name: "Bob" - } - } + HS -> AS: User Query "@irc.freenode.net/Bob:hsdomain.com" + GET /users/%40irc.freenode.net%2FBob%3Ahsdomain.com?access_token=T_h + [Starts blocking] + AS -> HS: Creates user using CS API extension. + AS -> HS: Set user display name to "Bob". + [Finishes blocking] + [Finished blocking] - - HS provisions new user with display name "Bob". - HS sends room information back to client. 4. @alice:hsdomain.com says "hi!" in this room: @@ -486,4 +442,4 @@ Pre-conditions: msgtype: "m.text" } - - HS modifies the user_id and origin_server_ts on the event and sends it. \ No newline at end of file + - HS modifies the user_id and origin_server_ts on the event and sends it. From 117b0901927ac814a0bc1e8f17532ed74ce19f73 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 19 Jan 2015 10:33:08 +0000 Subject: [PATCH 17/25] Add registration API extension. For AS provisioning of users with its namespace. --- drafts/as-http-api.rst | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 931c8a07..11a404b2 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -166,6 +166,7 @@ Notes: the CS API. - It can also set arbitrary information about the room (e.g. name, topic, etc) using the CS API. + - It can send messages as other users in order to populate scrollback. - When this setup is complete, the AS should respond to the HS request. This means the AS blocks the HS until the room is created and configured. - This is deemed more flexible than alternative methods (e.g. returning an initial sync @@ -258,7 +259,7 @@ every request. It would be an annoying amount of book-keeping to maintain tokens for every virtual user. It would be preferable if the application service could use the CS API with its own ``as_token`` instead, and specify the virtual user they wish to be acting on behalf of. For real users, this would require -additional permissions (see "C-AS Linking"). +additional permissions granting the AS permission to masquerade as a matrix user. Inputs: - Application service token (``access_token``) @@ -312,9 +313,26 @@ Server admin style permissions The home server needs to give the application service *full control* over its namespace, both for users and for room aliases. This means that the AS should be able to create/edit/delete any room alias in its namespace, as well as -create/delete any user in its namespace. This does not require any additional -public APIs. +create/delete any user in its namespace. No additional API changes need to be +made in order for control of room aliases to be granted to the AS. Creation of +users needs API changes in order to: +- Work around captchas. +- Have a 'passwordless' user. + +This involves bypassing the registration flows entirely. This is achieved by +including the AS token on a ``/register`` request, along with a login type of +``m.login.application_service`` to set the desired user ID without a password. + +:: + + /register?access_token=$as_token + + Content: + { + type: "m.login.application_service", + user: "" + } ID conventions ``[TODO]`` ------------------------- @@ -394,6 +412,11 @@ Pre-conditions: GET /users/%40irc.freenode.net%2FBob%3Ahsdomain.com?access_token=T_h [Starts blocking] AS -> HS: Creates user using CS API extension. + POST /register?access_token=T_a + { + type: "m.login.application_service", + user: "irc.freenode.net/Bob" + } AS -> HS: Set user display name to "Bob". [Finishes blocking] [Finished blocking] From ae0fb938adf7c333e47720394c0d9f73f9330402 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 19 Jan 2015 10:45:51 +0000 Subject: [PATCH 18/25] Update IRC example to reflect new room ID --- drafts/as-http-api.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 11a404b2..c8aeca35 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -154,8 +154,7 @@ Inputs: - Room alias - HS Credentials Output: - - The current state events for the room if any. - - The message events for the room if any. + - Whether the room exists. Side effects: - Room is created on the HS by the AS via CS APIs during the processing of this request. @@ -397,10 +396,10 @@ Pre-conditions: HS -> AS: Room Query "#irc.freenode.net/#matrix:hsdomain.com" GET /rooms/%23irc.freenode.net%2F%23matrix%3Ahsdomain.com?access_token=T_h [Starts blocking] - AS -> HS: Creates room. Gets room ID "!aasaasasa:matrix.org". + AS -> HS: Creates room. Gets room ID "!aasaasasa:hsdomain.com". AS -> HS: Sets room name to "#matrix". AS -> HS: Sends message as ""@irc.freenode.net/Bob:hsdomain.com" - PUT /rooms/%21irc.freenode.net%2F%23matrix%3Ahsdomain.com/send/m.room.message + PUT /rooms/%21aasaasasa%3Ahsdomain.com/send/m.room.message ?access_token=T_a &user_id=%40irc.freenode.net%2FBob%3Ahsdomain.com &ts=1421416883133 @@ -426,7 +425,7 @@ Pre-conditions: 4. @alice:hsdomain.com says "hi!" in this room: :: - User -> HS: Send message "hi!" in room !irc.freenode.net/#matrix:hsdomain.com + User -> HS: Send message "hi!" in room !aasaasasa:hsdomain.com - HS sends message. - HS sees the room ID is in the AS namespace and pushes it to the AS. @@ -442,7 +441,7 @@ Pre-conditions: }, origin_server_ts: , user_id: "@alice:hsdomain.com", - room_id: "!irc.freenode.net/#matrix:hsdomain.com", + room_id: "!aasaasasa:hsdomain.com", type: "m.room.message" } ] @@ -456,7 +455,7 @@ Pre-conditions: IRC -> AS: "what's up?" AS -> HS: Send message via CS API extension - PUT /rooms/%21irc.freenode.net%2F%23matrix%3Ahsdomain.com/send/m.room.message + PUT /rooms/%21aasaasasa%3Ahsdomain.com/send/m.room.message ?access_token=T_a &user_id=%40irc.freenode.net%2FBob%3Ahsdomain.com &ts=1421418084816 From 7a2b6abee5ecc713452283034a196d70e5b2bc45 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 20 Jan 2015 14:22:48 +0000 Subject: [PATCH 19/25] Flesh out ID conventions Going down the API route. --- drafts/as-http-api.rst | 70 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 6 deletions(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index c8aeca35..cb64a53c 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -333,16 +333,74 @@ including the AS token on a ``/register`` request, along with a login type of user: "" } -ID conventions ``[TODO]`` +ID conventions ``[Draft]`` ------------------------- +.. NOTE:: + - Giving HSes the freedom to namespace still feels like the Right Thing here. + - Exposing a public API provides the consistency which was the main complaint + against namespacing. + - This may have knock-on effects for the AS registration API. E.g. why don't + we let ASes specify the *URI* regex they want? + This concerns the well-defined conventions for mapping 3P network IDs to matrix IDs, which we expect clients to be able to do by themselves. -- What do user IDs look like? Butchered URIs? Can all 3P network IDs be - reasonably expressed as URIs? (e.g. tel, email, irc, xmpp, ...) -- What do room aliases look like? Some cases are clear (e.g. IRC) but others - are a lot more fiddly (e.g. email? You don't want to share a room with - everyone who has ever sent an email to ``bob@gmail.com``)... +User IDs +~~~~~~~~ +Matrix users may wish to directly contact a virtual user, e.g. to send an email. +The URI format is a well-structured way to represent a number of different ID +types, including: + +- MSISDNs (``tel``) +- Email addresses (``mailto``) +- IRC nicks (``irc`` - https://tools.ietf.org/html/draft-butcher-irc-url-04) +- XMPP (xep-0032) +- SIP URIs (RFC 3261) + +As a result, virtual user IDs SHOULD relate to their URI counterpart. This +mapping from URI to user ID can be expressed in a number of ways: + +- Expose a C-S API on the HS which takes URIs and responds with user IDs. +- Munge the URI with the user ID. + +Exposing an API would allow HSes to internally map user IDs however they like, +at the cost of an extra round trip (of which the response can be cached). +Munging the URI would allow clients to apply the mapping locally, but would force +user X on service Y to always map to the same munged user ID. Considering the +exposed API could just be applying this munging, there is more flexibility if +an API is exposed. + +:: + + GET /_matrix/app/v1/user?uri=$url_encoded_uri + + Returns 200 OK: + { + user_id: + } + +Room Aliases +~~~~~~~~~~~~ +We may want to expose some 3P network rooms so Matrix users can join them directly, +e.g. IRC rooms. We don't want to expose every 3P network room though, e.g. mailto, +tel. Rooms which are publicly accessible (e.g. IRC rooms) can be exposed as an alias by +the application service. Private rooms (e.g. sending an email to someone) should not +be exposed in this way, but should instead operate using normal invite/join semantics. +Therefore, the ID conventions discussed below are only valid for public rooms which +expose room aliases. + +Matrix users may wish to join XMPP rooms (e.g. using XEP-0045) or IRC rooms. In both +cases, these rooms can be expressed as URIs. For consistency, these "room" URIs +SHOULD be mapped in the same way as "user" URIs. + +:: + + GET /_matrix/app/v1/alias?uri=$url_encoded_uri + + Returns 200 OK: + { + alias: + } Examples -------- From 90dfee1c275f350773f583d3a87f054e65b4688f Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 20 Jan 2015 17:49:09 +0000 Subject: [PATCH 20/25] Remove TODO note on regex --- drafts/as-http-api.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index cb64a53c..1c1f342f 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -10,8 +10,6 @@ This contains home server APIs which are used by the application service. Registration API ``[Draft]`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. NOTE:: - - Do we really have to use regex for this? Can't we do this a nicer way? This API registers the application service with its host homeserver to offer its services. From 36395c26111ef7eb09b3cad2d6d2ce4a4bbc40cb Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 6 Feb 2015 11:01:20 +0000 Subject: [PATCH 21/25] Add info on M_EXCLUSIVE --- drafts/as-http-api.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 1c1f342f..0d040868 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -330,6 +330,11 @@ including the AS token on a ``/register`` request, along with a login type of type: "m.login.application_service", user: "" } + +Application services which attempt to create users or aliases *outside* of +their defined namespaces will receive an error code ``M_EXCLUSIVE``. Similarly, +normal users who attempt to create users or alises *inside* an application +service-defined namespace will receive the same ``M_EXCLUSIVE`` error code. ID conventions ``[Draft]`` ------------------------- From 7a59d401f64fd6d1fd3c1c97b4c822b2bb2a0442 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Feb 2015 09:17:55 +0000 Subject: [PATCH 22/25] Fix syntax errors --- drafts/as-http-api.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 0d040868..0dfb6ca7 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -337,7 +337,7 @@ normal users who attempt to create users or alises *inside* an application service-defined namespace will receive the same ``M_EXCLUSIVE`` error code. ID conventions ``[Draft]`` -------------------------- +-------------------------- .. NOTE:: - Giving HSes the freedom to namespace still feels like the Right Thing here. - Exposing a public API provides the consistency which was the main complaint @@ -418,6 +418,7 @@ Pre-conditions: - Home server has the domain "hsdomain.com" 1. Application service registration + :: AS -> HS: Registers itself with the home server @@ -444,12 +445,14 @@ Pre-conditions: } 2. IRC user "Bob" says "hello?" on "#matrix" at timestamp 1421416883133: + :: - AS stores message as potential scrollback. - Nothing happens as no Matrix users are in the room. 3. Matrix user "@alice:hsdomain.com" wants to join "#matrix": + :: User -> HS: Request to join "#irc.freenode.net/#matrix:hsdomain.com" @@ -484,6 +487,7 @@ Pre-conditions: - HS sends room information back to client. 4. @alice:hsdomain.com says "hi!" in this room: + :: User -> HS: Send message "hi!" in room !aasaasasa:hsdomain.com @@ -512,6 +516,7 @@ Pre-conditions: 5. IRC user "Bob" says "what's up?" on "#matrix" at timestamp 1421418084816: + :: IRC -> AS: "what's up?" From bcba42651a2782998bc731f89f9761577d8a5820 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 16 Feb 2015 13:10:25 +0000 Subject: [PATCH 23/25] Add support for exclusive namespaces As per https://github.com/matrix-org/matrix-doc/pull/5#issuecomment-73071259 --- drafts/as-http-api.rst | 41 ++++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 0dfb6ca7..d7b076c3 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -27,11 +27,22 @@ Side effects: API called when: - The application service wants to register with a brand new home server. Notes: + - An application service can state whether they should be the only ones who + can manage a specified namespace. This is referred to as an "exclusive" + namespace. An exclusive namespace prevents humans and other application + services from creating/deleting entities in that namespace. Typically, + exclusive namespaces are used when the rooms represent real rooms on + another service (e.g. IRC). Non-exclusive namespaces are used when the + application service is merely augmenting the room itself (e.g. providing + logging or searching facilities). - Namespaces are represented by POSIX extended regular expressions in JSON. They look like:: users: [ - "@irc\.freenode\.net/.*", + { + "exclusive": true, + "regex": "@irc\.freenode\.net/.*" + } ] :: @@ -44,13 +55,22 @@ Notes: as_token: "some_AS_token", namespaces: { users: [ - "@irc\.freenode\.net/.*" + { + "exclusive": true, + "regex": "@irc\.freenode\.net/.*" + } ], aliases: [ - "#irc\.freenode\.net/.*" + { + "exclusive": true, + "regex": "#irc\.freenode\.net/.*" + } ], rooms: [ - "!irc\.freenode\.net/.*" + { + "exclusive": true, + "regex": "!irc\.freenode\.net/.*" + } ] } } @@ -428,13 +448,16 @@ Pre-conditions: as_token: "T_a", namespaces: { users: [ - "@irc\.freenode\.net/.*" + { + "exclusive": true, + "regex": "@irc\.freenode\.net/.*" + } ], aliases: [ - "#irc\.freenode\.net/.*" - ], - rooms: [ - "!irc\.freenode\.net/.*" + { + "exclusive": true, + "regex": "#irc\.freenode\.net/.*" + } ] } } From f3a498d0bb49a68fcffaadb65afd6ccc2a06e6a3 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 16 Feb 2015 22:09:10 +0000 Subject: [PATCH 24/25] document the concept of a standard `external_url` field for gatewayed events --- drafts/as-http-api.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index d7b076c3..488bfe30 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -425,6 +425,15 @@ SHOULD be mapped in the same way as "user" URIs. alias: } +Event fields +~~~~~~~~~~~~ +We recommend that any gatewayed events should include an `external_url` field in +their content to provide a way for Matrix clients to link into the 'native' +client from which the event originated. For instance, this could contain the +message-ID for emails/nntp posts, or a link to a blog comment when gatewaying +blog comment traffic in & out of matrix + + Examples -------- .. NOTE:: From 3ce1e559545b1ecd0e52e63b89be609142f19dbc Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 19 Feb 2015 11:44:10 +0000 Subject: [PATCH 25/25] Move TODO comments into the doc in prep for merging --- drafts/as-http-api.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drafts/as-http-api.rst b/drafts/as-http-api.rst index 488bfe30..3cc8efbb 100644 --- a/drafts/as-http-api.rst +++ b/drafts/as-http-api.rst @@ -1,3 +1,10 @@ +.. TODO + Sometimes application services need to create rooms (e.g. when lazy loading + from room aliases). Created rooms need to have a user that created them, so + federation works (as it relies on an entry existing in m.room.member). We + should be able to add metadata to m.room.member to state that this user is an + application service, a virtual user, etc. + Application Services HTTP API =============================