Merge remote-tracking branch 'matrix-org/master' into travis/s2s/transactions-swagger

pull/977/head
Travis Ralston 6 years ago
commit cb4fcd1d09

@ -4,12 +4,26 @@ gendoc: &gendoc
source /env/bin/activate source /env/bin/activate
scripts/gendoc.py scripts/gendoc.py
gendoc: &genswagger genswagger: &genswagger
name: Generate the swagger name: Generate the swagger
command: | command: |
source /env/bin/activate source /env/bin/activate
scripts/dump-swagger.py scripts/dump-swagger.py
buildswaggerui: &buildswaggerui
name: Build Swagger UI
command: |
ls scripts/
mkdir -p api/client-server
git clone https://github.com/matrix-org/swagger-ui swagger-ui
cp -r swagger-ui/dist/* api/client-server/
mkdir -p api/client-server/json
cp scripts/swagger/api-docs.json api/client-server/json/
wget https://raw.githubusercontent.com/matrix-org/matrix.org/master/content/swagger.css -O api/client-server/swagger.css
wget https://raw.githubusercontent.com/matrix-org/matrix.org/master/scripts/swagger-ui.patch
patch api/client-server/index.html swagger-ui.patch
version: 2 version: 2
jobs: jobs:
build-docs: build-docs:
@ -30,11 +44,12 @@ jobs:
steps: steps:
- checkout - checkout
- run: *genswagger - run: *genswagger
- run: *buildswaggerui
- store_artifacts: - store_artifacts:
path: scripts/swagger/api-docs.json path: api/client-server/
- run: - run:
name: "Swagger UI is available at:" name: "Swagger UI is available at:"
command: SWAGGER_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/scripts/swagger/api-docs.json"; echo "https://matrix.org/docs/api/client-server/?url="$SWAGGER_URL command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/api/client-server/index.html"; echo $DOCS_URL
workflows: workflows:
version: 2 version: 2

@ -0,0 +1,96 @@
# Copyright 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
type: object
title: Server Keys
description: Server keys
example:
$ref: "../examples/server_key.json"
properties:
server_name:
type: string
description: DNS name of the homeserver.
required: true # TODO: Verify
example: "example.org"
verify_keys:
type: object
description: Public keys of the homeserver for verifying digital signatures.
required: true # TODO: Verify
additionalProperties:
type: object
title: Verify Key
example: {
"ed25519:auto2": {
"key": "Base+64+Encoded+Signature+Verification+Key"
}
}
properties:
key:
type: string
description: The key
required: true
example: "Base+64+Encoded+Signature+Verification+Key"
old_verify_keys:
type: object
description: The public keys that the server used to use and when it stopped using them.
additionalProperties:
type: object
title: Old Verify Key
example: {
"ed25519:auto1": {
"expired_ts": 922834800000,
"key": "Base+64+Encoded+Signature+Verification+Key"
}
}
properties:
expired_ts:
type: integer
format: int64
description: The expiration time.
required: true
example: 922834800000
key:
type: string
description: The key.
required: true
example: "Base+64+Encoded+Signature+Verification+Key"
signatures:
type: object
description: Digital signatures for this object signed using the ``verify_keys``.
additionalProperties:
type: object
title: Signed Server
example: {
"example.org": {
"ad25519:auto2": "Base+64+Encoded+Signature+Verification+Key"
}
}
additionalProperties:
type: string
name: Encoded Signature Verification Key
tls_fingerprints:
type: array
description: Hashes of X.509 TLS certificates used by this server encoded as `Unpadded Base64`_.
items:
type: object
title: TLS Fingerprint
properties:
sha256:
type: string
description: The encoded fingerprint.
example: Base+64+Encoded+SHA-256-Fingerprint
valid_until_ts:
type: integer
format: int64
description: POSIX timestamp when the list of valid keys should be refreshed.
example: 1052262000000

@ -0,0 +1,27 @@
# Copyright 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
type: object
description: Server keys
example: {
"server_keys": [{
$ref: "../examples/server_key.json"
}]
}
properties:
server_keys:
type: array
title: Server Keys
description: The server keys.
items:
$ref: "keys.yaml"

@ -0,0 +1,23 @@
{
"server_name": "example.org",
"verify_keys": {
"ed25519:auto2": {
"key": "Base+64+Encoded+Signature+Verification+Key"
}
},
"old_verify_keys": {
"ed25519:auto1": {
"expired_ts": 922834800000,
"key": "Base+64+Encoded+Old+Verify+Key"
}
},
"signatures": {
"example.org": {
"ed25519:auto2": "Base+64+Encoded+Signature"
}
},
"tls_fingerprints": [{
"sha256": "Base+64+Encoded+SHA-256-Fingerprint"
}],
"valid_until_ts": 1052262000000
}

@ -0,0 +1,99 @@
# Copyright 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
swagger: '2.0'
info:
title: "Matrix Federation Key Exchange API"
version: "1.0.0"
host: localhost:8448
schemes:
- https
basePath: /_matrix/key/v2
produces:
- application/json
paths:
"/query/{serverName}/{keyId}":
get:
summary: Retrieve a server key.
description: Retrieve a server key.
operationId: perspectivesKeyQuery
parameters:
- in: path
name: serverName
type: string
description: Server name.
required: true
x-example: matrix.org
- in: path
name: keyId
type: string
description: Key ID.
required: true
x-example: TODO # No examples in spec so far
- in: query
name: minimum_valid_until_ts
type: integer
format: int64
description: Minimum Valid Until Milliseconds.
required: true # TODO: Verify
x-example: 1234567890
responses:
200:
description: The keys for the server
schema:
$ref: "definitions/keys_query_response.yaml"
"/query":
post:
summary: Retrieve a server key
description: Retrieve a server key.
operationId: bulkPerspectivesKeyQuery
parameters:
- in: body
name: body
schema:
type: object
# TODO: Improve example
example: {
"server_keys": {
"{server_name}": {
"{key_id}": {
"minimum_valid_until_ts": 1234567890
}
}
}
}
properties:
server_keys:
type: object
description: The query criteria.
additionalProperties:
type: object
name: ServerName
description: The server names to query.
additionalProperties:
type: object
title: Query Criteria
description: The server keys to query.
properties:
minimum_valid_until_ts:
type: integer
format: int64
description: Minimum Valid Until MS.
example: 1234567890
required: ['server_keys']
responses:
200:
description: The keys for the server.
schema:
$ref: "definitions/keys_query_response.yaml"

@ -0,0 +1,42 @@
# Copyright 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
swagger: '2.0'
info:
title: "Matrix Federation Key Exchange API"
version: "1.0.0"
host: localhost:8448
schemes:
- https
basePath: /_matrix/key/v2
produces:
- application/json
paths:
"/server/{keyId}":
get:
summary: Get the server's key
description: Get the server's key.
operationId: getServerKey
parameters:
- in: path
name: keyId
type: string
description: Key ID
required: false
x-example: TODO # No examples in the spec so far
responses:
200:
description: The server's keys.
schema:
$ref: "definitions/keys.yaml"

@ -0,0 +1,190 @@
# Copyright 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
swagger: '2.0'
info:
title: "Matrix Federation Third Party Invites API"
version: "1.0.0"
host: localhost:8448
schemes:
- https
basePath: /_matrix/federation/v1
produces:
- application/json
paths:
"/exchange_third_party_invite/{roomId}":
put:
summary: Request a server to auth a third party invite event
description: |-
The receiving server will verify the partial ``m.room.member`` event
given in the request body. If valid, the receiving server will issue
an invite as per the `Inviting to a room`_ section before returning a
response to this request.
operationId: exchangeThirdPartyInvite
parameters:
- in: path
name: roomId
type: string
description: The room ID to exchange a third party invite in
required: true
x-example: "!abc123:matrix.org"
- in: body
name: body
type: object
description: A partial ``m.room.member`` event
required: true
schema:
type: object
properties:
type:
type: string
description: The event type. Must be ``m.room.member``
example: "m.room.member"
room_id:
type: string
description: |-
The room ID the event is for. Must match the ID given in
the path.
example: "!abc123:matrix.org"
sender:
type: string
description: |-
The user ID of the user who sent the original ``m.room.third_party_invite``
event.
example: "@joe:matrix.org"
state_key:
type: string
description: The user ID of the invited user
example: "@someone:example.org"
content:
type: object
description: The event content
title: Event Content
properties:
membership:
type: string
description: The membership state. Must be ``invite``
example: invite
third_party_invite:
type: object
description: The third party invite
properties:
display_name:
type: string
description: |-
A name which can be displayed to represent the user instead of their
third party identifier.
example: "alice"
signed:
type: object
description: |-
A block of content which has been signed, which servers can use to
verify the event.
properties:
signatures:
type: object
description: The server signatures for this event.
additionalProperties:
type: object
title: Server Signatures
additionalProperties:
type: string
example: {
"magic.forest": {
"ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg"
}
}
mxid:
type: string
description: The invited matrix user ID
example: "@alice:localhost"
token:
type: string
description: The token used to verify the event
example: abc123
required: ['signatures', 'mxid', 'token']
example: {
"mxid": "@alice:localhost",
"token": "abc123",
"signatures": {
"magic.forest": {
"ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg"
}
}
}
required: ['display_name', 'signed']
example: {
"display_name": "alice",
"signed": {
"mxid": "@alice:localhost",
"token": "abc123",
"signatures": {
"magic.forest": {
"ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg"
}
}
}
}
required: ['membership', 'third_party_invite']
example: {
"membership": "invite",
"third_party_invite": {
"display_name": "alice",
"signed": {
"mxid": "@alice:localhost",
"token": "abc123",
"signatures": {
"magic.forest": {
"ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg"
}
}
}
}
}
required:
- type
- room_id
- sender
- state_key
- content
example: {
"type": "m.room.member",
"room_id": "!abc123:matrix.org",
"sender": "@joe:matrix.org",
"state_key": "@someone:example.org",
"content": {
"membership": "invite",
"third_party_invite": {
"display_name": "alice",
"signed": {
"mxid": "@alice:localhost",
"token": "abc123",
"signatures": {
"magic.forest": {
"ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg"
}
}
}
}
}
}
responses:
200:
description: The invite has been issued successfully.
examples:
application/json: {}
schema:
type: object
description: An empty object
example: {}

@ -0,0 +1,9 @@
[ tool.giles ]
[ tool.giles.circleci_artifacts.docs ]
url = "gen/index.html"
message = "Click details to preview the HTML documentation."
[ tool.giles.circleci_artifacts.swagger ]
url = "client-server/index.html"
message = "Click to preview the swagger build."

@ -1,4 +1,4 @@
#!/usr/bin/env python2 #!/usr/bin/env python3
# dump-swagger reads all of the swagger API docs used in spec generation and # dump-swagger reads all of the swagger API docs used in spec generation and
# outputs a JSON file which merges them all, for use as input to a swagger UI # outputs a JSON file which merges them all, for use as input to a swagger UI

@ -1417,7 +1417,7 @@ have to wait in milliseconds before they can try again.
.. References .. References
.. _`macaroon`: http://research.google.com/pubs/pub41892.html .. _`macaroon`: http://research.google.com/pubs/pub41892.html
.. _`devices`: ../intro.html#devices .. _`devices`: ../index.html#devices
.. Links through the external API docs are below .. Links through the external API docs are below
.. ============================================= .. =============================================

@ -30,18 +30,392 @@ communication ecosystem.
To propose a change to the Matrix Spec, see the explanations at To propose a change to the Matrix Spec, see the explanations at
`Proposals for Spec Changes to Matrix <proposals>`_. `Proposals for Spec Changes to Matrix <proposals>`_.
.. contents:: Table of Contents
.. sectnum::
Matrix APIs Matrix APIs
----------- -----------
The specification consists of the following parts: The specification consists of the following parts:
`Introduction to Matrix <intro.html>`_ provides a full introduction to Matrix and the spec.
{{apis}} {{apis}}
The `Appendices <appendices.html>`_ contain supplemental information not specific to The `Appendices <appendices.html>`_ contain supplemental information not specific to
one of the above APIs. one of the above APIs.
Introduction to the Matrix APIs
-------------------------------
.. WARNING::
The Matrix specification is still evolving: the APIs are not yet frozen
and this document is in places a work in progress or stale. We have made every
effort to clearly flag areas which are still being finalised.
We're publishing it at this point because it's complete enough to be more than
useful and provide a canonical reference to how Matrix is evolving. Our end
goal is to mirror WHATWG's `Living Standard
<https://whatwg.org/faq?#living-standard>`_.
Matrix is a set of open APIs for open-federated Instant Messaging (IM), Voice
over IP (VoIP) and Internet of Things (IoT) communication, designed to create
and support a new global real-time communication ecosystem. The intention is to
provide an open decentralised pubsub layer for the internet for securely
persisting and publishing/subscribing JSON objects. This specification is the
ongoing result of standardising the APIs used by the various components of the
Matrix ecosystem to communicate with one another.
The principles that Matrix attempts to follow are:
- Pragmatic Web-friendly APIs (i.e. JSON over REST)
- Keep It Simple & Stupid
+ provide a simple architecture with minimal third-party dependencies.
- Fully open:
+ Fully open federation - anyone should be able to participate in the global
Matrix network
+ Fully open standard - publicly documented standard with no IP or patent
licensing encumbrances
+ Fully open source reference implementation - liberally-licensed example
implementations with no IP or patent licensing encumbrances
- Empowering the end-user
+ The user should be able to choose the server and clients they use
+ The user should be control how private their communication is
+ The user should know precisely where their data is stored
- Fully decentralised - no single points of control over conversations or the
network as a whole
- Learning from history to avoid repeating it
+ Trying to take the best aspects of XMPP, SIP, IRC, SMTP, IMAP and NNTP
whilst trying to avoid their failings
The functionality that Matrix provides includes:
- Creation and management of fully distributed chat rooms with no
single points of control or failure
- Eventually-consistent cryptographically secure synchronisation of room
state across a global open network of federated servers and services
- Sending and receiving extensible messages in a room with (optional)
end-to-end encryption
- Extensible user management (inviting, joining, leaving, kicking, banning)
mediated by a power-level based user privilege system.
- Extensible room state management (room naming, aliasing, topics, bans)
- Extensible user profile management (avatars, display names, etc)
- Managing user accounts (registration, login, logout)
- Use of 3rd Party IDs (3PIDs) such as email addresses, phone numbers,
Facebook accounts to authenticate, identify and discover users on Matrix.
- Trusted federation of Identity servers for:
+ Publishing user public keys for PKI
+ Mapping of 3PIDs to Matrix IDs
The end goal of Matrix is to be a ubiquitous messaging layer for synchronising
arbitrary data between sets of people, devices and services - be that for
instant messages, VoIP call setups, or any other objects that need to be
reliably and persistently pushed from A to B in an interoperable and federated
manner.
Spec Change Proposals
~~~~~~~~~~~~~~~~~~~~~
To propose a change to the Matrix Spec, see the explanations at `Proposals
for Spec Changes to Matrix <proposals>`_.
Architecture
------------
Matrix defines APIs for synchronising extensible JSON objects known as
"events" between compatible clients, servers and services. Clients are
typically messaging/VoIP applications or IoT devices/hubs and communicate by
synchronising communication history with their "homeserver" using the
"Client-Server API". Each homeserver stores the communication history and
account information for all of its clients, and shares data with the wider
Matrix ecosystem by synchronising communication history with other homeservers
and their clients.
Clients typically communicate with each other by emitting events in the
context of a virtual "room". Room data is replicated across *all of the
homeservers* whose users are participating in a given room. As such, *no
single homeserver has control or ownership over a given room*. Homeservers
model communication history as a partially ordered graph of events known as
the room's "event graph", which is synchronised with eventual consistency
between the participating servers using the "Server-Server API". This process
of synchronising shared conversation history between homeservers run by
different parties is called "Federation". Matrix optimises for the
Availability and Partitioned properties of CAP theorem at
the expense of Consistency.
For example, for client A to send a message to client B, client A performs an
HTTP PUT of the required JSON event on its homeserver (HS) using the
client-server API. A's HS appends this event to its copy of the room's event
graph, signing the message in the context of the graph for integrity. A's HS
then replicates the message to B's HS by performing an HTTP PUT using the
server-server API. B's HS authenticates the request, validates the event's
signature, authorises the event's contents and then adds it to its copy of the
room's event graph. Client B then receives the message from his homeserver via
a long-lived GET request.
::
How data flows between clients
==============================
{ Matrix client A } { Matrix client B }
^ | ^ |
| events | Client-Server API | events |
| V | V
+------------------+ +------------------+
| |---------( HTTPS )--------->| |
| homeserver | | homeserver |
| |<--------( HTTPS )----------| |
+------------------+ Server-Server API +------------------+
History Synchronisation
(Federation)
Users
~~~~~
Each client is associated with a user account, which is identified in Matrix
using a unique "user ID". This ID is namespaced to the homeserver which
allocated the account and has the form::
@localpart:domain
See `'Identifier Grammar' the appendices <appendices.html#identifier-grammar>`_ for full details of
the structure of user IDs.
Devices
~~~~~~~
The Matrix specification has a particular meaning for the term "device". As a
user, I might have several devices: a desktop client, some web browsers, an
Android device, an iPhone, etc. They broadly relate to a real device in the
physical world, but you might have several browsers on a physical device, or
several Matrix client applications on a mobile device, each of which would be
its own device.
Devices are used primarily to manage the keys used for end-to-end encryption
(each device gets its own copy of the decryption keys), but they also help
users manage their access - for instance, by revoking access to particular
devices.
When a user first uses a client, it registers itself as a new device. The
longevity of devices might depend on the type of client. A web client will
probably drop all of its state on logout, and create a new device every time
you log in, to ensure that cryptography keys are not leaked to a new user. In
a mobile client, it might be acceptable to reuse the device if a login session
expires, provided the user is the same.
Devices are identified by a ``device_id``, which is unique within the scope of
a given user.
A user may assign a human-readable display name to a device, to help them
manage their devices.
Events
~~~~~~
All data exchanged over Matrix is expressed as an "event". Typically each client
action (e.g. sending a message) correlates with exactly one event. Each event
has a ``type`` which is used to differentiate different kinds of data. ``type``
values MUST be uniquely globally namespaced following Java's `package naming
conventions`_, e.g.
``com.example.myapp.event``. The special top-level namespace ``m.`` is reserved
for events defined in the Matrix specification - for instance ``m.room.message``
is the event type for instant messages. Events are usually sent in the context
of a "Room".
.. _package naming conventions: https://en.wikipedia.org/wiki/Java_package#Package_naming_conventions
Event Graphs
~~~~~~~~~~~~
.. _sect:event-graph:
Events exchanged in the context of a room are stored in a directed acyclic graph
(DAG) called an "event graph". The partial ordering of this graph gives the
chronological ordering of events within the room. Each event in the graph has a
list of zero or more "parent" events, which refer to any preceding events
which have no chronological successor from the perspective of the homeserver
which created the event.
Typically an event has a single parent: the most recent message in the room at
the point it was sent. However, homeservers may legitimately race with each
other when sending messages, resulting in a single event having multiple
successors. The next event added to the graph thus will have multiple parents.
Every event graph has a single root event with no parent.
To order and ease chronological comparison between the events within the graph,
homeservers maintain a ``depth`` metadata field on each event. An event's
``depth`` is a positive integer that is strictly greater than the depths of any
of its parents. The root event should have a depth of 1. Thus if one event is
before another, then it must have a strictly smaller depth.
Room structure
~~~~~~~~~~~~~~
A room is a conceptual place where users can send and receive events. Events are
sent to a room, and all participants in that room with sufficient access will
receive the event. Rooms are uniquely identified internally via "Room IDs",
which have the form::
!opaque_id:domain
There is exactly one room ID for each room. Whilst the room ID does contain a
domain, it is simply for globally namespacing room IDs. The room does NOT
reside on the domain specified.
See `'Identifier Grammar' in the appendices <appendices.html#identifier-grammar>`_ for full details of
the structure of a room ID.
The following conceptual diagram shows an
``m.room.message`` event being sent to the room ``!qporfwt:matrix.org``::
{ @alice:matrix.org } { @bob:domain.com }
| ^
| |
[HTTP POST] [HTTP GET]
Room ID: !qporfwt:matrix.org Room ID: !qporfwt:matrix.org
Event type: m.room.message Event type: m.room.message
Content: { JSON object } Content: { JSON object }
| |
V |
+------------------+ +------------------+
| homeserver | | homeserver |
| matrix.org | | domain.com |
+------------------+ +------------------+
| ^
| [HTTP PUT] |
| Room ID: !qporfwt:matrix.org |
| Event type: m.room.message |
| Content: { JSON object } |
`-------> Pointer to the preceding message ------`
PKI signature from matrix.org
Transaction-layer metadata
PKI Authorization header
...................................
| Shared Data |
| State: |
| Room ID: !qporfwt:matrix.org |
| Servers: matrix.org, domain.com |
| Members: |
| - @alice:matrix.org |
| - @bob:domain.com |
| Messages: |
| - @alice:matrix.org |
| Content: { JSON object } |
|...................................|
Federation maintains *shared data structures* per-room between multiple home
servers. The data is split into ``message events`` and ``state events``.
Message events:
These describe transient 'once-off' activity in a room such as an
instant messages, VoIP call setups, file transfers, etc. They generally
describe communication activity.
State events:
These describe updates to a given piece of persistent information
('state') related to a room, such as the room's name, topic, membership,
participating servers, etc. State is modelled as a lookup table of key/value
pairs per room, with each key being a tuple of ``state_key`` and ``event type``.
Each state event updates the value of a given key.
The state of the room at a given point is calculated by considering all events
preceding and including a given event in the graph. Where events describe the
same state, a merge conflict algorithm is applied. The state resolution
algorithm is transitive and does not depend on server state, as it must
consistently select the same event irrespective of the server or the order the
events were received in. Events are signed by the originating server (the
signature includes the parent relations, type, depth and payload hash) and are
pushed over federation to the participating servers in a room, currently using
full mesh topology. Servers may also request backfill of events over federation
from the other servers participating in a room.
Room Aliases
++++++++++++
Each room can also have multiple "Room Aliases", which look like::
#room_alias:domain
See `'Identifier Grammar' in the appendices <appendices.html#identifier-grammar>`_ for full details of
the structure of a room alias.
A room alias "points" to a room ID and is the human-readable label by which
rooms are publicised and discovered. The room ID the alias is pointing to can
be obtained by visiting the domain specified. Note that the mapping from a room
alias to a room ID is not fixed, and may change over time to point to a
different room ID. For this reason, Clients SHOULD resolve the room alias to a
room ID once and then use that ID on subsequent requests.
When resolving a room alias the server will also respond with a list of servers
that are in the room that can be used to join via.
::
HTTP GET
#matrix:domain.com !aaabaa:matrix.org
| ^
| |
_______V____________________|____
| domain.com |
| Mappings: |
| #matrix >> !aaabaa:matrix.org |
| #golf >> !wfeiofh:sport.com |
| #bike >> !4rguxf:matrix.org |
|________________________________|
Identity
~~~~~~~~
Users in Matrix are identified via their Matrix user ID. However,
existing 3rd party ID namespaces can also be used in order to identify Matrix
users. A Matrix "Identity" describes both the user ID and any other existing IDs
from third party namespaces *linked* to their account.
Matrix users can *link* third-party IDs (3PIDs) such as email addresses, social
network accounts and phone numbers to their user ID. Linking 3PIDs creates a
mapping from a 3PID to a user ID. This mapping can then be used by Matrix
users in order to discover the user IDs of their contacts.
In order to ensure that the mapping from 3PID to user ID is genuine, a globally
federated cluster of trusted "Identity Servers" (IS) are used to verify the 3PID
and persist and replicate the mappings.
Usage of an IS is not required in order for a client application to be part of
the Matrix ecosystem. However, without one clients will not be able to look up
user IDs using 3PIDs.
Profiles
~~~~~~~~
Users may publish arbitrary key/value data associated with their account - such
as a human readable display name, a profile photo URL, contact information
(email address, phone numbers, website URLs etc).
.. TODO
Actually specify the different types of data - e.g. what format are display
names allowed to be?
Private User Data
~~~~~~~~~~~~~~~~~
Users may also store arbitrary private key/value data in their account - such as
client preferences, or server configuration settings which lack any other
dedicated API. The API is symmetrical to managing Profile data.
.. TODO
Would it really be overengineered to use the same API for both profile &
private user data, but with different ACLs?
Specification Versions Specification Versions
---------------------- ----------------------
@ -57,3 +431,9 @@ The specification for each API is versioned in the form ``rX.Y.Z``.
* A change to ``Z`` represents a change which is backwards-compatible on both * A change to ``Z`` represents a change which is backwards-compatible on both
sides. Typically this implies a clarification to the specification, rather sides. Typically this implies a clarification to the specification, rather
than a change which must be implemented. than a change which must be implemented.
License
-------
The Matrix specification is licensed under the `Apache License, Version 2.0
<http://www.apache.org/licenses/LICENSE-2.0>`_.

@ -1,400 +0,0 @@
.. Copyright 2016 OpenMarket Ltd
..
.. Licensed under the Apache License, Version 2.0 (the "License");
.. you may not use this file except in compliance with the License.
.. You may obtain a copy of the License at
..
.. http://www.apache.org/licenses/LICENSE-2.0
..
.. Unless required by applicable law or agreed to in writing, software
.. distributed under the License is distributed on an "AS IS" BASIS,
.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.. See the License for the specific language governing permissions and
.. limitations under the License.
.. contents:: Table of Contents
.. sectnum::
.. Note that this file is specifically unversioned because we don't want to
.. have to add Yet Another version number, and the commentary on what specs we
.. have should hopefully not get complex enough that we need to worry about
.. versioning it.
Introduction
------------
.. WARNING::
The Matrix specification is still evolving: the APIs are not yet frozen
and this document is in places a work in progress or stale. We have made every
effort to clearly flag areas which are still being finalised.
We're publishing it at this point because it's complete enough to be more than
useful and provide a canonical reference to how Matrix is evolving. Our end
goal is to mirror WHATWG's `Living Standard
<https://whatwg.org/faq?#living-standard>`_.
Matrix is a set of open APIs for open-federated Instant Messaging (IM), Voice
over IP (VoIP) and Internet of Things (IoT) communication, designed to create
and support a new global real-time communication ecosystem. The intention is to
provide an open decentralised pubsub layer for the internet for securely
persisting and publishing/subscribing JSON objects. This specification is the
ongoing result of standardising the APIs used by the various components of the
Matrix ecosystem to communicate with one another.
The principles that Matrix attempts to follow are:
- Pragmatic Web-friendly APIs (i.e. JSON over REST)
- Keep It Simple & Stupid
+ provide a simple architecture with minimal third-party dependencies.
- Fully open:
+ Fully open federation - anyone should be able to participate in the global
Matrix network
+ Fully open standard - publicly documented standard with no IP or patent
licensing encumbrances
+ Fully open source reference implementation - liberally-licensed example
implementations with no IP or patent licensing encumbrances
- Empowering the end-user
+ The user should be able to choose the server and clients they use
+ The user should be control how private their communication is
+ The user should know precisely where their data is stored
- Fully decentralised - no single points of control over conversations or the
network as a whole
- Learning from history to avoid repeating it
+ Trying to take the best aspects of XMPP, SIP, IRC, SMTP, IMAP and NNTP
whilst trying to avoid their failings
The functionality that Matrix provides includes:
- Creation and management of fully distributed chat rooms with no
single points of control or failure
- Eventually-consistent cryptographically secure synchronisation of room
state across a global open network of federated servers and services
- Sending and receiving extensible messages in a room with (optional)
end-to-end encryption
- Extensible user management (inviting, joining, leaving, kicking, banning)
mediated by a power-level based user privilege system.
- Extensible room state management (room naming, aliasing, topics, bans)
- Extensible user profile management (avatars, display names, etc)
- Managing user accounts (registration, login, logout)
- Use of 3rd Party IDs (3PIDs) such as email addresses, phone numbers,
Facebook accounts to authenticate, identify and discover users on Matrix.
- Trusted federation of Identity servers for:
+ Publishing user public keys for PKI
+ Mapping of 3PIDs to Matrix IDs
The end goal of Matrix is to be a ubiquitous messaging layer for synchronising
arbitrary data between sets of people, devices and services - be that for
instant messages, VoIP call setups, or any other objects that need to be
reliably and persistently pushed from A to B in an interoperable and federated
manner.
Spec Change Proposals
~~~~~~~~~~~~~~~~~~~~~
To propose a change to the Matrix Spec, see the explanations at `Proposals
for Spec Changes to Matrix <proposals>`_.
Architecture
------------
Matrix defines APIs for synchronising extensible JSON objects known as
"events" between compatible clients, servers and services. Clients are
typically messaging/VoIP applications or IoT devices/hubs and communicate by
synchronising communication history with their "homeserver" using the
"Client-Server API". Each homeserver stores the communication history and
account information for all of its clients, and shares data with the wider
Matrix ecosystem by synchronising communication history with other homeservers
and their clients.
Clients typically communicate with each other by emitting events in the
context of a virtual "room". Room data is replicated across *all of the
homeservers* whose users are participating in a given room. As such, *no
single homeserver has control or ownership over a given room*. Homeservers
model communication history as a partially ordered graph of events known as
the room's "event graph", which is synchronised with eventual consistency
between the participating servers using the "Server-Server API". This process
of synchronising shared conversation history between homeservers run by
different parties is called "Federation". Matrix optimises for the
Availability and Partitioned properties of CAP theorem at
the expense of Consistency.
For example, for client A to send a message to client B, client A performs an
HTTP PUT of the required JSON event on its homeserver (HS) using the
client-server API. A's HS appends this event to its copy of the room's event
graph, signing the message in the context of the graph for integrity. A's HS
then replicates the message to B's HS by performing an HTTP PUT using the
server-server API. B's HS authenticates the request, validates the event's
signature, authorises the event's contents and then adds it to its copy of the
room's event graph. Client B then receives the message from his homeserver via
a long-lived GET request.
::
How data flows between clients
==============================
{ Matrix client A } { Matrix client B }
^ | ^ |
| events | Client-Server API | events |
| V | V
+------------------+ +------------------+
| |---------( HTTPS )--------->| |
| homeserver | | homeserver |
| |<--------( HTTPS )----------| |
+------------------+ Server-Server API +------------------+
History Synchronisation
(Federation)
Users
~~~~~
Each client is associated with a user account, which is identified in Matrix
using a unique "user ID". This ID is namespaced to the homeserver which
allocated the account and has the form::
@localpart:domain
See the `appendices <appendices.html#identifier-grammar>`_ for full details of
the structure of user IDs.
Devices
~~~~~~~
The Matrix specification has a particular meaning for the term "device". As a
user, I might have several devices: a desktop client, some web browsers, an
Android device, an iPhone, etc. They broadly relate to a real device in the
physical world, but you might have several browsers on a physical device, or
several Matrix client applications on a mobile device, each of which would be
its own device.
Devices are used primarily to manage the keys used for end-to-end encryption
(each device gets its own copy of the decryption keys), but they also help
users manage their access - for instance, by revoking access to particular
devices.
When a user first uses a client, it registers itself as a new device. The
longevity of devices might depend on the type of client. A web client will
probably drop all of its state on logout, and create a new device every time
you log in, to ensure that cryptography keys are not leaked to a new user. In
a mobile client, it might be acceptable to reuse the device if a login session
expires, provided the user is the same.
Devices are identified by a ``device_id``, which is unique within the scope of
a given user.
A user may assign a human-readable display name to a device, to help them
manage their devices.
Events
~~~~~~
All data exchanged over Matrix is expressed as an "event". Typically each client
action (e.g. sending a message) correlates with exactly one event. Each event
has a ``type`` which is used to differentiate different kinds of data. ``type``
values MUST be uniquely globally namespaced following Java's `package naming
conventions`_, e.g.
``com.example.myapp.event``. The special top-level namespace ``m.`` is reserved
for events defined in the Matrix specification - for instance ``m.room.message``
is the event type for instant messages. Events are usually sent in the context
of a "Room".
.. _package naming conventions: https://en.wikipedia.org/wiki/Java_package#Package_naming_conventions
Event Graphs
~~~~~~~~~~~~
.. _sect:event-graph:
Events exchanged in the context of a room are stored in a directed acyclic graph
(DAG) called an "event graph". The partial ordering of this graph gives the
chronological ordering of events within the room. Each event in the graph has a
list of zero or more "parent" events, which refer to any preceding events
which have no chronological successor from the perspective of the homeserver
which created the event.
Typically an event has a single parent: the most recent message in the room at
the point it was sent. However, homeservers may legitimately race with each
other when sending messages, resulting in a single event having multiple
successors. The next event added to the graph thus will have multiple parents.
Every event graph has a single root event with no parent.
To order and ease chronological comparison between the events within the graph,
homeservers maintain a ``depth`` metadata field on each event. An event's
``depth`` is a positive integer that is strictly greater than the depths of any
of its parents. The root event should have a depth of 1. Thus if one event is
before another, then it must have a strictly smaller depth.
Room structure
~~~~~~~~~~~~~~
A room is a conceptual place where users can send and receive events. Events are
sent to a room, and all participants in that room with sufficient access will
receive the event. Rooms are uniquely identified internally via "Room IDs",
which have the form::
!opaque_id:domain
There is exactly one room ID for each room. Whilst the room ID does contain a
domain, it is simply for globally namespacing room IDs. The room does NOT
reside on the domain specified.
See the `appendices <appendices.html#identifier-grammar>`_ for full details of
the structure of a room ID.
The following conceptual diagram shows an
``m.room.message`` event being sent to the room ``!qporfwt:matrix.org``::
{ @alice:matrix.org } { @bob:domain.com }
| ^
| |
[HTTP POST] [HTTP GET]
Room ID: !qporfwt:matrix.org Room ID: !qporfwt:matrix.org
Event type: m.room.message Event type: m.room.message
Content: { JSON object } Content: { JSON object }
| |
V |
+------------------+ +------------------+
| homeserver | | homeserver |
| matrix.org | | domain.com |
+------------------+ +------------------+
| ^
| [HTTP PUT] |
| Room ID: !qporfwt:matrix.org |
| Event type: m.room.message |
| Content: { JSON object } |
`-------> Pointer to the preceding message ------`
PKI signature from matrix.org
Transaction-layer metadata
PKI Authorization header
...................................
| Shared Data |
| State: |
| Room ID: !qporfwt:matrix.org |
| Servers: matrix.org, domain.com |
| Members: |
| - @alice:matrix.org |
| - @bob:domain.com |
| Messages: |
| - @alice:matrix.org |
| Content: { JSON object } |
|...................................|
Federation maintains *shared data structures* per-room between multiple home
servers. The data is split into ``message events`` and ``state events``.
Message events:
These describe transient 'once-off' activity in a room such as an
instant messages, VoIP call setups, file transfers, etc. They generally
describe communication activity.
State events:
These describe updates to a given piece of persistent information
('state') related to a room, such as the room's name, topic, membership,
participating servers, etc. State is modelled as a lookup table of key/value
pairs per room, with each key being a tuple of ``state_key`` and ``event type``.
Each state event updates the value of a given key.
The state of the room at a given point is calculated by considering all events
preceding and including a given event in the graph. Where events describe the
same state, a merge conflict algorithm is applied. The state resolution
algorithm is transitive and does not depend on server state, as it must
consistently select the same event irrespective of the server or the order the
events were received in. Events are signed by the originating server (the
signature includes the parent relations, type, depth and payload hash) and are
pushed over federation to the participating servers in a room, currently using
full mesh topology. Servers may also request backfill of events over federation
from the other servers participating in a room.
Room Aliases
++++++++++++
Each room can also have multiple "Room Aliases", which look like::
#room_alias:domain
See the `appendices <appendices.html#identifier-grammar>`_ for full details of
the structure of a room alias.
A room alias "points" to a room ID and is the human-readable label by which
rooms are publicised and discovered. The room ID the alias is pointing to can
be obtained by visiting the domain specified. Note that the mapping from a room
alias to a room ID is not fixed, and may change over time to point to a
different room ID. For this reason, Clients SHOULD resolve the room alias to a
room ID once and then use that ID on subsequent requests.
When resolving a room alias the server will also respond with a list of servers
that are in the room that can be used to join via.
::
HTTP GET
#matrix:domain.com !aaabaa:matrix.org
| ^
| |
_______V____________________|____
| domain.com |
| Mappings: |
| #matrix >> !aaabaa:matrix.org |
| #golf >> !wfeiofh:sport.com |
| #bike >> !4rguxf:matrix.org |
|________________________________|
Identity
~~~~~~~~
Users in Matrix are identified via their Matrix user ID. However,
existing 3rd party ID namespaces can also be used in order to identify Matrix
users. A Matrix "Identity" describes both the user ID and any other existing IDs
from third party namespaces *linked* to their account.
Matrix users can *link* third-party IDs (3PIDs) such as email addresses, social
network accounts and phone numbers to their user ID. Linking 3PIDs creates a
mapping from a 3PID to a user ID. This mapping can then be used by Matrix
users in order to discover the user IDs of their contacts.
In order to ensure that the mapping from 3PID to user ID is genuine, a globally
federated cluster of trusted "Identity Servers" (IS) are used to verify the 3PID
and persist and replicate the mappings.
Usage of an IS is not required in order for a client application to be part of
the Matrix ecosystem. However, without one clients will not be able to look up
user IDs using 3PIDs.
Profiles
~~~~~~~~
Users may publish arbitrary key/value data associated with their account - such
as a human readable display name, a profile photo URL, contact information
(email address, phone numbers, website URLs etc).
.. TODO
Actually specify the different types of data - e.g. what format are display
names allowed to be?
Private User Data
~~~~~~~~~~~~~~~~~
Users may also store arbitrary private key/value data in their account - such as
client preferences, or server configuration settings which lack any other
dedicated API. The API is symmetrical to managing Profile data.
.. TODO
Would it really be overengineered to use the same API for both profile &
private user data, but with different ACLs?
License
-------
The Matrix specification is licensed under the `Apache License, Version 2.0
<http://www.apache.org/licenses/LICENSE-2.0>`_.

@ -98,9 +98,9 @@ check for certain user attributes in the response. Any required attributes
should be configured by the server administrator. should be configured by the server administrator.
Once the ticket has been validated, the server MUST map the CAS ``user_id`` Once the ticket has been validated, the server MUST map the CAS ``user_id``
to a valid `Matrix user identifier <../intro.html#user-identifiers>`_. The to a valid `Matrix user identifier <../index.html#user-identifiers>`_. The
guidance in `Mapping from other character sets guidance in `Mapping from other character sets
<../intro.html#mapping-from-other-character-sets>`_ may be useful. <../index.html#mapping-from-other-character-sets>`_ may be useful.
If the generated user identifier represents a new user, it should be registered If the generated user identifier represents a new user, it should be registered
as a new user. as a new user.

@ -1,5 +1,6 @@
.. Copyright 2016 OpenMarket Ltd .. Copyright 2016 OpenMarket Ltd
.. Copyright 2017 New Vector Ltd .. Copyright 2017 New Vector Ltd
.. Copyright 2018 New Vector Ltd
.. ..
.. Licensed under the Apache License, Version 2.0 (the "License"); .. Licensed under the Apache License, Version 2.0 (the "License");
.. you may not use this file except in compliance with the License. .. you may not use this file except in compliance with the License.
@ -166,50 +167,8 @@ If a server goes offline intermediate notary servers should continue to return
the last response they received from that server so that the signatures of old the last response they received from that server so that the signatures of old
events sent by that server can still be checked. events sent by that server can still be checked.
==================== =================== ====================================== {{keys_server_ss_http_api}}
Key Type Description
==================== =================== ======================================
``server_name`` String DNS name of the homeserver.
``verify_keys`` Object Public keys of the homeserver for
verifying digital signatures.
``old_verify_keys`` Object The public keys that the server used
to use and when it stopped using them.
``signatures`` Object Digital signatures for this object
signed using the ``verify_keys``.
``tls_fingerprints`` Array of Objects Hashes of X.509 TLS certificates used
by this server encoded as `Unpadded Base64`_.
``valid_until_ts`` Integer POSIX timestamp when the list of valid
keys should be refreshed.
==================== =================== ======================================
.. code:: json
{
"old_verify_keys": {
"ed25519:auto1": {
"expired_ts": 922834800000,
"key": "Base+64+Encoded+Old+Verify+Key"
}
},
"server_name": "example.org",
"signatures": {
"example.org": {
"ed25519:auto2": "Base+64+Encoded+Signature"
}
},
"tls_fingerprints": [
{
"sha256": "Base+64+Encoded+SHA-256-Fingerprint"
}
],
"valid_until_ts": 1052262000000,
"verify_keys": {
"ed25519:auto2": {
"key": "Base+64+Encoded+Signature+Verification+Key"
}
}
}
Querying Keys Through Another Server Querying Keys Through Another Server
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@ -232,38 +191,7 @@ This API can return keys for servers that are offline by using cached responses
taken from when the server was online. Keys can be queried from multiple taken from when the server was online. Keys can be queried from multiple
servers to mitigate against DNS spoofing. servers to mitigate against DNS spoofing.
Example request: {{keys_query_ss_http_api}}
.. code::
GET /_matrix/key/v2/query/{server_name}/{key_id}/?minimum_valid_until_ts={minimum_valid_until_ts} HTTP/1.1
POST /_matrix/key/v2/query HTTP/1.1
Content-Type: application/json
{
"server_keys": {
"{server_name}": {
"{key_id}": {
"minimum_valid_until_ts": {posix_timestamp}
}
}
}
}
Response:
.. code::
HTTP/1.1 200 OK
Content-Type: application/json
{
"server_keys": [
# List of responses with same format as /_matrix/key/v2/server
# signed by both the originating server and this server.
]
}
Version 1 Version 1
+++++++++ +++++++++
@ -1061,38 +989,9 @@ If the invited homeserver is in the room the invite came from, it can auth the
event and send it. event and send it.
However, if the invited homeserver isn't in the room the invite came from, it However, if the invited homeserver isn't in the room the invite came from, it
will need to request the room's homeserver to auth the event:: will need to request the room's homeserver to auth the event.
PUT .../exchange_third_party_invite/{roomId}
Where ``roomId`` is the ID of the room the invite is for.
The required fields in the JSON body are:
==================== ======= ==================================================
Key Type Description
==================== ======= ==================================================
``type`` String The event type. Must be ``m.room.member``.
``room_id`` String The ID of the room the event is for. Must be the
same as the ID specified in the path.
``sender`` String The Matrix ID of the user who sent the original
``m.room.third_party_invite``.
``state_key`` String The Matrix ID of the invited user.
``content`` Object The content of the event.
==================== ======= ==================================================
Where the ``content`` key contains the content for the ``m.room.member`` event
as described in the `Client-Server API`_. Its ``membership`` key must be
``invite`` and its content must include the ``third_party_invite`` object.
The inviting homeserver will then be able to authenticate the event. It will send
a fully authenticated event to the invited homeserver as described in the `Inviting
to a room`_ section above.
Once the invited homeserver responds with the event to which it appended its {{third_party_invite_ss_http_api}}
signature, the inviting homeserver will respond with ``200 OK`` and an empty body
(``{}``) to the initial request on ``/exchange_third_party_invite/{roomId}`` and
send the now verified ``m.room.member`` invite event to the room's members.
Verifying the invite Verifying the invite
++++++++++++++++++++ ++++++++++++++++++++

@ -2,9 +2,6 @@ targets:
index: index:
files: files:
- index.rst - index.rst
intro:
files:
- intro.rst
client_server: client_server:
files: files:
- client_server_api.rst - client_server_api.rst

Loading…
Cancel
Save