Merge branch 'master' into erikj/public_rooms

pull/977/head
Richard van der Hoff 8 years ago
commit dca1d0c745

@ -0,0 +1,44 @@
# Copyright 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
type: object
description: A client device
title: Device
properties:
device_id:
type: string
description: Identifier of this device.
example: QBUAZIFURK
display_name:
type: string
description: |-
Display name set by the user for this device. Absent if no name has been
set.
example: android
last_seen_ip:
type: string
description: |-
The IP address where this device was last seen. (May be a few minutes out
of date, for efficiency reasons).
example: 1.2.3.4
last_seen_ts:
type: integer
format: int64
description: |-
The timestamp (in milliseconds since the unix epoch) when this devices
was last seen. (May be a few minutes out of date, for efficiency
reasons).
example: 1474491775024
required:
- device_id

@ -0,0 +1,177 @@
# Copyright 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
swagger: '2.0'
info:
title: "Matrix Client-Server device management API"
version: "1.0.0"
host: localhost:8008
schemes:
- https
- http
basePath: /_matrix/client/%CLIENT_MAJOR_VERSION%
consumes:
- application/json
produces:
- application/json
securityDefinitions:
$ref: definitions/security.yaml
paths:
"/devices":
get:
summary: List registered devices for the current user
description: |-
Gets information about all devices for the current user.
security:
- accessToken: []
responses:
200:
description: Device information
schema:
type: object
properties:
devices:
type: array
description: A list of all registered devices for this user.
items:
type: object
allOf:
- $ref: "definitions/client_device.yaml"
examples:
application/json: |-
{
"devices": [
{
"device_id": "QBUAZIFURK",
"display_name": "android",
"last_seen_ip": "1.2.3.4",
"last_seen_ts": 1474491775024
}
]
}
tags:
- Device management
"/devices/{deviceId}":
get:
summary: Get a single device
description: |-
Gets information on a single device, by device id.
security:
- accessToken: []
parameters:
- in: path
type: string
name: deviceId
description: The device to retrieve.
required: true
x-example: "QBUAZIFURK"
responses:
200:
description: Device information
schema:
type: object
allOf:
- $ref: "definitions/client_device.yaml"
examples:
application/json: |-
{
"device_id": "QBUAZIFURK",
"display_name": "android",
"last_seen_ip": "1.2.3.4",
"last_seen_ts": 1474491775024
}
404:
description: The current user has no device with the given ID.
tags:
- Device management
put:
summary: Update a device
description: |-
Updates the metadata on the given device.
security:
- accessToken: []
parameters:
- in: path
type: string
name: deviceId
description: The device to update.
required: true
x-example: "QBUAZIFURK"
- in: body
name: body
required: true
description: New information for the device.
schema:
type: object
properties:
display_name:
type: string
description: |-
The new display name for this device. If not given, the
display name is unchanged.
example: My other phone
responses:
200:
description: The device was successfully updated.
examples:
application/json: |-
{}
schema:
type: object # empty json object
404:
description: The current user has no device with the given ID.
tags:
- Device management
delete:
summary: Delete a device
description: |-
This API endpoint uses the `User-Interactive Authentication API`_.
Deletes the given device, and invalidates any access token assoicated with it.
security:
- accessToken: []
parameters:
- in: path
type: string
name: deviceId
description: The device to delete.
required: true
x-example: "QBUAZIFURK"
- in: body
name: body
schema:
type: object
properties:
auth:
description: |-
Additional authentication information for the
user-interactive authentication API.
"$ref": "definitions/auth_data.yaml"
responses:
200:
description: |-
The device was successfully removed, or had been removed
previously.
schema:
type: object
examples:
application/json: |-
{}
401:
description: |-
The homeserver requires additional authentication information.
schema:
"$ref": "definitions/auth_response.yaml"
tags:
- Device management

@ -33,6 +33,14 @@ paths:
description: |- description: |-
Authenticates the user, and issues an access token they can Authenticates the user, and issues an access token they can
use to authorize themself in subsequent requests. use to authorize themself in subsequent requests.
If the client does not supply a ``device_id``, the server must
auto-generate one.
The returned access token must be associated with the ``device_id``
supplied by the client or generated by the server. The server may
invalidate any access token previously associated with that device. See
`Relationship between access tokens and devices`_.
parameters: parameters:
- in: body - in: body
name: body name: body
@ -42,7 +50,8 @@ paths:
{ {
"type": "m.login.password", "type": "m.login.password",
"user": "cheeky_monkey", "user": "cheeky_monkey",
"password": "ilovebananas" "password": "ilovebananas",
"initial_device_display_name": "Jungle Phone"
} }
properties: properties:
type: type:
@ -67,6 +76,17 @@ paths:
type: string type: string
description: |- description: |-
Required when ``type`` is ``m.login.token``. The login token. Required when ``type`` is ``m.login.token``. The login token.
device_id:
type: string
description: |-
ID of the client device. If this does not correspond to a
known client device, a new device will be created. The server
will auto-generate a device_id if this is not specified.
initial_device_display_name:
type: string
description: |-
A display name to assign to the newly-created device. Ignored
if ``device_id`` corresponds to a known device.
required: ["type"] required: ["type"]
responses: responses:
@ -77,7 +97,8 @@ paths:
{ {
"user_id": "@cheeky_monkey:matrix.org", "user_id": "@cheeky_monkey:matrix.org",
"access_token": "abc123", "access_token": "abc123",
"home_server": "matrix.org" "home_server": "matrix.org",
"device_id": "GHTYAJCE"
} }
schema: schema:
type: object type: object
@ -93,6 +114,11 @@ paths:
home_server: home_server:
type: string type: string
description: The hostname of the homeserver on which the account has been registered. description: The hostname of the homeserver on which the account has been registered.
device_id:
type: string
description: |-
ID of the logged-in device. Will be the same as the
corresponding parameter in the request, if one was specified.
400: 400:
description: |- description: |-
Part of the request was invalid. For example, the login type may not be recognised. Part of the request was invalid. For example, the login type may not be recognised.

@ -42,6 +42,13 @@ paths:
If registration is successful, this endpoint will issue an access token If registration is successful, this endpoint will issue an access token
the client can use to authorize itself in subsequent requests. the client can use to authorize itself in subsequent requests.
If the client does not supply a ``device_id``, the server must
auto-generate one.
The returned access token must be associated with the ``device_id``
supplied by the client or generated by the server. The server may
invalidate any access token previously associated with that device. See
`Relationship between access tokens and devices`_.
parameters: parameters:
- in: query - in: query
name: kind name: kind
@ -84,7 +91,19 @@ paths:
type: string type: string
description: The desired password for the account. description: The desired password for the account.
example: ilovebananas example: ilovebananas
required: ["password"] device_id:
type: string
description: |-
ID of the client device. If this does not correspond to a
known client device, a new device will be created. The server
will auto-generate a device_id if this is not specified.
example: GHTYAJCE
initial_device_display_name:
type: string
description: |-
A display name to assign to the newly-created device. Ignored
if ``device_id`` corresponds to a known device.
example: Jungle Phone
responses: responses:
200: 200:
description: The account has been registered. description: The account has been registered.
@ -93,7 +112,8 @@ paths:
{ {
"user_id": "@cheeky_monkey:matrix.org", "user_id": "@cheeky_monkey:matrix.org",
"access_token": "abc123", "access_token": "abc123",
"home_server": "matrix.org" "home_server": "matrix.org",
"device_id": "GHTYAJCE"
} }
schema: schema:
type: object type: object
@ -109,6 +129,11 @@ paths:
home_server: home_server:
type: string type: string
description: The hostname of the homeserver on which the account has been registered. description: The hostname of the homeserver on which the account has been registered.
device_id:
type: string
description: |-
ID of the registered device. Will be the same as the
corresponding parameter in the request, if one was specified.
400: 400:
description: |- description: |-
Part of the request was invalid. This may include one of the following error codes: Part of the request was invalid. This may include one of the following error codes:

@ -7,11 +7,11 @@
`underride` to `override`. This works with all known clients `underride` to `override`. This works with all known clients
which support push rules, but any other clients implementing which support push rules, but any other clients implementing
the push rules API should be aware of this change. This the push rules API should be aware of this change. This
makes it simple to mute rooms correctly in the API. makes it simple to mute rooms correctly in the API
(`#373 <https://github.com/matrix-org/matrix-doc/pull/373>`_). (`#373 <https://github.com/matrix-org/matrix-doc/pull/373>`_).
- Remove ``/tokenrefresh`` from the API. - Remove ``/tokenrefresh`` from the API
(`#395 <https://github.com/matrix-org/matrix-doc/pull/395>`_). (`#395 <https://github.com/matrix-org/matrix-doc/pull/395>`_).
- Remove requirement that tokens used in token-based login be macaroons. - Remove requirement that tokens used in token-based login be macaroons
(`#395 <https://github.com/matrix-org/matrix-doc/pull/395>`_). (`#395 <https://github.com/matrix-org/matrix-doc/pull/395>`_).
- Changes to the API which will be backwards-compatible for clients: - Changes to the API which will be backwards-compatible for clients:
@ -25,15 +25,17 @@
- Add top-level ``account_data`` key to the responses to ``GET /sync`` and - Add top-level ``account_data`` key to the responses to ``GET /sync`` and
``GET /initialSync`` ``GET /initialSync``
(`#380 <https://github.com/matrix-org/matrix-doc/pull/380>`_). (`#380 <https://github.com/matrix-org/matrix-doc/pull/380>`_).
- Add ``is_direct`` flag to |/createRoom|_ and invite member event. - Add ``is_direct`` flag to ``POST /createRoom`` and invite member event.
Add 'Direct Messaging' module. Add 'Direct Messaging' module
(`#389 <https://github.com/matrix-org/matrix-doc/pull/389>`_). (`#389 <https://github.com/matrix-org/matrix-doc/pull/389>`_).
- Add ``contains_url`` option to ``RoomEventFilter``. - Add ``contains_url`` option to ``RoomEventFilter``
(`#390 <https://github.com/matrix-org/matrix-doc/pull/390>`_). (`#390 <https://github.com/matrix-org/matrix-doc/pull/390>`_).
- Add ``filter`` optional query param to ``/messages`` - Add ``filter`` optional query param to ``/messages``
(`#390 <https://github.com/matrix-org/matrix-doc/pull/390>`_). (`#390 <https://github.com/matrix-org/matrix-doc/pull/390>`_).
- Add "Send-to-Device messaging" module - Add 'Send-to-Device messaging' module
(`#386 <https://github.com/matrix-org/matrix-doc/pull/386>`_). (`#386 <https://github.com/matrix-org/matrix-doc/pull/386>`_).
- Add 'Device management' module
(`#402 <https://github.com/matrix-org/matrix-doc/pull/402>`_).
- Require that User-Interactive auth fallback pages call - Require that User-Interactive auth fallback pages call
``window.postMessage`` to notify apps of completion ``window.postMessage`` to notify apps of completion
(`#398 <https://github.com/matrix-org/matrix-doc/pull/398>`_). (`#398 <https://github.com/matrix-org/matrix-doc/pull/398>`_).

@ -3,12 +3,14 @@
"content": { "content": {
"body": "Big Ben, London, UK", "body": "Big Ben, London, UK",
"geo_uri": "geo:51.5008,0.1247", "geo_uri": "geo:51.5008,0.1247",
"thumbnail_url": "mxc://localhost/FHyPlCeYUSFFxlgbQYZmoEoe", "info": {
"thumbnail_info": { "thumbnail_url": "mxc://localhost/FHyPlCeYUSFFxlgbQYZmoEoe",
"mimetype": "image/jpeg", "thumbnail_info": {
"size": 46144, "mimetype": "image/jpeg",
"w": 300, "size": 46144,
"h": 300 "w": 300,
"h": 300
}
}, },
"msgtype": "m.location" "msgtype": "m.location"
}, },

@ -4,13 +4,21 @@ properties:
h: h:
description: The height of the image in pixels. description: The height of the image in pixels.
type: integer type: integer
w:
description: The width of the image in pixels.
type: integer
mimetype: mimetype:
description: The mimetype of the image, e.g. ``image/jpeg``. description: The mimetype of the image, e.g. ``image/jpeg``.
type: string type: string
size: size:
description: Size of the image in bytes. description: Size of the image in bytes.
type: integer type: integer
w: thumbnail_url:
description: The width of the image in pixels. description: The URL to a thumbnail of the image.
type: integer type: string
thumbnail_info:
allOf:
- $ref: thumbnail_info.yaml
description: Metadata about the image referred to in ``thumbnail_url``.
title: ImageInfo title: ImageInfo
type: object

@ -0,0 +1,17 @@
$schema: http://json-schema.org/draft-04/schema#
description: Metadata about a thumbnail image.
properties:
h:
description: The height of the image in pixels.
type: integer
w:
description: The width of the image in pixels.
type: integer
mimetype:
description: The mimetype of the image, e.g. ``image/jpeg``.
type: string
size:
description: Size of the image in bytes.
type: integer
title: ThumbnailInfo
type: object

@ -12,6 +12,8 @@ properties:
additionalProperties: additionalProperties:
type: array type: array
title: User ID title: User ID
items:
type: string
type: object type: object
type: type:
enum: enum:

@ -9,17 +9,6 @@ properties:
allOf: allOf:
- $ref: core-event-schema/msgtype_infos/image_info.yaml - $ref: core-event-schema/msgtype_infos/image_info.yaml
description: Metadata about the image referred to in ``url``. description: Metadata about the image referred to in ``url``.
title: ImageInfo
type: object
thumbnail_info:
allOf:
- $ref: core-event-schema/msgtype_infos/image_info.yaml
description: Metadata about the image referred to in ``thumbnail_url``.
title: ImageInfo
type: object
thumbnail_url:
description: The URL to the thumbnail of the image.
type: string
url: url:
description: The URL to the image. description: The URL to the image.
type: string type: string

@ -20,21 +20,19 @@ properties:
size: size:
description: The size of the file in bytes. description: The size of the file in bytes.
type: integer type: integer
thumbnail_url:
description: The URL to the thumbnail of the file.
type: string
thumbnail_info:
allOf:
- $ref: core-event-schema/msgtype_infos/thumbnail_info.yaml
description: Metadata about the image referred to in ``thumbnail_url``.
title: FileInfo title: FileInfo
type: object type: object
msgtype: msgtype:
enum: enum:
- m.file - m.file
type: string type: string
thumbnail_info:
allOf:
- $ref: core-event-schema/msgtype_infos/image_info.yaml
description: Metadata about the image referred to in ``thumbnail_url``.
title: ImageInfo
type: object
thumbnail_url:
description: The URL to the thumbnail of the file.
type: string
url: url:
description: The URL to the file. description: The URL to the file.
type: string type: string

@ -9,35 +9,13 @@ properties:
description: "A textual representation of the image. This could be the alt text of the image, the filename of the image, or some kind of content description for accessibility e.g. 'image attachment'." description: "A textual representation of the image. This could be the alt text of the image, the filename of the image, or some kind of content description for accessibility e.g. 'image attachment'."
type: string type: string
info: info:
allOf:
- $ref: core-event-schema/msgtype_infos/image_info.yaml
description: Metadata about the image referred to in ``url``. description: Metadata about the image referred to in ``url``.
properties:
h:
description: The height of the image in pixels.
type: integer
mimetype:
description: 'The mimetype of the image, e.g. ``image/jpeg``.'
type: string
size:
description: Size of the image in bytes.
type: integer
w:
description: The width of the image in pixels.
type: integer
title: ImageInfo
type: object
msgtype: msgtype:
enum: enum:
- m.image - m.image
type: string type: string
thumbnail_info:
allOf:
- $ref: core-event-schema/msgtype_infos/image_info.yaml
description: Metadata about the image referred to in ``thumbnail_url``.
title: ImageInfo
type: object
thumbnail_url:
description: The URL to the thumbnail of the image.
type: string
url: url:
description: The URL to the image. description: The URL to the image.
type: string type: string

@ -15,14 +15,17 @@ properties:
enum: enum:
- m.location - m.location
type: string type: string
thumbnail_info: info:
allOf:
- $ref: core-event-schema/msgtype_infos/image_info.yaml
title: ImageInfo
type: object type: object
thumbnail_url: properties:
description: The URL to a thumbnail of the location being represented. thumbnail_url:
type: string description: The URL to a thumbnail of the location being represented.
type: string
thumbnail_info:
allOf:
- $ref: core-event-schema/msgtype_infos/thumbnail_info.yaml
description: Metadata about the image referred to in ``thumbnail_url``.
title: LocationInfo
required: required:
- msgtype - msgtype
- body - body

@ -17,23 +17,22 @@ properties:
h: h:
description: The height of the video in pixels. description: The height of the video in pixels.
type: integer type: integer
w:
description: The width of the video in pixels.
type: integer
mimetype: mimetype:
description: The mimetype of the video e.g. ``video/mp4``. description: The mimetype of the video e.g. ``video/mp4``.
type: string type: string
size: size:
description: The size of the video in bytes. description: The size of the video in bytes.
type: integer type: integer
thumbnail_info:
allOf:
- $ref: core-event-schema/msgtype_infos/image_info.yaml
title: ImageInfo
type: object
thumbnail_url: thumbnail_url:
description: The URL to a thumbnail of the video clip. description: The URL to an image thumbnail of the video clip.
type: string type: string
w: thumbnail_info:
description: The width of the video in pixels. allOf:
type: integer - $ref: core-event-schema/msgtype_infos/thumbnail_info.yaml
description: Metadata about the image referred to in ``thumbnail_url``.
title: VideoInfo title: VideoInfo
type: object type: object
msgtype: msgtype:

@ -0,0 +1,15 @@
<head>
<script>
window.onload = function() {
var url = new URL(window.location);
url.pathname += "api-docs.json";
var newLoc = "http://matrix.org/docs/api/client-server/?url=" + encodeURIComponent(url);
document.getElementById("apidocs").href = newLoc;
};
</script>
</head>
<body><ul>
<li><a id="apidocs">api docs</a></li>
<li><a href="index.html">spec</a></li>
</ul>
</body>

@ -84,10 +84,14 @@ func watchFS(ch chan struct{}, w *fsnotify.Watcher) {
} }
func makeWalker(base string, w *fsnotify.Watcher) filepath.WalkFunc { func makeWalker(base string, w *fsnotify.Watcher) filepath.WalkFunc {
return func(path string, _ os.FileInfo, err error) error { return func(path string, i os.FileInfo, err error) error {
if err != nil { if err != nil {
log.Fatalf("Error walking: %v", err) log.Fatalf("Error walking: %v", err)
} }
if !i.IsDir() {
// we set watches on directories, not files
return nil
}
rel, err := filepath.Rel(base, path) rel, err := filepath.Rel(base, path)
if err != nil { if err != nil {
@ -129,20 +133,26 @@ func serve(w http.ResponseWriter, req *http.Request) {
wg.Wait() wg.Wait()
wgMu.Unlock() wgMu.Unlock()
file := req.URL.Path
if file[0] == '/' {
file = file[1:]
}
if file == "" {
file = "index.html"
}
m := toServe.Load().(bytesOrErr) m := toServe.Load().(bytesOrErr)
if m.err != nil { if m.err != nil {
w.Header().Set("Content-Type", "text/plain") w.Header().Set("Content-Type", "text/plain")
w.Write([]byte(m.err.Error())) w.Write([]byte(m.err.Error()))
return return
} }
b, ok := m.bytes[file]
ok := true
var b []byte
file := req.URL.Path
if file[0] == '/' {
file = file[1:]
}
b, ok = m.bytes[file]
if ok && file == "api-docs.json" {
w.Header().Set("Access-Control-Allow-Origin", "*")
}
if ok { if ok {
w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Type", "text/html")
w.Write([]byte(b)) w.Write([]byte(b))
@ -153,18 +163,23 @@ func serve(w http.ResponseWriter, req *http.Request) {
w.Write([]byte("Not found")) w.Write([]byte("Not found"))
} }
func populateOnce(dir string) { func generate(dir string) (map[string][]byte, error) {
defer wg.Done()
mu.Lock()
defer mu.Unlock()
cmd := exec.Command("python", "gendoc.py") cmd := exec.Command("python", "gendoc.py")
cmd.Dir = path.Join(dir, "scripts") cmd.Dir = path.Join(dir, "scripts")
var b bytes.Buffer var b bytes.Buffer
cmd.Stderr = &b cmd.Stderr = &b
err := cmd.Run() err := cmd.Run()
if err != nil { if err != nil {
toServe.Store(bytesOrErr{nil, fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())}) return nil, fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())
return }
// cheekily dump the swagger docs into the gen directory so that it is
// easy to serve
cmd = exec.Command("python", "dump-swagger.py", "gen/api-docs.json")
cmd.Dir = path.Join(dir, "scripts")
cmd.Stderr = &b
if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("error generating api docs: %v\nOutput from dump-swagger:\n%v", err, b.String())
} }
files := make(map[string][]byte) files := make(map[string][]byte)
@ -190,12 +205,28 @@ func populateOnce(dir string) {
return nil return nil
} }
err = filepath.Walk(base, walker) if err := filepath.Walk(base, walker); err != nil {
return nil, fmt.Errorf("error reading spec: %v", err)
}
// load the special index
indexpath := path.Join(dir, "scripts", "continuserv", "index.html")
bytes, err := ioutil.ReadFile(indexpath)
if err != nil { if err != nil {
toServe.Store(bytesOrErr{nil, fmt.Errorf("error reading spec: %v", err)}) return nil, fmt.Errorf("error reading index: %v", err)
return
} }
toServe.Store(bytesOrErr{files, nil}) files[""] = bytes
return files, nil
}
func populateOnce(dir string) {
defer wg.Done()
mu.Lock()
defer mu.Unlock()
files, err := generate(dir)
toServe.Store(bytesOrErr{files, err})
} }
func doPopulate(ch chan struct{}, dir string) { func doPopulate(ch chan struct{}, dir string) {

@ -63,7 +63,8 @@ output = {
"basePath": "/", "basePath": "/",
"consumes": ["application/json"], "consumes": ["application/json"],
"produces": ["application/json"], "produces": ["application/json"],
"host": "localhost:8008", "host": "matrix.org:8448",
"schemes": ["https"],
"info": { "info": {
"title": "Matrix Client-Server API", "title": "Matrix Client-Server API",
"version": release_label, "version": release_label,

@ -15,287 +15,5 @@
Appendices Appendices
========== ==========
Security Threat Model .. contents:: Table of Contents
---------------------- .. sectnum::
Denial of Service
~~~~~~~~~~~~~~~~~
The attacker could attempt to prevent delivery of messages to or from the
victim in order to:
* Disrupt service or marketing campaign of a commercial competitor.
* Censor a discussion or censor a participant in a discussion.
* Perform general vandalism.
Threat: Resource Exhaustion
+++++++++++++++++++++++++++
An attacker could cause the victims server to exhaust a particular resource
(e.g. open TCP connections, CPU, memory, disk storage)
Threat: Unrecoverable Consistency Violations
++++++++++++++++++++++++++++++++++++++++++++
An attacker could send messages which created an unrecoverable "split-brain"
state in the cluster such that the victim's servers could no longer derive a
consistent view of the chatroom state.
Threat: Bad History
+++++++++++++++++++
An attacker could convince the victim to accept invalid messages which the
victim would then include in their view of the chatroom history. Other servers
in the chatroom would reject the invalid messages and potentially reject the
victims messages as well since they depended on the invalid messages.
.. TODO-spec
Track trustworthiness of HS or users based on if they try to pretend they
haven't seen recent events, and fake a splitbrain... --M
Threat: Block Network Traffic
+++++++++++++++++++++++++++++
An attacker could try to firewall traffic between the victim's server and some
or all of the other servers in the chatroom.
Threat: High Volume of Messages
+++++++++++++++++++++++++++++++
An attacker could send large volumes of messages to a chatroom with the victim
making the chatroom unusable.
Threat: Banning users without necessary authorisation
+++++++++++++++++++++++++++++++++++++++++++++++++++++
An attacker could attempt to ban a user from a chatroom with the necessary
authorisation.
Spoofing
~~~~~~~~
An attacker could try to send a message claiming to be from the victim without
the victim having sent the message in order to:
* Impersonate the victim while performing illicit activity.
* Obtain privileges of the victim.
Threat: Altering Message Contents
+++++++++++++++++++++++++++++++++
An attacker could try to alter the contents of an existing message from the
victim.
Threat: Fake Message "origin" Field
+++++++++++++++++++++++++++++++++++
An attacker could try to send a new message purporting to be from the victim
with a phony "origin" field.
Spamming
~~~~~~~~
The attacker could try to send a high volume of solicited or unsolicited
messages to the victim in order to:
* Find victims for scams.
* Market unwanted products.
Threat: Unsolicited Messages
++++++++++++++++++++++++++++
An attacker could try to send messages to victims who do not wish to receive
them.
Threat: Abusive Messages
++++++++++++++++++++++++
An attacker could send abusive or threatening messages to the victim
Spying
~~~~~~
The attacker could try to access message contents or metadata for messages sent
by the victim or to the victim that were not intended to reach the attacker in
order to:
* Gain sensitive personal or commercial information.
* Impersonate the victim using credentials contained in the messages.
(e.g. password reset messages)
* Discover who the victim was talking to and when.
Threat: Disclosure during Transmission
++++++++++++++++++++++++++++++++++++++
An attacker could try to expose the message contents or metadata during
transmission between the servers.
Threat: Disclosure to Servers Outside Chatroom
++++++++++++++++++++++++++++++++++++++++++++++
An attacker could try to convince servers within a chatroom to send messages to
a server it controls that was not authorised to be within the chatroom.
Threat: Disclosure to Servers Within Chatroom
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
An attacker could take control of a server within a chatroom to expose message
contents or metadata for messages in that room.
Cryptographic Test Vectors
--------------------------
To assist in the development of compatible implementations, the following test
values may be useful for verifying the cryptographic event signing code.
Signing Key
~~~~~~~~~~~
The following test vectors all use the 32-byte value given by the following
Base64-encoded string as the seed for generating the ``ed25519`` signing key:
.. code::
SIGNING_KEY_SEED = decode_base64(
"YJDBA9Xnr2sVqXD9Vj7XVUnmFZcZrlw8Md7kMW+3XA1"
)
In each case, the server name and key ID are as follows:
.. code::
SERVER_NAME = "domain"
KEY_ID = "ed25519:1"
JSON Signing
~~~~~~~~~~~~
Given an empty JSON object:
.. code:: json
{}
The JSON signing algorithm should emit the following signed data:
.. code:: json
{
"signatures": {
"domain": {
"ed25519:1": "K8280/U9SSy9IVtjBuVeLr+HpOB4BQFWbg+UZaADMtTdGYI7Geitb76LTrr5QV/7Xg4ahLwYGYZzuHGZKM5ZAQ"
}
}
}
Given the following JSON object with data values in it:
.. code:: json
{
"one": 1,
"two": "Two"
}
The JSON signing algorithm should emit the following signed JSON:
.. code:: json
{
"one": 1,
"signatures": {
"domain": {
"ed25519:1": "KqmLSbO39/Bzb0QIYE82zqLwsA+PDzYIpIRA2sRQ4sL53+sN6/fpNSoqE7BP7vBZhG6kYdD13EIMJpvhJI+6Bw"
}
},
"two": "Two"
}
Event Signing
~~~~~~~~~~~~~
Given the following minimally-sized event:
.. code:: json
{
"event_id": "$0:domain",
"origin": "domain",
"origin_server_ts": 1000000,
"signatures": {},
"type": "X",
"unsigned": {
"age_ts": 1000000
}
}
The event signing algorithm should emit the following signed event:
.. code:: json
{
"event_id": "$0:domain",
"hashes": {
"sha256": "6tJjLpXtggfke8UxFhAKg82QVkJzvKOVOOSjUDK4ZSI"
},
"origin": "domain",
"origin_server_ts": 1000000,
"signatures": {
"domain": {
"ed25519:1": "2Wptgo4CwmLo/Y8B8qinxApKaCkBG2fjTWB7AbP5Uy+aIbygsSdLOFzvdDjww8zUVKCmI02eP9xtyJxc/cLiBA"
}
},
"type": "X",
"unsigned": {
"age_ts": 1000000
}
}
Given the following event containing redactable content:
.. code:: json
{
"content": {
"body": "Here is the message content",
},
"event_id": "$0:domain",
"origin": "domain",
"origin_server_ts": 1000000,
"type": "m.room.message",
"room_id": "!r:domain",
"sender": "@u:domain",
"signatures": {},
"unsigned": {
"age_ts": 1000000
}
}
The event signing algorithm should emit the following signed event:
.. code:: json
{
"content": {
"body": "Here is the message content",
},
"event_id": "$0:domain",
"hashes": {
"sha256": "onLKD1bGljeBWQhWZ1kaP9SorVmRQNdN5aM2JYU2n/g"
},
"origin": "domain",
"origin_server_ts": 1000000,
"type": "m.room.message",
"room_id": "!r:domain",
"sender": "@u:domain",
"signatures": {
"domain": {
"ed25519:1": "Wm+VzmOUOz08Ds+0NTWb1d4CZrVsJSikkeRxh6aCcUwu6pNC78FunoD7KNWzqFn241eYHYMGCA5McEiVPdhzBA"
}
},
"unsigned": {
"age_ts": 1000000
}
}

@ -0,0 +1,167 @@
.. Copyright 2016 OpenMarket Ltd
..
.. Licensed under the Apache License, Version 2.0 (the "License");
.. you may not use this file except in compliance with the License.
.. You may obtain a copy of the License at
..
.. http://www.apache.org/licenses/LICENSE-2.0
..
.. Unless required by applicable law or agreed to in writing, software
.. distributed under the License is distributed on an "AS IS" BASIS,
.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.. See the License for the specific language governing permissions and
.. limitations under the License.
Signing JSON
------------
Various points in the Matrix specification require JSON objects to be
cryptographically signed. This requires us to encode the JSON as a binary
string. Unfortunately the same JSON can be encoded in different ways by
changing how much white space is used or by changing the order of keys within
objects.
Signing an object therefore requires it to be encoded as a sequence of bytes
using `Canonical JSON`_, computing the signature for that sequence and then
adding the signature to the original JSON object.
Canonical JSON
~~~~~~~~~~~~~~
We define the canonical JSON encoding for a value to be the shortest UTF-8 JSON
encoding with dictionary keys lexicographically sorted by unicode codepoint.
Numbers in the JSON must be integers in the range ``[-(2**53)+1, (2**53)-1]``.
We pick UTF-8 as the encoding as it should be available to all platforms and
JSON received from the network is likely to be already encoded using UTF-8.
We sort the keys to give a consistent ordering. We force integers to be in the
range where they can be accurately represented using IEEE double precision
floating point numbers since a number of JSON libraries represent all numbers
using this representation.
.. code:: python
import json
def canonical_json(value):
return json.dumps(
value,
# Encode code-points outside of ASCII as UTF-8 rather than \u escapes
ensure_ascii=False,
# Remove unnecessary white space.
separators=(',',':'),
# Sort the keys of dictionaries.
sort_keys=True,
# Encode the resulting unicode as UTF-8 bytes.
).encode("UTF-8")
Grammar
+++++++
Adapted from the grammar in http://tools.ietf.org/html/rfc7159 removing
insignificant whitespace, fractions, exponents and redundant character escapes
.. code::
value = false / null / true / object / array / number / string
false = %x66.61.6c.73.65
null = %x6e.75.6c.6c
true = %x74.72.75.65
object = %x7B [ member *( %x2C member ) ] %7D
member = string %x3A value
array = %x5B [ value *( %x2C value ) ] %5B
number = [ %x2D ] int
int = %x30 / ( %x31-39 *digit )
digit = %x30-39
string = %x22 *char %x22
char = unescaped / %x5C escaped
unescaped = %x20-21 / %x23-5B / %x5D-10FFFF
escaped = %x22 ; " quotation mark U+0022
/ %x5C ; \ reverse solidus U+005C
/ %x62 ; b backspace U+0008
/ %x66 ; f form feed U+000C
/ %x6E ; n line feed U+000A
/ %x72 ; r carriage return U+000D
/ %x74 ; t tab U+0009
/ %x75.30.30.30 (%x30-37 / %x62 / %x65-66) ; u000X
/ %x75.30.30.31 (%x30-39 / %x61-66) ; u001X
Signing Details
~~~~~~~~~~~~~~~
JSON is signed by encoding the JSON object without ``signatures`` or keys grouped
as ``unsigned``, using the canonical encoding described above. The JSON bytes are then signed using the
signature algorithm and the signature is encoded using base64 with the padding
stripped. The resulting base64 signature is added to an object under the
*signing key identifier* which is added to the ``signatures`` object under the
name of the entity signing it which is added back to the original JSON object
along with the ``unsigned`` object.
The *signing key identifier* is the concatenation of the *signing algorithm*
and a *key identifier*. The *signing algorithm* identifies the algorithm used
to sign the JSON. The currently supported value for *signing algorithm* is
``ed25519`` as implemented by NACL (http://nacl.cr.yp.to/). The *key identifier*
is used to distinguish between different signing keys used by the same entity.
The ``unsigned`` object and the ``signatures`` object are not covered by the
signature. Therefore intermediate entities can add unsigned data such as
timestamps and additional signatures.
.. code:: json
{
"name": "example.org",
"signing_keys": {
"ed25519:1": "XSl0kuyvrXNj6A+7/tkrB9sxSbRi08Of5uRhxOqZtEQ"
},
"unsigned": {
"age_ts": 922834800000
},
"signatures": {
"example.org": {
"ed25519:1": "s76RUgajp8w172am0zQb/iPTHsRnb4SkrzGoeCOSFfcBY2V/1c8QfrmdXHpvnc2jK5BD1WiJIxiMW95fMjK7Bw"
}
}
}
.. code:: python
def sign_json(json_object, signing_key, signing_name):
signatures = json_object.pop("signatures", {})
unsigned = json_object.pop("unsigned", None)
signed = signing_key.sign(encode_canonical_json(json_object))
signature_base64 = encode_base64(signed.signature)
key_id = "%s:%s" % (signing_key.alg, signing_key.version)
signatures.setdefault(signing_name, {})[key_id] = signature_base64
json_object["signatures"] = signatures
if unsigned is not None:
json_object["unsigned"] = unsigned
return json_object
Checking for a Signature
~~~~~~~~~~~~~~~~~~~~~~~~
To check if an entity has signed a JSON object an implementation does the
following:
1. Checks if the ``signatures`` member of the object contains an entry with
the name of the entity. If the entry is missing then the check fails.
2. Removes any *signing key identifiers* from the entry with algorithms it
doesn't understand. If there are no *signing key identifiers* left then the
check fails.
3. Looks up *verification keys* for the remaining *signing key identifiers*
either from a local cache or by consulting a trusted key server. If it
cannot find a *verification key* then the check fails.
4. Decodes the base64 encoded signature bytes. If base64 decoding fails then
the check fails.
5. Removes the ``signatures`` and ``unsigned`` members of the object.
6. Encodes the remainder of the JSON object using the `Canonical JSON`_
encoding.
7. Checks the signature bytes against the encoded object using the
*verification key*. If this fails then the check fails. Otherwise the check
succeeds.

@ -0,0 +1,171 @@
.. Copyright 2015 OpenMarket Ltd
..
.. Licensed under the Apache License, Version 2.0 (the "License");
.. you may not use this file except in compliance with the License.
.. You may obtain a copy of the License at
..
.. http://www.apache.org/licenses/LICENSE-2.0
..
.. Unless required by applicable law or agreed to in writing, software
.. distributed under the License is distributed on an "AS IS" BASIS,
.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.. See the License for the specific language governing permissions and
.. limitations under the License.
Cryptographic Test Vectors
--------------------------
To assist in the development of compatible implementations, the following test
values may be useful for verifying the cryptographic event signing code.
Signing Key
~~~~~~~~~~~
The following test vectors all use the 32-byte value given by the following
Base64-encoded string as the seed for generating the ``ed25519`` signing key:
.. code::
SIGNING_KEY_SEED = decode_base64(
"YJDBA9Xnr2sVqXD9Vj7XVUnmFZcZrlw8Md7kMW+3XA1"
)
In each case, the server name and key ID are as follows:
.. code::
SERVER_NAME = "domain"
KEY_ID = "ed25519:1"
JSON Signing
~~~~~~~~~~~~
Given an empty JSON object:
.. code:: json
{}
The JSON signing algorithm should emit the following signed data:
.. code:: json
{
"signatures": {
"domain": {
"ed25519:1": "K8280/U9SSy9IVtjBuVeLr+HpOB4BQFWbg+UZaADMtTdGYI7Geitb76LTrr5QV/7Xg4ahLwYGYZzuHGZKM5ZAQ"
}
}
}
Given the following JSON object with data values in it:
.. code:: json
{
"one": 1,
"two": "Two"
}
The JSON signing algorithm should emit the following signed JSON:
.. code:: json
{
"one": 1,
"signatures": {
"domain": {
"ed25519:1": "KqmLSbO39/Bzb0QIYE82zqLwsA+PDzYIpIRA2sRQ4sL53+sN6/fpNSoqE7BP7vBZhG6kYdD13EIMJpvhJI+6Bw"
}
},
"two": "Two"
}
Event Signing
~~~~~~~~~~~~~
Given the following minimally-sized event:
.. code:: json
{
"event_id": "$0:domain",
"origin": "domain",
"origin_server_ts": 1000000,
"signatures": {},
"type": "X",
"unsigned": {
"age_ts": 1000000
}
}
The event signing algorithm should emit the following signed event:
.. code:: json
{
"event_id": "$0:domain",
"hashes": {
"sha256": "6tJjLpXtggfke8UxFhAKg82QVkJzvKOVOOSjUDK4ZSI"
},
"origin": "domain",
"origin_server_ts": 1000000,
"signatures": {
"domain": {
"ed25519:1": "2Wptgo4CwmLo/Y8B8qinxApKaCkBG2fjTWB7AbP5Uy+aIbygsSdLOFzvdDjww8zUVKCmI02eP9xtyJxc/cLiBA"
}
},
"type": "X",
"unsigned": {
"age_ts": 1000000
}
}
Given the following event containing redactable content:
.. code:: json
{
"content": {
"body": "Here is the message content",
},
"event_id": "$0:domain",
"origin": "domain",
"origin_server_ts": 1000000,
"type": "m.room.message",
"room_id": "!r:domain",
"sender": "@u:domain",
"signatures": {},
"unsigned": {
"age_ts": 1000000
}
}
The event signing algorithm should emit the following signed event:
.. code:: json
{
"content": {
"body": "Here is the message content",
},
"event_id": "$0:domain",
"hashes": {
"sha256": "onLKD1bGljeBWQhWZ1kaP9SorVmRQNdN5aM2JYU2n/g"
},
"origin": "domain",
"origin_server_ts": 1000000,
"type": "m.room.message",
"room_id": "!r:domain",
"sender": "@u:domain",
"signatures": {
"domain": {
"ed25519:1": "Wm+VzmOUOz08Ds+0NTWb1d4CZrVsJSikkeRxh6aCcUwu6pNC78FunoD7KNWzqFn241eYHYMGCA5McEiVPdhzBA"
}
},
"unsigned": {
"age_ts": 1000000
}
}

@ -0,0 +1,140 @@
.. Copyright 2015 OpenMarket Ltd
..
.. Licensed under the Apache License, Version 2.0 (the "License");
.. you may not use this file except in compliance with the License.
.. You may obtain a copy of the License at
..
.. http://www.apache.org/licenses/LICENSE-2.0
..
.. Unless required by applicable law or agreed to in writing, software
.. distributed under the License is distributed on an "AS IS" BASIS,
.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.. See the License for the specific language governing permissions and
.. limitations under the License.
Security Threat Model
----------------------
Denial of Service
~~~~~~~~~~~~~~~~~
The attacker could attempt to prevent delivery of messages to or from the
victim in order to:
* Disrupt service or marketing campaign of a commercial competitor.
* Censor a discussion or censor a participant in a discussion.
* Perform general vandalism.
Threat: Resource Exhaustion
+++++++++++++++++++++++++++
An attacker could cause the victims server to exhaust a particular resource
(e.g. open TCP connections, CPU, memory, disk storage)
Threat: Unrecoverable Consistency Violations
++++++++++++++++++++++++++++++++++++++++++++
An attacker could send messages which created an unrecoverable "split-brain"
state in the cluster such that the victim's servers could no longer derive a
consistent view of the chatroom state.
Threat: Bad History
+++++++++++++++++++
An attacker could convince the victim to accept invalid messages which the
victim would then include in their view of the chatroom history. Other servers
in the chatroom would reject the invalid messages and potentially reject the
victims messages as well since they depended on the invalid messages.
.. TODO-spec
Track trustworthiness of HS or users based on if they try to pretend they
haven't seen recent events, and fake a splitbrain... --M
Threat: Block Network Traffic
+++++++++++++++++++++++++++++
An attacker could try to firewall traffic between the victim's server and some
or all of the other servers in the chatroom.
Threat: High Volume of Messages
+++++++++++++++++++++++++++++++
An attacker could send large volumes of messages to a chatroom with the victim
making the chatroom unusable.
Threat: Banning users without necessary authorisation
+++++++++++++++++++++++++++++++++++++++++++++++++++++
An attacker could attempt to ban a user from a chatroom with the necessary
authorisation.
Spoofing
~~~~~~~~
An attacker could try to send a message claiming to be from the victim without
the victim having sent the message in order to:
* Impersonate the victim while performing illicit activity.
* Obtain privileges of the victim.
Threat: Altering Message Contents
+++++++++++++++++++++++++++++++++
An attacker could try to alter the contents of an existing message from the
victim.
Threat: Fake Message "origin" Field
+++++++++++++++++++++++++++++++++++
An attacker could try to send a new message purporting to be from the victim
with a phony "origin" field.
Spamming
~~~~~~~~
The attacker could try to send a high volume of solicited or unsolicited
messages to the victim in order to:
* Find victims for scams.
* Market unwanted products.
Threat: Unsolicited Messages
++++++++++++++++++++++++++++
An attacker could try to send messages to victims who do not wish to receive
them.
Threat: Abusive Messages
++++++++++++++++++++++++
An attacker could send abusive or threatening messages to the victim
Spying
~~~~~~
The attacker could try to access message contents or metadata for messages sent
by the victim or to the victim that were not intended to reach the attacker in
order to:
* Gain sensitive personal or commercial information.
* Impersonate the victim using credentials contained in the messages.
(e.g. password reset messages)
* Discover who the victim was talking to and when.
Threat: Disclosure during Transmission
++++++++++++++++++++++++++++++++++++++
An attacker could try to expose the message contents or metadata during
transmission between the servers.
Threat: Disclosure to Servers Outside Chatroom
++++++++++++++++++++++++++++++++++++++++++++++
An attacker could try to convince servers within a chatroom to send messages to
a server it controls that was not authorised to be within the chatroom.
Threat: Disclosure to Servers Within Chatroom
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
An attacker could take control of a server within a chatroom to expose message
contents or metadata for messages in that room.

@ -45,7 +45,7 @@ Other versions of this specification
The following other versions are also available, in reverse chronological order: The following other versions are also available, in reverse chronological order:
- `HEAD <https://matrix.org/speculator/spec/head/client_server.html>`_: Includes all changes since the latest versioned release. - `HEAD <http://matrix.org/speculator/spec/HEAD/client_server/unstable.html>`_: Includes all changes since the latest versioned release.
- `r0.2.0 <https://matrix.org/docs/spec/client_server/r0.2.0.html>`_ - `r0.2.0 <https://matrix.org/docs/spec/client_server/r0.2.0.html>`_
- `r0.1.0 <https://matrix.org/docs/spec/client_server/r0.1.0.html>`_ - `r0.1.0 <https://matrix.org/docs/spec/client_server/r0.1.0.html>`_
- `r0.0.1 <https://matrix.org/docs/spec/r0.0.1/client_server.html>`_ - `r0.0.1 <https://matrix.org/docs/spec/r0.0.1/client_server.html>`_
@ -72,6 +72,16 @@ MUST be encoded as UTF-8. Clients are authenticated using opaque
``access_token`` strings (see `Client Authentication`_ for details), passed as a ``access_token`` strings (see `Client Authentication`_ for details), passed as a
query string parameter on all requests. query string parameter on all requests.
The names of the API endponts for the HTTP transport follow a convention of
using underscores to separate words (for example ``/delete_devices``). The key
names in JSON objects passed over the API also follow this convention.
.. NOTE::
There are a few historical exceptions to this rule, such as
``/createRoom``. A future version of this specification will address the
inconsistency.
Any errors which occur at the Matrix API level MUST return a "standard error Any errors which occur at the Matrix API level MUST return a "standard error
response". This is a JSON object which looks like: response". This is a JSON object which looks like:
@ -175,6 +185,21 @@ return with a status of 401 and the error code, ``M_MISSING_TOKEN`` or
to choose an appropriate format. Server implementors may like to investigate to choose an appropriate format. Server implementors may like to investigate
`macaroons <macaroon_>`_. `macaroons <macaroon_>`_.
Relationship between access tokens and devices
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Client `devices`_ are closely related to access tokens. Matrix servers should
record which device each access token is assigned to, so that subsequent
requests can be handled correctly.
By default, the `Login`_ and `Registration`_ processes auto-generate a new
``device_id``. A client is also free to generate its own ``device_id`` or,
provided the user remains the same, reuse a device: in ether case the client
should pass the ``device_id`` in the request body. If the client sets the
``device_id``, the server will invalidate any access token previously assigned
to that device. There is therefore at most one active access token assigned to
each device at any one time.
User-Interactive Authentication API User-Interactive Authentication API
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -1356,6 +1381,7 @@ have to wait in milliseconds before they can try again.
.. References .. References
.. _`macaroon`: http://research.google.com/pubs/pub41892.html .. _`macaroon`: http://research.google.com/pubs/pub41892.html
.. _`devices`: ../intro.html#devices
.. Links through the external API docs are below .. Links through the external API docs are below
.. ============================================= .. =============================================

@ -1,306 +0,0 @@
.. Copyright 2016 OpenMarket Ltd
..
.. Licensed under the Apache License, Version 2.0 (the "License");
.. you may not use this file except in compliance with the License.
.. You may obtain a copy of the License at
..
.. http://www.apache.org/licenses/LICENSE-2.0
..
.. Unless required by applicable law or agreed to in writing, software
.. distributed under the License is distributed on an "AS IS" BASIS,
.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.. See the License for the specific language governing permissions and
.. limitations under the License.
Signing Events
--------------
Canonical JSON
~~~~~~~~~~~~~~
Matrix events are represented using JSON objects. If we want to sign JSON
events we need to encode the JSON as a binary string. Unfortunately the same
JSON can be encoded in different ways by changing how much white space is used
or by changing the order of keys within objects. Therefore we have to define an
encoding which can be reproduced byte for byte by any JSON library.
We define the canonical JSON encoding for a value to be the shortest UTF-8 JSON
encoding with dictionary keys lexicographically sorted by unicode codepoint.
Numbers in the JSON must be integers in the range [-(2**53)+1, (2**53)-1].
We pick UTF-8 as the encoding as it should be available to all platforms and
JSON received from the network is likely to be already encoded using UTF-8.
We sort the keys to give a consistent ordering. We force integers to be in the
range where they can be accurately represented using IEEE double precision
floating point numbers since a number of JSON libraries represent all numbers
using this representation.
.. code:: python
import json
def canonical_json(value):
return json.dumps(
value,
# Encode code-points outside of ASCII as UTF-8 rather than \u escapes
ensure_ascii=False,
# Remove unnecessary white space.
separators=(',',':'),
# Sort the keys of dictionaries.
sort_keys=True,
# Encode the resulting unicode as UTF-8 bytes.
).encode("UTF-8")
Grammar
+++++++
Adapted from the grammar in http://tools.ietf.org/html/rfc7159 removing
insignificant whitespace, fractions, exponents and redundant character escapes
.. code::
value = false / null / true / object / array / number / string
false = %x66.61.6c.73.65
null = %x6e.75.6c.6c
true = %x74.72.75.65
object = %x7B [ member *( %x2C member ) ] %7D
member = string %x3A value
array = %x5B [ value *( %x2C value ) ] %5B
number = [ %x2D ] int
int = %x30 / ( %x31-39 *digit )
digit = %x30-39
string = %x22 *char %x22
char = unescaped / %x5C escaped
unescaped = %x20-21 / %x23-5B / %x5D-10FFFF
escaped = %x22 ; " quotation mark U+0022
/ %x5C ; \ reverse solidus U+005C
/ %x62 ; b backspace U+0008
/ %x66 ; f form feed U+000C
/ %x6E ; n line feed U+000A
/ %x72 ; r carriage return U+000D
/ %x74 ; t tab U+0009
/ %x75.30.30.30 (%x30-37 / %x62 / %x65-66) ; u000X
/ %x75.30.30.31 (%x30-39 / %x61-66) ; u001X
Signing JSON
~~~~~~~~~~~~
We can now sign a JSON object by encoding it as a sequence of bytes, computing
the signature for that sequence and then adding the signature to the original
JSON object.
Signing Details
+++++++++++++++
JSON is signed by encoding the JSON object without ``signatures`` or keys grouped
as ``unsigned``, using the canonical encoding described above. The JSON bytes are then signed using the
signature algorithm and the signature is encoded using base64 with the padding
stripped. The resulting base64 signature is added to an object under the
*signing key identifier* which is added to the ``signatures`` object under the
name of the server signing it which is added back to the original JSON object
along with the ``unsigned`` object.
The *signing key identifier* is the concatenation of the *signing algorithm*
and a *key version*. The *signing algorithm* identifies the algorithm used to
sign the JSON. The currently support value for *signing algorithm* is
``ed25519`` as implemented by NACL (http://nacl.cr.yp.to/). The *key version*
is used to distinguish between different signing keys used by the same entity.
The ``unsigned`` object and the ``signatures`` object are not covered by the
signature. Therefore intermediate servers can add unsigned data such as timestamps
and additional signatures.
.. code:: json
{
"name": "example.org",
"signing_keys": {
"ed25519:1": "XSl0kuyvrXNj6A+7/tkrB9sxSbRi08Of5uRhxOqZtEQ"
},
"unsigned": {
"age_ts": 922834800000
},
"signatures": {
"example.org": {
"ed25519:1": "s76RUgajp8w172am0zQb/iPTHsRnb4SkrzGoeCOSFfcBY2V/1c8QfrmdXHpvnc2jK5BD1WiJIxiMW95fMjK7Bw"
}
}
}
.. code:: python
def sign_json(json_object, signing_key, signing_name):
signatures = json_object.pop("signatures", {})
unsigned = json_object.pop("unsigned", None)
signed = signing_key.sign(encode_canonical_json(json_object))
signature_base64 = encode_base64(signed.signature)
key_id = "%s:%s" % (signing_key.alg, signing_key.version)
signatures.setdefault(signing_name, {})[key_id] = signature_base64
json_object["signatures"] = signatures
if unsigned is not None:
json_object["unsigned"] = unsigned
return json_object
Checking for a Signature
++++++++++++++++++++++++
To check if an entity has signed a JSON object a server does the following
1. Checks if the ``signatures`` object contains an entry with the name of the
entity. If the entry is missing then the check fails.
2. Removes any *signing key identifiers* from the entry with algorithms it
doesn't understand. If there are no *signing key identifiers* left then the
check fails.
3. Looks up *verification keys* for the remaining *signing key identifiers*
either from a local cache or by consulting a trusted key server. If it
cannot find a *verification key* then the check fails.
4. Decodes the base64 encoded signature bytes. If base64 decoding fails then
the check fails.
5. Checks the signature bytes using the *verification key*. If this fails then
the check fails. Otherwise the check succeeds.
Signing Events
~~~~~~~~~~~~~~
Signing events is a more complicated process since servers can choose to redact
non-essential parts of an event. Before signing the event it is encoded as
Canonical JSON and hashed using SHA-256. The resulting hash is then stored
in the event JSON in a ``hash`` object under a ``sha256`` key.
.. code:: python
def hash_event(event_json_object):
# Keys under "unsigned" can be modified by other servers.
# They are useful for conveying information like the age of an
# event that will change in transit.
# Since they can be modifed we need to exclude them from the hash.
unsigned = event_json_object.pop("unsigned", None)
# Signatures will depend on the current value of the "hashes" key.
# We cannot add new hashes without invalidating existing signatures.
signatures = event_json_object.pop("signatures", None)
# The "hashes" key might contain multiple algorithms if we decide to
# migrate away from SHA-2. We don't want to include an existing hash
# output in our hash so we exclude the "hashes" dict from the hash.
hashes = event_json_object.pop("hashes", {})
# Encode the JSON using a canonical encoding so that we get the same
# bytes on every server for the same JSON object.
event_json_bytes = encode_canonical_json(event_json_bytes)
# Add the base64 encoded bytes of the hash to the "hashes" dict.
hashes["sha256"] = encode_base64(sha256(event_json_bytes).digest())
# Add the "hashes" dict back the event JSON under a "hashes" key.
event_json_object["hashes"] = hashes
if unsigned is not None:
event_json_object["unsigned"] = unsigned
return event_json_object
The event is then stripped of all non-essential keys both at the top level and
within the ``content`` object. Any top-level keys not in the following list
MUST be removed:
.. code::
auth_events
depth
event_id
hashes
membership
origin
origin_server_ts
prev_events
prev_state
room_id
sender
signatures
state_key
type
A new ``content`` object is constructed for the resulting event that contains
only the essential keys of the original ``content`` object. If the original
event lacked a ``content`` object at all, a new empty JSON object is created
for it.
The keys that are considered essential for the ``content`` object depend on the
the ``type`` of the event. These are:
.. code::
type is "m.room.aliases":
aliases
type is "m.room.create":
creator
type is "m.room.history_visibility":
history_visibility
type is "m.room.join_rules":
join_rule
type is "m.room.member":
membership
type is "m.room.power_levels":
ban
events
events_default
kick
redact
state_default
users
users_default
The resulting stripped object with the new ``content`` object and the original
``hashes`` key is then signed using the JSON signing algorithm outlined below:
.. code:: python
def sign_event(event_json_object, name, key):
# Make sure the event has a "hashes" key.
if "hashes" not in event_json_object:
event_json_object = hash_event(event_json_object)
# Strip all the keys that would be removed if the event was redacted.
# The hashes are not stripped and cover all the keys in the event.
# This means that we can tell if any of the non-essential keys are
# modified or removed.
stripped_json_object = strip_non_essential_keys(event_json_object)
# Sign the stripped JSON object. The signature only covers the
# essential keys and the hashes. This means that we can check the
# signature even if the event is redacted.
signed_json_object = sign_json(stripped_json_object)
# Copy the signatures from the stripped event to the original event.
event_json_object["signatures"] = signed_json_oject["signatures"]
return event_json_object
Servers can then transmit the entire event or the event with the non-essential
keys removed. If the entire event is present, receiving servers can then check
the event by computing the SHA-256 of the event, excluding the ``hash`` object.
If the keys have been redacted, then the ``hash`` object is included when
calculating the SHA-256 instead.
New hash functions can be introduced by adding additional keys to the ``hash``
object. Since the ``hash`` object cannot be redacted a server shouldn't allow
too many hashes to be listed, otherwise a server might embed illict data within
the ``hash`` object. For similar reasons a server shouldn't allow hash values
that are too long.
.. TODO
[[TODO(markjh): We might want to specify a maximum number of keys for the
``hash`` and we might want to specify the maximum output size of a hash]]
[[TODO(markjh) We might want to allow the server to omit the output of well
known hash functions like SHA-256 when none of the keys have been redacted]]

@ -160,6 +160,35 @@ allocated the account and has the form::
See the `Identifier Grammar`_ section for full details of the structure of See the `Identifier Grammar`_ section for full details of the structure of
user IDs. user IDs.
Devices
~~~~~~~
The Matrix specification has a particular meaning for the term "device". As a
user, I might have several devices: a desktop client, some web browsers, an
Android device, an iPhone, etc. They broadly relate to a real device in the
physical world, but you might have several browsers on a physical device, or
several Matrix client applications on a mobile device, each of which would be
its own device.
Devices are used primarily to manage the keys used for end-to-end encryption
(each device gets its own copy of the decryption keys), but they also help
users manage their access - for instance, by revoking access to particular
devices.
When a user first uses a client, it registers itself as a new device. The
longevity of devices might depend on the type of client. A web client will
probably drop all of its state on logout, and create a new device every time
you log in, to ensure that cryptography keys are not leaked to a new user. In
a mobile client, it might be acceptable to reuse the device if a login session
expires, provided the user is the same.
Devices are identified by a ``device_id``, which is unique within the scope of
a given user.
A user may assign a human-readable display name to a device, to help them
manage their devices.
Events Events
~~~~~~ ~~~~~~

@ -0,0 +1,41 @@
.. Copyright 2016 OpenMarket Ltd
..
.. Licensed under the Apache License, Version 2.0 (the "License");
.. you may not use this file except in compliance with the License.
.. You may obtain a copy of the License at
..
.. http://www.apache.org/licenses/LICENSE-2.0
..
.. Unless required by applicable law or agreed to in writing, software
.. distributed under the License is distributed on an "AS IS" BASIS,
.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.. See the License for the specific language governing permissions and
.. limitations under the License.
Device Management
=================
.. _module:device-management:
This module provides a means for a user to manage their `devices`_.
Client behaviour
----------------
Clients that implement this module should offer the user a list of registered
devices, as well as the means to update their display names. Clients should
also allow users to delete disused devices.
{{device_management_cs_http_api}}
Security considerations
-----------------------
Deleting devices has security implications: it invalidates the access_token
assigned to the device, so an attacker could use it to log out the real user
(and do it repeatedly every time the real user tries to log in to block the
attacker). Servers should require additional authentication beyond the access
token when deleting devices (for example, requiring that the user resubmit
their password).
The display names of devices are publicly visible. Clients should consider
advising the user of this.

@ -71,6 +71,12 @@ Events which have attachments (e.g. ``m.image``, ``m.file``) SHOULD be
uploaded using the `content repository module`_ where available. The uploaded using the `content repository module`_ where available. The
resulting ``mxc://`` URI can then be used in the ``url`` key. resulting ``mxc://`` URI can then be used in the ``url`` key.
Clients MAY include a client generated thumbnail image for an attachment under
a ``info.thumbnail_url`` key. The thumbnail SHOULD also be a ``mxc://`` URI.
Clients displaying events with attachments can either use the client generated
thumbnail or ask its homeserver to generate a thumbnail from the original
attachment using the `content repository module`_.
.. _`content repository module`: `module:content`_ .. _`content repository module`: `module:content`_
Recommendations when sending messages Recommendations when sending messages

@ -545,14 +545,14 @@ Examples
To create a rule that suppresses notifications for the room with ID To create a rule that suppresses notifications for the room with ID
``!dj234r78wl45Gh4D:matrix.org``:: ``!dj234r78wl45Gh4D:matrix.org``::
curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/%CLIENT_MAJOR_VERSION%/pushrules/global/room/%21dj234r78wl45Gh4D%3Amatrix.org?access_token=123456" -d \ curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/%CLIENT_MAJOR_VERSION%/pushrules/global/room/%21dj234r78wl45Gh4D%3Amatrix.org?access_token=123456" -d \
'{ '{
"actions" : ["dont_notify"] "actions" : ["dont_notify"]
}' }'
To suppress notifications for the user ``@spambot:matrix.org``:: To suppress notifications for the user ``@spambot:matrix.org``::
curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/%CLIENT_MAJOR_VERSION%/pushrules/global/sender/%40spambot%3Amatrix.org?access_token=123456" -d \ curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/%CLIENT_MAJOR_VERSION%/pushrules/global/sender/%40spambot%3Amatrix.org?access_token=123456" -d \
'{ '{
"actions" : ["dont_notify"] "actions" : ["dont_notify"]
}' }'
@ -560,7 +560,7 @@ To suppress notifications for the user ``@spambot:matrix.org``::
To always notify for messages that contain the work 'cake' and set a specific To always notify for messages that contain the work 'cake' and set a specific
sound (with a rule_id of ``SSByZWFsbHkgbGlrZSBjYWtl``):: sound (with a rule_id of ``SSByZWFsbHkgbGlrZSBjYWtl``)::
curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/%CLIENT_MAJOR_VERSION%/pushrules/global/content/SSByZWFsbHkgbGlrZSBjYWtl?access_token=123456" -d \ curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/%CLIENT_MAJOR_VERSION%/pushrules/global/content/SSByZWFsbHkgbGlrZSBjYWtl?access_token=123456" -d \
'{ '{
"pattern": "cake", "pattern": "cake",
"actions" : ["notify", {"set_sound":"cakealarm.wav"}] "actions" : ["notify", {"set_sound":"cakealarm.wav"}]
@ -569,7 +569,7 @@ sound (with a rule_id of ``SSByZWFsbHkgbGlrZSBjYWtl``)::
To add a rule suppressing notifications for messages starting with 'cake' but To add a rule suppressing notifications for messages starting with 'cake' but
ending with 'lie', superseding the previous rule:: ending with 'lie', superseding the previous rule::
curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/%CLIENT_MAJOR_VERSION%/pushrules/global/content/U3BvbmdlIGNha2UgaXMgYmVzdA?access_token=123456&before=SSByZWFsbHkgbGlrZSBjYWtl" -d \ curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/%CLIENT_MAJOR_VERSION%/pushrules/global/content/U3BvbmdlIGNha2UgaXMgYmVzdA?access_token=123456&before=SSByZWFsbHkgbGlrZSBjYWtl" -d \
'{ '{
"pattern": "cake*lie", "pattern": "cake*lie",
"actions" : ["notify"] "actions" : ["notify"]
@ -579,7 +579,7 @@ To add a custom sound for notifications messages containing the word 'beer' in
any rooms with 10 members or fewer (with greater importance than the room, any rooms with 10 members or fewer (with greater importance than the room,
sender and content rules):: sender and content rules)::
curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/api/%CLIENT_MAJOR_VERSION%/pushrules/global/override/U2VlIHlvdSBpbiBUaGUgRHVrZQ?access_token=123456" -d \ curl -X PUT -H "Content-Type: application/json" "https://example.com/_matrix/client/%CLIENT_MAJOR_VERSION%/pushrules/global/override/U2VlIHlvdSBpbiBUaGUgRHVrZQ?access_token=123456" -d \
'{ '{
"conditions": [ "conditions": [
{"kind": "event_match", "key": "content.body", "pattern": "beer" }, {"kind": "event_match", "key": "content.body", "pattern": "beer" },

@ -998,3 +998,149 @@ the following EDU::
messages: The messages to send. A map from user ID, to a map from device ID messages: The messages to send. A map from user ID, to a map from device ID
to message body. The device ID may also be *, meaning all known devices to message body. The device ID may also be *, meaning all known devices
for the user. for the user.
Signing Events
--------------
Signing events is complicated by the fact that servers can choose to redact
non-essential parts of an event.
Before signing the event, the ``unsigned`` and ``signature`` members are
removed, it is encoded as `Canonical JSON`_, and then hashed using SHA-256. The
resulting hash is then stored in the event JSON in a ``hash`` object under a
``sha256`` key.
.. code:: python
def hash_event(event_json_object):
# Keys under "unsigned" can be modified by other servers.
# They are useful for conveying information like the age of an
# event that will change in transit.
# Since they can be modifed we need to exclude them from the hash.
unsigned = event_json_object.pop("unsigned", None)
# Signatures will depend on the current value of the "hashes" key.
# We cannot add new hashes without invalidating existing signatures.
signatures = event_json_object.pop("signatures", None)
# The "hashes" key might contain multiple algorithms if we decide to
# migrate away from SHA-2. We don't want to include an existing hash
# output in our hash so we exclude the "hashes" dict from the hash.
hashes = event_json_object.pop("hashes", {})
# Encode the JSON using a canonical encoding so that we get the same
# bytes on every server for the same JSON object.
event_json_bytes = encode_canonical_json(event_json_bytes)
# Add the base64 encoded bytes of the hash to the "hashes" dict.
hashes["sha256"] = encode_base64(sha256(event_json_bytes).digest())
# Add the "hashes" dict back the event JSON under a "hashes" key.
event_json_object["hashes"] = hashes
if unsigned is not None:
event_json_object["unsigned"] = unsigned
return event_json_object
The event is then stripped of all non-essential keys both at the top level and
within the ``content`` object. Any top-level keys not in the following list
MUST be removed:
.. code::
auth_events
depth
event_id
hashes
membership
origin
origin_server_ts
prev_events
prev_state
room_id
sender
signatures
state_key
type
A new ``content`` object is constructed for the resulting event that contains
only the essential keys of the original ``content`` object. If the original
event lacked a ``content`` object at all, a new empty JSON object is created
for it.
The keys that are considered essential for the ``content`` object depend on the
the ``type`` of the event. These are:
.. code::
type is "m.room.aliases":
aliases
type is "m.room.create":
creator
type is "m.room.history_visibility":
history_visibility
type is "m.room.join_rules":
join_rule
type is "m.room.member":
membership
type is "m.room.power_levels":
ban
events
events_default
kick
redact
state_default
users
users_default
The resulting stripped object with the new ``content`` object and the original
``hashes`` key is then signed using the JSON signing algorithm outlined below:
.. code:: python
def sign_event(event_json_object, name, key):
# Make sure the event has a "hashes" key.
if "hashes" not in event_json_object:
event_json_object = hash_event(event_json_object)
# Strip all the keys that would be removed if the event was redacted.
# The hashes are not stripped and cover all the keys in the event.
# This means that we can tell if any of the non-essential keys are
# modified or removed.
stripped_json_object = strip_non_essential_keys(event_json_object)
# Sign the stripped JSON object. The signature only covers the
# essential keys and the hashes. This means that we can check the
# signature even if the event is redacted.
signed_json_object = sign_json(stripped_json_object)
# Copy the signatures from the stripped event to the original event.
event_json_object["signatures"] = signed_json_oject["signatures"]
return event_json_object
Servers can then transmit the entire event or the event with the non-essential
keys removed. If the entire event is present, receiving servers can then check
the event by computing the SHA-256 of the event, excluding the ``hash`` object.
If the keys have been redacted, then the ``hash`` object is included when
calculating the SHA-256 instead.
New hash functions can be introduced by adding additional keys to the ``hash``
object. Since the ``hash`` object cannot be redacted a server shouldn't allow
too many hashes to be listed, otherwise a server might embed illict data within
the ``hash`` object. For similar reasons a server shouldn't allow hash values
that are too long.
.. TODO
[[TODO(markjh): We might want to specify a maximum number of keys for the
``hash`` and we might want to specify the maximum output size of a hash]]
[[TODO(markjh) We might want to allow the server to omit the output of well
known hash functions like SHA-256 when none of the keys have been redacted]]
.. _`Canonical JSON`: ../appendices.html#canonical-json

@ -20,7 +20,6 @@ targets:
server_server: server_server:
files: files:
- server_server_api.rst - server_server_api.rst
- { 1: event_signing.rst }
version_label: "%SERVER_RELEASE_LABEL%" version_label: "%SERVER_RELEASE_LABEL%"
identity_service: identity_service:
files: files:
@ -33,6 +32,9 @@ targets:
appendices: appendices:
files: files:
- appendices.rst - appendices.rst
- appendices/signing_json.rst
- appendices/threat_model.rst
- appendices/test_vectors.rst
groups: # reusable blobs of files when prefixed with 'group:' groups: # reusable blobs of files when prefixed with 'group:'
modules: modules:
- modules/instant_messaging.rst - modules/instant_messaging.rst
@ -42,6 +44,7 @@ groups: # reusable blobs of files when prefixed with 'group:'
- modules/presence.rst - modules/presence.rst
- modules/content_repo.rst - modules/content_repo.rst
- modules/send_to_device.rst - modules/send_to_device.rst
- modules/device_management.rst
- modules/end_to_end_encryption.rst - modules/end_to_end_encryption.rst
- modules/history_visibility.rst - modules/history_visibility.rst
- modules/push.rst - modules/push.rst

@ -328,7 +328,7 @@ and contributed by the wider community. The full list can be seen at
* [matrix-appservice-irc](https://github.com/matrix-org/matrix-appservice-irc) - an increasingly comprehensive Matrix\<-\>IRC bridge * [matrix-appservice-irc](https://github.com/matrix-org/matrix-appservice-irc) - an increasingly comprehensive Matrix\<-\>IRC bridge
* [matrix-appservice-verto](https://github.com/matrix-org/matrix-appservice-verto) - links from Matrix to FreeSWITCH via the Verto protocol * [matrix-appservice-verto](https://github.com/matrix-org/matrix-appservice-verto) - links from Matrix to FreeSWITCH via the Verto protocol
* [matrix-appservice-slack](https://github.com/matrix-org/matrix-appservice-slack) - a basic bridge to Slack * [matrix-appservice-slack](https://github.com/matrix-org/matrix-appservice-slack) - a basic bridge to Slack
* [matrix-appservice-purple](https://github.com/matrix-org/purple-matrix) - lets you access any of the 20+ protocols supported by * [node-purple](https://github.com/matrix-org/node-purple) - lets you access any of the 20+ protocols supported by
[libpurple](https://developer.pidgin.im/wiki/WhatIsLibpurple), including [libpurple](https://developer.pidgin.im/wiki/WhatIsLibpurple), including
Skype, Lync, XMPP, etc) Skype, Lync, XMPP, etc)
* [matrix-appservice-bridge](https://github.com/matrix-org/matrix-appservice-bridge) - a general NodeJS framework for writing bridges * [matrix-appservice-bridge](https://github.com/matrix-org/matrix-appservice-bridge) - a general NodeJS framework for writing bridges
@ -368,7 +368,7 @@ letting the user interact with users and rooms anywhere within the
Matrix federation.  Text and image messages are supported, and basic Matrix federation.  Text and image messages are supported, and basic
voice-only VoIP calling via WebRTC is supported in one-to-one rooms. voice-only VoIP calling via WebRTC is supported in one-to-one rooms.
(As of October 2015, experimental multi-way calling is also available (As of October 2015, experimental multi-way calling is also available
on Vector.im). on Riot.im).
##### How do I connect my homeserver to the public Matrix network? ##### How do I connect my homeserver to the public Matrix network?
@ -492,12 +492,16 @@ Yes. Matrix is just a spec, so implementations of the spec are very welcome! It
##### How secure is this? ##### How secure is this?
Server-server traffic is mandatorily TLS from the outset. Server-client traffic mandates transport layer encryption other than for tinkering. Servers maintain a public/private key pair, and sign the integrity of all messages in the context of the historical conversation, preventing tampering. Server keys are distributed using a PERSPECTIVES-style system. Server-server traffic is mandatorily TLS from the outset. Server-client traffic mandates transport layer encryption other than for tinkering. Servers maintain a public/private key pair, and sign the integrity of all messages in the context of the historical conversation, preventing tampering. Server keys are distributed using a [Perspectives](https://perspectives-project.org/)-style system.
End-to-end encryption is coming shortly to clients for both 1:1 and group chats to protect user data stored on servers, using the [Olm](https://matrix.org/git/olm) cryptographic ratchet implementation. As of October 2015 this is blocked on implementing the necessary key distribution and fingerprint management. End-to-end encryption is coming shortly to clients for both 1:1 and group chats to protect user data stored on servers, using the [Olm](https://matrix.org/git/olm) cryptographic ratchet implementation. As of October 2015 this is blocked on implementing the necessary key distribution and fingerprint management.
Privacy of metadata is not currently protected from server administrators - a malicious homeserver administrator can see who is talking to who and when, but not what is being said (once E2E encryption is enabled). See [this presentation from Jardin Entropique](http://matrix.org/~matthew/2015-06-26%20Matrix%20Jardin%20Entropique.pdf) for a more comprehensive discussion of privacy in Matrix. Privacy of metadata is not currently protected from server administrators - a malicious homeserver administrator can see who is talking to who and when, but not what is being said (once E2E encryption is enabled). See [this presentation from Jardin Entropique](http://matrix.org/~matthew/2015-06-26%20Matrix%20Jardin%20Entropique.pdf) for a more comprehensive discussion of privacy in Matrix.
##### What is Perspectives?
Rather than relying on Certificate Authorities (CAs) as in traditional SSL, a [Perspectives](https://perspectives-project.org/)-style system uses a more decentralized model for verifying keys. Perspectives uses notary servers to verify that the same key is seen across the network, making a man-in-the-middle attack much harder since an attacker must insert itself into multiple places. For federation in Matrix, each Home Server acts as a notary. When one Home Server connects to another Home Server that uses a key that it doesn't recognize, it contacts other Home Servers to ensure that they all see the same key from that Home Server.
##### Why HTTP? Doesn't HTTP suck? ##### Why HTTP? Doesn't HTTP suck?
HTTP is indeed not the most efficient transport, but it is ubiquitous, very well understood and has numerous implementations on almost every platform and language. It also has a simple upgrade path to HTTP/2, which is relatively bandwidth and round-trip efficient. HTTP is indeed not the most efficient transport, but it is ubiquitous, very well understood and has numerous implementations on almost every platform and language. It also has a simple upgrade path to HTTP/2, which is relatively bandwidth and round-trip efficient.
@ -563,33 +567,55 @@ Data is only shared between servers of participating users of a room. If all use
##### Where can I find a mobile app? ##### Where can I find a mobile app?
Vector is available for Android and iOS. Riot is available for Android and iOS.
The iOS version can be downloaded from the [Apple store](https://itunes.apple.com/us/app/vector.im/id1083446067). The iOS version can be downloaded from the [Apple store](https://itunes.apple.com/us/app/vector.im/id1083446067).
The Android version can be downloaded from the [Google Play store](https://play.google.com/store/apps/details?id=im.vector.alpha) or [F-Droid](https://f-droid.org/repository/browse/?fdid=im.vector.alpha). If you are not sure which one to choose, install Vector from the [Google Play store](https://play.google.com/store/apps/details?id=im.vector.alpha). The Android version can be downloaded from the [Google Play store](https://play.google.com/store/apps/details?id=im.vector.alpha) or [F-Droid](https://f-droid.org/repository/browse/?fdid=im.vector.alpha). If you are not sure which one to choose, install Riot from the [Google Play store](https://play.google.com/store/apps/details?id=im.vector.alpha).
For the Android app, you can also install the latest development version For the Android app, you can also install the latest development version
built by [Jenkins](http://matrix.org/jenkins/job/VectorAndroidDevelop). Use it at your own risk and only if you know what you are doing. built by [Jenkins](http://matrix.org/jenkins/job/VectorAndroidDevelop). Use it at your own risk and only if you know what you are doing.
##### I installed Vector via F-Droid, why is it draining my battery? ##### I installed Riot via F-Droid, why is it draining my battery?
The F-Droid release of Vector does not use [Google Cloud Messaging](https://developers.google.com/cloud-messaging/). This allows users that do not have or want Google Services installed to use Vector. The F-Droid release of Riot does not use [Google Cloud Messaging](https://developers.google.com/cloud-messaging/). This allows users that do not have or want Google Services installed to use Riot.
The drawback is that Vector has to pull for new messages, which can drain your battery. To counter this, you can change the delay between polls in the settings. Higher delay means better battery life (but may delay receiving messages). You can also disable the background sync entirely (which means that you won't get any notifications at all). The drawback is that Riot has to pull for new messages, which can drain your battery. To counter this, you can change the delay between polls in the settings. Higher delay means better battery life (but may delay receiving messages). You can also disable the background sync entirely (which means that you won't get any notifications at all).
If you don't mind using Google Services, you might be better off installing the [Google Play store](https://play.google.com/store/apps/details?id=im.vector.alpha) version. If you don't mind using Google Services, you might be better off installing the [Google Play store](https://play.google.com/store/apps/details?id=im.vector.alpha) version.
##### Where can I find a web app? ##### Where can I find a web app?
You can use [Vector.im](https://vector.im) - a glossy web client written on top of [matrix-react-sdk](https://github.com/matrix-org/matrix-react-sdk). You can use [Riot.im](https://Riot.im) - a glossy web client written on top of [matrix-react-sdk](https://github.com/matrix-org/matrix-react-sdk).
You can also run Vector on your own server. It's a static web application, just download the [last release](https://github.com/vector-im/vector-web/) and unpack it. You can also run Vector, the code that Riot.im uses, on your own server. It's a static web application, just download the [last release](https://github.com/vector-im/vector-web/) and unpack it.
##### Where can I find a desktop client? ##### Where can I find a desktop client?
There are several, but they don't have all the features that synapse has. Check the list of clients on [matrix.org](http://matrix.org/docs/projects/try-matrix-now.html#clients). There are several, but they don't have all the features that synapse has. Check the list of clients on [matrix.org](http://matrix.org/docs/projects/try-matrix-now.html#clients).
##### Why can't end-to-end encryption be turned off?
When encryption is enabled in a room, a flag is set in the room state, so that
all clients know to encrypt any messages they send. The room state stores
information about the room like the topic, the avatar, and the membership list.
Imagine if encryption could be turned off the same way as it is turned
on. Anyone with admin rights in the room could clear the flag and then messages
would start being transmitted unencrypted. It would be very easy for a user to
miss the change in configuration, and accidentally send a sensitive message
without encryption.
Worse yet, anyone with sysadmin access to a server could also clear the flag
(remember that the main reason for using e2e encryption is that we don't trust
the sysadmins), and could then easily read any sensitive content which was
sent.
The solution we have taken for now is to make clients ignore any requests to
disable encryption. We might experiment with ways to improve this in the future
- for instance, by alerting the user next time they try to send a message in
the room if encryption has been disabled.
| |
### QUESTIONS TO BE ANSWERED! ### QUESTIONS TO BE ANSWERED!

@ -7,11 +7,11 @@ categories: guides
<h1>Guides</h1> <h1>Guides</h1>
<p>Here is a collection of guides that might help you get involved with Matrix.</p> <p>Here is a collection of guides that might help you get involved with Matrix.</p>
<p>First, there is the <a href="./getting_involved.html" title="Getting Involved">Getting Involved</a> guide, which explains various ways of getting started with Matrix.</p> <p>First, there is the <a href="./getting_involved.html" title="Getting Involved">Getting Involved</a> guide, which explains various ways of getting started with Matrix, and the <a href="./faq.html" title="FAQ">FAQ</a>, which tries to answer all your questions relating to Matrix.</p>
<p>The (updated) <a href="/docs/guides/client-server.html" title="Client-Server API">Client-Server API</a> guide explains in detail how to use the CS API, which is useful if you want to write a client (or modify an existing one) - or if you're just interested in how it works "under the hood".</p> <p>The (updated) <a href="/docs/guides/client-server.html" title="Client-Server API">Client-Server API</a> guide explains in detail how to use the CS API, which is useful if you want to write a client (or modify an existing one) - or if you're just interested in how it works "under the hood".</p>
<p>If you were using the old v1 CS API, there is also the <a href="/docs/guides/client-server-migrating-from-v1.html">v1 migration guide</a> which justs lists the changes from v1 to r0.</p> <p>If you were using the old v1 CS API, there is also the <a href="/docs/guides/client-server-migrating-from-v1.html">v1 migration guide</a> which justs lists the changes from v1 to r0.</p>
<p><a href="./lets-encrypt.html">Let's Encrypt Matrix</a> explains how to use Let's Encrypt's certificates with your Synapse installation. This guide was written by William A Stevens.</p> <p><a href="./lets-encrypt.html">Let's Encrypt Matrix</a> explains how to use Let's Encrypt's certificates with your Synapse installation. This guide was written by William A Stevens.</p>
<p>The <a href="./application_services.html" title="Application services">Application services</a> guide introduces and explains Application services, and what they can be used for. <p>The <a href="./application_services.html" title="Application services">Application services</a> guide introduces and explains Application services, and what they can be used for.</p>
<p>Finally there is the <a href="./faq.html" title="FAQ">FAQ</a>, which tries to answer all your questions relating to Matrix!</p> <p>The <a href="./e2e_implementation.html">End-to-end Encryption Implementation Guide</a> is intended for client developers who wish to add support for end-to-end encryption.</p>
</div> </div>

@ -0,0 +1,778 @@
---
layout: post
title: End-to-End Encryption implementation guide
categories: guides
---
Implementing End-to-End Encryption in Matrix clients
====================================================
This guide is intended for authors of Matrix clients who wish to add
support for end-to-end encryption. It is highly recommended that readers
be familiar with the Matrix protocol and the use of access tokens before
proceeding.
.. contents::
The libolm library
------------------
End-to-end encryption in Matrix is based on the Olm and Megolm
cryptographic ratchets. The recommended starting point for any client
authors is with the `libolm <http://matrix.org/git/olm>`__ library,
which contains implementations of all of the cryptographic primitives
required. The library itself is written in C/C++, but is architected in
a way which makes it easy to write wrappers for higher-level languages.
Devices
-------
We have a particular meaning for “device”. As a user, I might have
several devices (a desktop client, some web browsers, an Android device,
an iPhone, etc). When I first use a client, it should register itself as
a new device. If I log out and log in again as a different user, the
client must register as a new device. Critically, the client must create
a new set of keys (see below) for each “device”.
The longevity of devices will depend on the client. In the web client,
we create a new device every single time you log in. In a mobile client,
it might be acceptable to reuse the device if a login session expires,
**provided** the user is the same. **Never** share keys between
different users.
Devices are identified by their ``device_id`` (which is unique within
the scope of a given user). By default, the ``/login`` and ``/register``
endpoints will auto-generate a ``device_id`` and return it in the
response; a client is also free to generate its own ``device_id`` or, as
above, reuse a device, in which case the client should pass the
``device_id`` in the request body.
The lifetime of devices and ``access_token``\ s are closely related. In
the simple case where a new device is created each time you log in,
there is a one-to-one mapping between a ``device_id`` and an
``access_token``. If a client reuses a ``device_id`` when logging
in, there will be several ``access_token``\ s associated with a
given ``device_id`` - but still, we would expect only one of these to be
active at once (though we do not currently enforce that in Synapse).
Keys used in End-to-End encryption
----------------------------------
There are a number of keys involved in encrypted communication: a
summary of them follows.
Ed25519 fingerprint key pair
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Ed25519 is a public-key cryptographic system for signing messages. In
Matrix, each device has an Ed25519 key pair which serves to identify
that device. The private part of the key pair should never leave the
device, but the public part is published to the Matrix network.
Curve25519 identity key pair
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Curve25519 is a public-key cryptographic system which can be used to
establish a shared secret. In Matrix, each device has a long-lived
Curve25519 identity key which is used to establish Olm sessions with
that device. Again, the private key should never leave the device, but
the public part is signed with the Ed25519 fingerprint key and published
to the network.
Theoretically we should rotate the Curve25519 identity key from time to
time, but we haven't implemented this yet.
Curve25519 one-time keys
~~~~~~~~~~~~~~~~~~~~~~~~
As well as the identity key, each device creates a number of Curve25519
key pairs which are also used to establish Olm sessions, but can only be
used once. Once again, the private part remains on the device.
At startup, Alice creates a number of one-time key pairs, and publishes
them to her homeserver. If Bob wants to establish an Olm session with
Alice, he needs to claim one of Alices one-time keys, and creates a new
one of his own. Those two keys, along with Alices and Bobs identity
keys, are used in establishing an Olm session between Alice and Bob.
Megolm encryption keys
~~~~~~~~~~~~~~~~~~~~~~
The Megolm key is used to encrypt group messages (in fact it is used to
derive an AES-256 key, and an HMAC-SHA-256 key). It is initialised with
random data. Each time a message is sent, a hash calculation is done on
the Megolm key to derive the key for the next message. It is therefore
possible to share the current state of the Megolm key with a user,
allowing them to decrypt future messages but not past messages.
Ed25519 Megolm signing key pair
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
When a sender creates a Megolm session, he also creates another Ed25519
signing key pair. This is used to sign messages sent via that Megolm
session, to authenticate the sender. Once again, the private part of the
key remains on the device. The public part is shared with other devices
in the room alongside the encryption key.
Creating and registering device keys
------------------------------------
This process only happens once, when a device first starts.
It must create the Ed25519 fingerprint key pair and the Curve25519
identity key pair. This is done by calling ``olm_create_account`` in
libolm. The (base64-encoded) keys are retrieved by calling
``olm_account_identity_keys``. The account should be stored for future
use.
It should then publish these keys to the homeserver. To do this, it
should construct a JSON object as follows:
.. code:: json
{
"algorithms": ["m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"],
"device_id": "<device_id>",
"keys": {
"curve25519:<device_id>": "<curve25519_key>",
"ed25519:<device_id>": "<ed25519_key>"
},
"user_id: <user_id>"
}
The object should be formatted as `Canonical
JSON <http://matrix.org/docs/spec/server_server/unstable.html#canonical-json>`__,
then signed with ``olm_account_sign``; the signature should be added to
the JSON as ``signatures.<user_id>.ed25519:<device_id>``.
The signed JSON is then uploaded via
``POST /_matrix/client/unstable/keys/upload``.
Creating and registering one-time keys
--------------------------------------
At first start, and at regular intervals
thereafter\ [#]_, the client should check how
many one-time keys the homeserver has stored for it, and, if necessary,
generate and upload some more.
.. [#] Every 10 minutes is suggested.
The number of one-time keys currently stored is returned by
``POST /_matrix/client/unstable/keys/upload``. (Post an empty JSON object
``{}`` if you dont want to upload the device keys.)
The maximum number of active keys supported by libolm is returned by
``olm_account_max_number_of_one_time_keys``. The client should try to
maintain about half this number on the homeserver.
To generate new one-time keys:
* Call ``olm_account_generate_one_time_keys`` to generate new keys.
* Call ``olm_account_one_time_keys`` to retrieve the unpublished keys. This
returns a JSON-formatted object with the single property ``curve25519``,
which is itself an object mapping key id to base64-encoded Curve25519
key. For example:
.. code:: json
{
"curve25519": {
"AAAAAA": "wo76WcYtb0Vk/pBOdmduiGJ0wIEjW4IBMbbQn7aSnTo",
"AAAAAB": "LRvjo46L1X2vx69sS9QNFD29HWulxrmW11Up5AfAjgU"
}
}
* Each key should be signed with the account key. To do this:
* Construct a JSON object as follows:
.. code:: json
{
"key": "<curve25519_key>"
}
* Call ``olm_account_sign`` to calculate the signature.
* Add the signature should be added to the JSON as
``signatures.<user_id>.ed25519:<device_id>``.
* The complete key object should now look like:
.. code:: json
{
"key": "wo76WcYtb0Vk/pBOdmduiGJ0wIEjW4IBMbbQn7aSnTo",
"signatures": {
"@alice:example.com": {
"ed25519:JLAFKJWSCS": "dSO80A01XiigH3uBiDVx/EjzaoycHcjq9lfQX0uWsqxl2giMIiSPR8a4d291W1ihKJL/a+myXS367WT6NAIcBA"
}
}
}
* Aggregate all the signed one-time keys into a single JSON object as follows:
.. code:: json
{
"one_time_keys": {
"signed_curve25519:<key_id>": {
"key": "<curve25519_key>",
"signatures": {
"<user_id>": {
"ed25519:<device_id>": "<signature>"
}
}
},
"signed_curve25519:<key_id>": {
...
},
...
}
}
* Upload the object via ``POST /_matrix/client/unstable/keys/upload``.
* Call ``olm_account_mark_keys_as_published`` to tell the olm library not to
return the same keys from a future call to ``olm_account_one_time_keys``.
Configuring a room to use encryption
------------------------------------
To enable encryption in a room, a client should send a state event of
type ``m.room.encryption``, and content ``{ "algorithm":
"m.megolm.v1.aes-sha2" }``.
.. |m.room.encryption| replace:: ``m.room.encryption``
.. _`m.room.encryption`:
Handling an ``m.room.encryption`` state event
---------------------------------------------
When a client receives an ``m.room.encryption`` event as above, it
should set a flag to indicate that messages sent in the room should be
encrypted.
This flag should **not** be cleared if a later ``m.room.encryption``
event changes the configuration. This is to avoid a situation where a
MITM can simply ask participants to disable encryption. In short: once
encryption is enabled in a room, it can never be disabled.
The event should contain an ``algorithm`` property which defines which
encryption algorithm should be used for encryption. Currently only
``m.megolm.v1-aes-sha2`` is permitted here.
The event may also include other settings for how messages sent in the room
should be encrypted (for example, ``rotation_period_ms`` to define how often
the session should be replaced).
Handling an ``m.room.encrypted`` event
--------------------------------------
Encrypted events have a type of ``m.room.encrypted``. They have a
content property ``algorithm`` which gives the encryption algorithm in
use, as well as other properties specific to the algorithm.
The encrypted payload is a JSON object with the properties ``type``
(giving the decrypted event type), and ``content`` (giving the decrypted
content). Depending on the algorithm in use, the payload may contain
additional keys.
There are currently two defined algorithms:
``m.olm.v1.curve25519-aes-sha2``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Encrypted events using this algorithm should have a ``sender_key`` and a
``ciphertext`` property.
The ``sender_key`` property of the event content gives the Curve25519
identity key of the sender. Clients should maintain a list of known Olm
sessions for each device they speak to; it is recommended to index them
by Curve25519 identity key.
Olm messages are encrypted separately for each recipient device.
``ciphertext`` is an object mapping from the Curve25519 identity key for
the recipient device. The receiving client should, of course, look for
its own identity key in this object. (If it isn't listed, the message
wasn't sent for it, and the client can't decrypt it; it should show an
error instead, or similar).
This should result in an object with the properties ``type`` and
``body``. Messages of type '0' are 'prekey' messages which are used to
establish a new Olm session between two devices; type '1' are normal
messages which are used once a message has been received on the session.
When a message (of either type) is received, a client should first
attempt to decrypt it with each of the known sessions for that sender.
There are two steps to this:
- If (and only if) ``type==0``, the client should call
``olm_matches_inbound_session`` with the session and ``body``. This
returns a flag indicating whether the message was encrypted using
that session.
- The client calls ``olm_decrypt``, with the session, ``type``, and
``body``. If this is successful, it returns the plaintext of the
event.
If the client was unable to decrypt the message using any known sessions
(or if there are no known sessions yet), **and** the message had type 0,
**and** ``olm_matches_inbound_session`` wasn't true for any existing
sessions, then the client can try establishing a new session. This is
done as follows:
- Call ``olm_create_inbound_session_from`` using the olm account, and
the ``sender_key`` and ``body`` of the message.
- If the session was established successfully:
- call ``olm_remove_one_time_keys`` to ensure that the same
one-time-key cannot be reused.
- Call ``olm_decrypt`` with the new session
- Store the session for future use
At the end of this, the client will hopefully have successfully
decrypted the payload.
As well as the ``type`` and ``content`` properties, the payload should
contain a number of other properties. Each of these should be checked as
follows [#]_.
``sender``
The user ID of the sender. The client should check that this matches the
``sender`` in the event.
``recipient``
The user ID of the recipient. The client should check that this matches the
local user ID.
``keys``
an object with a property ``ed25519``, The client should check that the
value of this property matches the sender's fingerprint key when `marking
the event as verified`_\ .
``recipient_keys``
an object with a property ``ed25519``. The client should check that the
value of this property matches its own fingerprint key.
.. [#] These tests prevent an attacker publishing someone else's curve25519
keys as their own and subsequently claiming to have sent messages which they
didn't.
``m.megolm.v1.aes-sha2``
~~~~~~~~~~~~~~~~~~~~~~~~
Encrypted events using this algorithm should have ``sender_key``,
``session_id`` and ``ciphertext`` content properties. If the
``room_id``, ``sender_key`` and ``session_id`` correspond to a known
Megolm session (see `below`__), the ciphertext can be
decrypted by passing the ciphertext into ``olm_group_decrypt``.
__ `m.room_key`_
In order to avoid replay attacks a client should remember the megolm
``message_index`` returned by ``olm_group_decrypt`` of each event they decrypt
for each session. If the client decrypts an event with the same
``message_index`` as one that it has already received using that session then
it should treat the message as invalid.
The client should check that the sender's fingerprint key matches the
``keys.ed25519`` property of the event which established the Megolm session
when `marking the event as verified`_.
.. _`m.room_key`:
Handling an ``m.room_key`` event
--------------------------------
These events contain key data to allow decryption of other messages.
They are sent to specific devices, so they appear in the ``to_device``
section of the response to ``GET /_matrix/client/r0/sync``. They will
also be encrypted, so will need decrypting as above before they can be
seen.
The event content will contain an 'algorithm' property, indicating the
encryption algorithm the key data is to be used for. Currently, this
will always be ``m.megolm.v1.aes-sha2``.
Room key events for Megolm will also have ``room_id``, ``session_id``, and
``session_key`` keys. They are used to establish a Megolm session. The
``room_id`` identifies which room the session will be used in. The ``room_id``,
together with the ``sender_key`` of the ``room_key`` event before it was
decrypted, and the ``session_id``, uniquely identify a Megolm session. If they
do not represent a known session, the client should start a new inbound Megolm
session by calling ``olm_init_inbound_group_session`` with the ``session_key``.
The client should remember the value of the keys property of the payload
of the encrypted ``m.room_key`` event and store it with the inbound
session. This is used as above when marking the event as verified.
.. _`download the device list`:
Downloading the device list for users in the room
-------------------------------------------------
Before an encrypted message can be sent, it is necessary to retrieve the
list of devices for each user in the room. This can be done proactively,
or deferred until the first message is sent. The information is also
required to allow users to `verify or block devices`__.
__ `blocking`_
The client should build a JSON query object as follows:
.. code:: json
{
"<user_id>": {},
...
}
Each member in the room should be included in the query. This is then
sent via ``POST /_matrix/client/unstable/keys/query.``
The result includes, for each listed user id, a map from device ID to an
object containing information on the device, as follows:
.. code:: json
{
"algorithms": [...],
"device_id": "<device_id>",
"keys": {
"curve25519:<device_id>": "<curve25519_key>",
"ed25519:<device_id>": "<ed25519_key>"
},
"signatures": {
"<userId>": {
"ed25519:<device_id>": "<signature>"
},
},
"unsigned": {
"device_display_name": "<display name>"
},
"user_id: <user_id>"
}
The client should first check the signature on this object. To do this,
it should remove the ``signatures`` and ``unsigned`` properties, format
the remainder as Canonical JSON, and pass the result into
``olm_ed25519_verify``, using the Ed25519 key for the ``key`` parameter,
and the corresponding signature for the ``signature`` parameter. If the
signature check fails, no further processing should be done on the
device.
The client must also check that the ``user_id`` and ``device_id`` fields in the
object match those in the top-level map [#]_.
The client should check if the ``user_id``/``device_id`` correspond to a device
it had seen previously. If it did, the client **must** check that the Ed25519
key hasn't changed. Again, if it has changed, no further processing should be
done on the device.
Otherwise the client stores the information about this device.
.. [#] This prevents a malicious or compromised homeserver replacing the keys
for the device with those of another.
Sending an encrypted event
--------------------------
When sending a message in a room `configured to use encryption`__, a client
first checks to see if it has an active outbound Megolm session. If not, it
first `creates one as per below`__. If an outbound session exists, it should
check if it is time to `rotate`__ it, and create a new one if so.
__ `Configuring a room to use encryption`_
__ `Starting a Megolm session`_
__ `Rotating Megolm sessions`_
The client then builds an encryption payload as follows:
.. code:: json
{
"type": "<event type>",
"content": "<event content>",
"room_id": "<id of destination room>"
}
and calls ``olm_group_encrypt`` to encrypt the payload. This is then packaged
into event content as follows:
.. code:: json
{
"algorithm": "m.megolm.v1.aes-sha2",
"sender_key": "<our curve25519 device key>",
"ciphertext": "<encrypted payload>",
"session_id": "<outbound group session id>",
"device_id": "<our device ID>"
}
Finally, the encrypted event is sent to the room with ``POST
/_matrix/client/r0/rooms/<room_id>/send/m.room.encrypted/<txn_id>``.
Starting a Megolm session
~~~~~~~~~~~~~~~~~~~~~~~~~
When a message is first sent in an encrypted room, the client should
start a new outbound Megolm session. This should **not** be done
proactively, to avoid proliferation of unnecessary Megolm sessions.
To create the session, the client should call
``olm_init_outbound_group_session``, and store the details of the
outbound session for future use.
The client should then call ``olm_outbound_group_session_id`` to get the
unique ID of the new session, and ``olm_outbound_group_session_key`` to
retrieve the current ratchet key and index. It should store these
details as an inbound session, just as it would when `receiving them via
an m.room_key event`__.
__ `m.room_key`_
The client must then share the keys for this session with each device in the
room. It must therefore `download the device list`_ if it hasn't already done
so, and for each device in the room which has not been `blocked`__, the client
should:
__ `blocking`_
* Build a content object as follows:
.. code:: json
{
"algorithm": "m.megolm.v1.aes-sha2",
"room_id": "<id of destination room>",
"session_id": "<session id>",
"session_key": "<session_key>"
}
- Encrypt the content as an ``m.room_key`` event using Olm, as below.
Once all of the key-sharing event contents have been assembled, the
events should be sent to the corresponding devices via
``PUT /_matrix/client/unstable/sendToDevice/m.room.encrypted/<txnId>``.
Rotating Megolm sessions
~~~~~~~~~~~~~~~~~~~~~~~~
Megolm sessions may not be reused indefinitely.
The number of messages which can be sent before a session should be rotated is
given by the ``rotation_period_msgs`` property of the |m.room.encryption|_
event, or ``100`` if that property isn't present.
Similarly, the maximum age of a megolm session is given, in milliseconds, by
the ``rotation_period_ms`` property of the ``m.room.encryption``
event. ``604800000`` (a week) is the recommended default here.
Once either the message limit or time limit have been reached, the client
should start a new session before sending any more messages.
Encrypting an event with Olm
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Olm is not used for encrypting room events, as it requires a separate
copy of the ciphertext for each device, and because the receiving device
can only decrypt received messages once. However, it is used for
encrypting key-sharing events for Megolm.
When encrypting an event using Olm, the client should:
- Build an encryption payload as follows:
.. code:: json
{
"type": "<event type>",
"content": "<event content>",
"sender": "<our user ID>",
"sender_device": "<our device ID>",
"keys": {
"ed25519": "<our ed25519 fingerprint key>"
},
"recipient": "<recipient user ID>",
"recipient_keys": {
"ed25519": "<recipient's ed25519 fingerprint key>"
},
}
- Check if it has an existing Olm session; if it does not, `start a new
one`__. If it has several (as may happen due to
races when establishing sessions), it should use the one with the
first session_id when sorted by their ASCII codepoints (ie, 'A'
would be before 'Z', which would be before 'a').
__ `Starting an Olm session`_
- Encrypt the payload by calling ``olm_encrypt``.
- Package the payload into event content as follows:
.. code:: json
{
"algorithm": "m.olm.v1.curve25519-aes-sha2",
"sender_key": "<our curve25519 identity key>",
"ciphertext": "<encrypted payload>"
}
Starting an Olm session
~~~~~~~~~~~~~~~~~~~~~~~
To start a new Olm session with another device, a client must first
claim one of the other device's one-time keys. To do this, it should
create a query object as follows:
.. code:: json
{
"<user id>": {
"<device_id>": "signed_curve25519",
...
},
...
}
and send this via ``POST /_matrix/client/unstable/keys/claim``. Claims
for multiple devices should be aggregated into a single request.
This will return a result as follows:
.. code:: json
{
"<user id>": {
"<device_id>": {
"signed_curve25519:<key_id>": {
"key": "<curve25519_key>",
"signatures": {
"<user_id>": {
"ed25519:<device_id>": "<signature>"
}
}
},
},
...
},
...
}
The client should first check the signatures on the signed key objects. As with
checking the signatures on the device keys, it should remove the ``signatures``
and (if present) ``unsigned`` properties, format the remainder as Canonical
JSON, and pass the result into ``olm_ed25519_verify``, using the Ed25519 device
key for the ``key`` parameter.
Provided the key object passes verification, the client should then pass the
key, along with the Curve25519 Identity key for the remote device, into
``olm_create_outbound_session``.
Handling membership changes
---------------------------
The client should monitor rooms which are configured to use encryption for
membership changes.
When a member leaves a room, the client should invalidate any active outbound
Megolm session, to ensure that a new session is used next time the user sends a
message.
When a new member joins a room, the client should first `download the device
list`_ for the new member, if it doesn't already have it.
After giving the user an opportunity to `block`__ any suspicious devices, the
client should share the keys for the outbound Megolm session with all the new
member's devices. This is done in the same way as `creating a new session`__,
except that there is no need to start a new Megolm session: due to the design
of the Megolm ratchet, the new user will only be able to decrypt messages
starting from the current state. The recommended method is to maintain a list
of members who are waiting for the session keys, and share them when the user
next sends a message.
__ `blocking`_
__ `Starting a Megolm session`_
Sending New Device announcements
--------------------------------
When a user logs in on a new device, it is necessary to make sure that
other devices in any rooms with encryption enabled are aware of the new
device. This is done as follows.
Once the initial call to the ``/sync`` API completes, the client should
iterate through each room where encryption is enabled. For each user
(including the client's own user), it should build a content object as
follows:
.. code:: json
{
"device_id": "<our device ID>",
"rooms": ["<shared room id 1>", "<room id 2>", ... ]
}
Once all of these have been constructed, they should be sent to all of the
relevant user's devices (using the wildcard ``*`` in place of the
``device_id``) via ``PUT
/_matrix/client/unstable/sendToDevice/m.new_device/<txnId>.``
Handling an ``m.new_device`` event
----------------------------------
As with ``m.room_key`` events, these will appear in the ``to_device``
section of the ``/sync`` response.
The client should `download the device list`_ of the sender, to get the details
of the new device.
The event content will contain a ``rooms`` property, as well as the
``device_id`` of the new device. For each room in the list, the client
should check if encryption is enabled, and if the sender of the event is
a member of that room. If so, the client should share the keys for the
outbound Megolm session with the new device, in the same way as
`handling a new user in the room`__.
__ `Handling membership changes`_
.. _`blocking`:
Blocking / Verifying devices
----------------------------
It should be possible for a user to mark each device belonging to
another user as 'Blocked' or 'Verified'.
When a user chooses to block a device, this means that no further
encrypted messages should be shared with that device. In short, it
should be excluded when sharing room keys when `starting a new Megolm
session <#_p5d1esx6gkrc>`__. Any active outbound Megolm sessions whose
keys have been shared with the device should also be invalidated so that
no further messages are sent over them.
Verifying a device involves ensuring that the device belongs to the
claimed user. Currently this must be done by showing the user the
Ed25519 fingerprint key for the device, and prompting the user to verify
out-of-band that it matches the key shown on the other user's device.
.. _`marking the event as verified`:
Marking events as 'verified'
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Once a device has been verified, it is possible to verify that events
have been sent from a particular device. See the section on `Handling an
m.room.encrypted event`_ for notes on how to do this
for each algorithm. Events sent from a verified device can be decorated
in the UI to show that they have been sent from a verified device.

@ -1,6 +1,6 @@
--- ---
layout: project layout: project
title: Mclient.el title: matrix-client.el
categories: projects client categories: projects client
thumbnail: https://matrix.org/blog/wp-content/uploads/2015/10/mclient1-400x284.png thumbnail: https://matrix.org/blog/wp-content/uploads/2015/10/mclient1-400x284.png
description: A Matrix.org frontend for Emacs! description: A Matrix.org frontend for Emacs!
@ -11,4 +11,4 @@ maturity: Alpha
![screenshot](https://matrix.org/blog/wp-content/uploads/2015/10/mclient1-1080x365.png "{{ page.title }}") ![screenshot](https://matrix.org/blog/wp-content/uploads/2015/10/mclient1-1080x365.png "{{ page.title }}")
# {{ page.title }} # {{ page.title }}
Mclient.el is a Matrix.org frontend for Emacs. For more information and install instructions visit [http://whatthefuck.computer/blog/2015/10/26/mclient/](http://whatthefuck.computer/blog/2015/10/26/mclient/). The code is available from [http://fort.kickass.systems:10082/cgit/personal/rrix/pub/matrix.el.git/](http://fort.kickass.systems:10082/cgit/personal/rrix/pub/matrix.el.git/). `matrix-client.el` is a Matrix.org frontend for Emacs. For more information and install instructions visit [the MELPA Emacs package repository](https://melpa.org/#/matrix-client). The code is available at [https://fort.kickass.systems/git/rrix/matrix-client.git](https://fort.kickass.systems/git/rrix/matrix-client.git).

@ -8,3 +8,5 @@ maturity: Alpha
# {{ page.title }} # {{ page.title }}
This project creates a bridge between a Matrix room and an XMPP MUC: [https://github.com/jfrederickson/matrix-xmpp-bridge](https://github.com/jfrederickson/matrix-xmpp-bridge) This project creates a bridge between a Matrix room and an XMPP MUC: [https://github.com/jfrederickson/matrix-xmpp-bridge](https://github.com/jfrederickson/matrix-xmpp-bridge)
Update: there is also a forked and refactored version at [pztrn's GitHub space](https://github.com/pztrn/matrix-xmpp-bridge)

@ -4,10 +4,12 @@ title: Perpetually Talking Online (PTO)
categories: projects client categories: projects client
description: PTO is an IRC frontend to the federated Matrix network. description: PTO is an IRC frontend to the federated Matrix network.
author: tdfischer author: tdfischer
maturity: Alpha maturity: Abandoned
--- ---
# {{ page.title }} # {{ page.title }}
Perpetually Talking Online (PTO) is an IRC frontend to the federated Matrix network. It aims to enable as many people as possible to use an existing Matrix homeserver with their existing IRC clients, and provides a radically expanded feature set for existing IRC communities looking to migrate to Matrix. Perpetually Talking Online (PTO) is an IRC frontend to the federated Matrix network. It aims to enable as many people as possible to use an existing Matrix homeserver with their existing IRC clients, and provides a radically expanded feature set for existing IRC communities looking to migrate to Matrix.
See [pto.im](http://pto.im) for more info - or grab the code from [github](https://github.com/tdfischer/pto). See [pto.im](http://pto.im) for more info - or grab the code from [GitHub](https://github.com/tdfischer/pto).
Unfortunately, it appears that this project is no longer being worked on. Check out [matrix-ircd](./matrix-ircd.html) instead!

@ -8,6 +8,6 @@ maturity: Early beta
--- ---
# {{ page.title }} # {{ page.title }}
Connect to RSS and Twitter feeds via Ryan Rix' feedbot! Connect to RSS and Twitter feeds via Ryan Rix's feedbot!
Check it out from [Ryan's git repo](http://fort.kickass.systems:10082/cgit/personal/rrix/pub/feedbot.git/) Check it out from [Ryan's git repo](https://fort.kickass.systems/git/rrix/matrix-feedbot)

@ -10,4 +10,4 @@ maturity: Early beta
# {{ page.title }} # {{ page.title }}
Control Adafruit Neopixels via Matrix! Control Adafruit Neopixels via Matrix!
Check it out from [Ryan's git repo](http://fort.kickass.systems:10082/cgit/personal/rrix/pub/lightrix.git/) Check it out from [Ryan's git repo](https://fort.kickass.systems/git/rrix/lightrix) or watch it in action on [YouTube](https://www.youtube.com/watch?v=4YG9Fk5aP24)

@ -10,4 +10,4 @@ maturity: Early beta
# {{ page.title }} # {{ page.title }}
Pipe in to and out of a Matrix room using the [Matrix Python SDK](https://github.com/matrix-org/matrix-python-sdk). Pipe in to and out of a Matrix room using the [Matrix Python SDK](https://github.com/matrix-org/matrix-python-sdk).
Check it out from [Ryan's git repo](http://fort.kickass.systems:10082/cgit/personal/rrix/pub/mcat.git/) Check it out from [Ryan's git repo](https://fort.kickass.systems/git/rrix/matrix-cat)

@ -10,4 +10,4 @@ maturity: Early beta
# {{ page.title }} # {{ page.title }}
Webring software built on top of Matrix.org which means it doesn't have a central point of failure. Webring software built on top of Matrix.org which means it doesn't have a central point of failure.
Check it out at [Ryan's blog](http://whatthefuck.computer/blog/2015/12/06/polynomial-a-decentralized-webring/) and grab the code from [Ryan's git repo](http://fort.kickass.systems:10082/cgit/personal/rrix/pub/webring.git/) Check it out at [Ryan's blog](http://whatthefuck.computer/blog/2015/12/06/polynomial-a-decentralized-webring/) and grab the code from [Ryan's git repo](https://fort.kickass.systems/git/rrix/polynomial)

@ -0,0 +1,14 @@
---
layout: project
title: Revolt
categories: projects client
description: Revolt wraps Riot to provide better integration with desktop environments.
author: Adrian Perez
maturity: Alpha
---
# {{ page.title }}
Revolt is a small application which wraps [Riot](./riot.html) to provide better integration with desktop environments in general, and GNOME in particular.
Check it out on [GitHub](https://github.com/aperezdc/revolt)!

@ -0,0 +1,29 @@
---
layout: project
title: Hello Matrix Bot
categories: projects other
description: Bot with an array of plugins
author: Alexander Rudyk
maturity: Alpha
---
# {{ page.title }}
This project is a simple attempt at providing a friendly face to as many popular services as possible, making them accessible from any Matrixroom. "Hello Matrix" is written as NodeJS application, building on matrix-js-sdk, the JavaScript SDK from the creators of Matrix. It is a hobby project and as such far from feature complete, in fact it is all very basic right now - hopefully this will change over the coming months.
You can either use the "Hello Matrix" bot running on our server (just invite @hello-matrix:matrix.org into your Matrix room) or you can check out the code from this repository and run your own instance of "Hello Matrix" (also providing your own API keys to the services you want to use).
"Hello Matrix" currently supports the following services:
* Sending and receiving messages using Bitmessage
* Numeric calculations using Wolfram Alpha
* Throwing the dice (generating a random number)
* Adding tasks to your Kanban board from Kanban Tool and getting notified of task status changes
* Tracerouting a given IP
* Weather from OpenWeatherMap
* Providing WHOIS information on a domain or IP address
* Adding tasks and monitoring progress on Wunderlist lists
* The goal is to add at least generic web hook functionality in the coming months, which would immediately make a large number of other integrations possible. Well keep you updated on any progress.
|
Follow the progress and grab the code from [gitlab](https://gitlab.com/argit/hello-matrix-bot)!

@ -0,0 +1,17 @@
---
layout: project
title: matrix-ircd
categories: projects client
description: An IRCd implementation backed by Matrix.
author: Erik
maturity: Alpha
---
# {{ page.title }}
An IRCd implementation backed by Matrix. Inspired by [PTO](./pto.html)!
This is a work in progress. Matrix IRCd should be stable enough to hack on and test, but has not been tested in production or for any length of time.
See the [GitHub issues page](https://github.com/matrix-org/matrix-ircd/issues) for a more detailed breakdown of what is left to do, and/or join the discussion on the Matrix channel: [#matrix-ircd:matrix.org](https://matrix.to/#/#matrix-ircd:matrix.org)
You can check out the code from [GitHub](https://github.com/matrix-org/matrix-ircd).

@ -0,0 +1,13 @@
---
layout: project
title: Hangouts Bridge
categories: projects as
author: CyrusTheHedgehog
maturity: Alpha
---
# {{ page.title }}
This project creates a bridge between Google Hangouts and Matrix. It was originally developed as part of the 2016 Global TADHack - see related [blogpost](https://matrix.org/blog/2016/10/20/tadhack-global-2016/) and [video of the presentation](https://www.youtube.com/watch?v=X41RbOKTrbE&t=17m31s).
You can get the source from [GitHub](https://github.com/CyrusTheHedgehog/Hangouts-Bridge).

@ -0,0 +1,12 @@
---
layout: project
title: mxpp
categories: projects other
description: Bot for bridging Matrix and one-to-one XMPP chats
author: anewusername
maturity: Alpha
---
# {{ page.title }}
[mxpp](https://github.com/anewusername/mxpp) is a bot for bridging XMPP and Matrix. It creates one Matrix room per user on your XMPP roster, and then passes messages between the Matrix room and XMPP user.

@ -2,9 +2,13 @@
``{{event.type}}`` ``{{event.type}}``
{{(4 + event.type | length) * title_kind}} {{(4 + event.type | length) * title_kind}}
{% if (event.typeof | length) %}
*{{event.typeof}}* *{{event.typeof}}*
{{event.typeof_info}} {{event.typeof_info}}
{% endif -%}
{{event.desc | wrap(80)}} {{event.desc | wrap(80)}}
{% for table in event.content_fields %} {% for table in event.content_fields %}
{{"``"+table.title+"``" if table.title else "" }} {{"``"+table.title+"``" if table.title else "" }}

@ -27,6 +27,12 @@ Request format:
`No parameters` `No parameters`
{% endif %} {% endif %}
{% if endpoint.res_headers|length > 0 -%}
Response headers:
{{ tables.paramtable(endpoint.res_headers) }}
{% endif -%}
{% if endpoint.res_tables|length > 0 -%} {% if endpoint.res_tables|length > 0 -%}
Response format: Response format:

@ -21,6 +21,7 @@ For the actual conversion of data -> RST (including templates), see the sections
file instead. file instead.
""" """
from batesian.units import Units from batesian.units import Units
from collections import OrderedDict
import logging import logging
import inspect import inspect
import json import json
@ -48,6 +49,20 @@ STATE_EVENT = "core-event-schema/state_event.yaml"
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# a yaml Loader which loads mappings into OrderedDicts instead of regular
# dicts, so that we preserve the ordering of properties from the api files.
#
# with thanks to http://stackoverflow.com/a/21912744/637864
class OrderedLoader(yaml.Loader):
pass
def construct_mapping(loader, node):
loader.flatten_mapping(node)
pairs = loader.construct_pairs(node)
return OrderedDict(pairs)
OrderedLoader.add_constructor(
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
construct_mapping)
def resolve_references(path, schema): def resolve_references(path, schema):
if isinstance(schema, dict): if isinstance(schema, dict):
# do $ref first # do $ref first
@ -55,11 +70,11 @@ def resolve_references(path, schema):
value = schema['$ref'] value = schema['$ref']
path = os.path.join(os.path.dirname(path), value) path = os.path.join(os.path.dirname(path), value)
with open(path) as f: with open(path) as f:
ref = yaml.load(f) ref = yaml.load(f, OrderedLoader)
result = resolve_references(path, ref) result = resolve_references(path, ref)
del schema['$ref'] del schema['$ref']
else: else:
result = {} result = OrderedDict()
for key, value in schema.items(): for key, value in schema.items():
result[key] = resolve_references(path, value) result[key] = resolve_references(path, value)
@ -91,7 +106,7 @@ def inherit_parents(obj):
for key in ('properties', 'additionalProperties', 'patternProperties'): for key in ('properties', 'additionalProperties', 'patternProperties'):
if p.get(key): if p.get(key):
result.setdefault(key, {}).update(p[key]) result.setdefault(key, OrderedDict()).update(p[key])
return result return result
@ -104,42 +119,27 @@ def get_json_schema_object_fields(obj, enforce_title=False):
"get_json_schema_object_fields: Object %s isn't an object." % obj "get_json_schema_object_fields: Object %s isn't an object." % obj
) )
logger.debug("Processing object with title '%s'", obj.get("title")) obj_title = obj.get("title")
logger.debug("Processing object with title '%s'", obj_title)
if enforce_title and not obj.get("title"): if enforce_title and not obj_title:
# Force a default titile of "NO_TITLE" to make it obvious in the # Force a default titile of "NO_TITLE" to make it obvious in the
# specification output which parts of the schema are missing a title # specification output which parts of the schema are missing a title
obj["title"] = 'NO_TITLE' obj_title = 'NO_TITLE'
additionalProps = obj.get("additionalProperties") additionalProps = obj.get("additionalProperties")
props = obj.get("properties") props = obj.get("properties")
if additionalProps and not props: if additionalProps and not props:
# not "really" an object, just a KV store # not "really" an object, just a KV store
additionalProps = inherit_parents(additionalProps) logger.debug("%s is a pseudo-object", obj_title)
logger.debug("%s is a pseudo-object", obj.get("title"))
key_type = additionalProps.get("x-pattern", "string") key_type = additionalProps.get("x-pattern", "string")
res = process_data_type(additionalProps)
value_type = additionalProps["type"] return {
if value_type == "object": "type": "{%s: %s}" % (key_type, res["type"]),
nested_objects = get_json_schema_object_fields( "tables": res["tables"],
additionalProps, }
enforce_title=True,
)
value_type = nested_objects[0]["title"]
tables = [x for x in nested_objects if not x.get("no-table")]
else:
key_type = "string"
tables = []
tables = [{
"title": "{%s: %s}" % (key_type, value_type),
"no-table": True
}]+tables
logger.debug("%s done: returning %s", obj.get("title"), tables)
return tables
if not props: if not props:
props = obj.get("patternProperties") props = obj.get("patternProperties")
@ -152,21 +152,15 @@ def get_json_schema_object_fields(obj, enforce_title=False):
del props[key_name] del props[key_name]
# Sometimes you just want to specify that a thing is an object without # Sometimes you just want to specify that a thing is an object without
# doing all the keys. Allow people to do that if they set a 'title'. # doing all the keys.
if not props and obj.get("title"):
return [{
"title": obj["title"],
"no-table": True
}]
if not props: if not props:
raise Exception( return {
"Object %s has no properties and no title" % obj "type": obj_title,
) "tables": [],
}
required_keys = set(obj.get("required", [])) required_keys = set(obj.get("required", []))
obj_title = obj.get("title")
first_table_rows = [] first_table_rows = []
tables = [] tables = []
@ -174,9 +168,14 @@ def get_json_schema_object_fields(obj, enforce_title=False):
try: try:
logger.debug("Processing property %s.%s", obj_title, key_name) logger.debug("Processing property %s.%s", obj_title, key_name)
required = key_name in required_keys required = key_name in required_keys
res = process_prop(key_name, props[key_name], required) res = process_data_type(props[key_name], required)
first_table_rows.append(res["row"]) first_table_rows.append({
"key": key_name,
"type": res["type"],
"required": required,
"desc": res["desc"],
})
tables.extend(res["tables"]) tables.extend(res["tables"])
logger.debug("Done property %s" % key_name) logger.debug("Done property %s" % key_name)
@ -192,98 +191,68 @@ def get_json_schema_object_fields(obj, enforce_title=False):
"rows": first_table_rows, "rows": first_table_rows,
}) })
return tables return {
"type": obj_title,
"tables": tables,
}
def process_prop(key_name, prop, required):
# process a data type definition. returns a dictionary with the keys:
# type: stringified type name
# desc: description
# enum_desc: description of permissible enum fields
# is_object: true if the data type is an object
# tables: list of additional table definitions
def process_data_type(prop, required=False, enforce_title=True):
prop = inherit_parents(prop) prop = inherit_parents(prop)
value_type = None prop_type = prop['type']
desc = prop.get("description", "")
prop_type = prop.get('type')
tables = [] tables = []
enum_desc = None
if prop_type is None: is_object = False
raise KeyError("Property '%s' of object '%s' missing 'type' field"
% (key_name, obj))
logger.debug("%s is a %s", key_name, prop_type)
if prop_type == "object": if prop_type == "object":
nested_objects = get_json_schema_object_fields( res = get_json_schema_object_fields(
prop, prop,
enforce_title=True, enforce_title=enforce_title,
) )
value_type = nested_objects[0]["title"] prop_type = res["type"]
value_id = value_type tables = res["tables"]
is_object = True
tables += [x for x in nested_objects if not x.get("no-table")]
elif prop_type == "array": elif prop_type == "array":
items = inherit_parents(prop["items"]) nested = process_data_type(prop["items"])
# if the items of the array are objects then recurse prop_type = "[%s]" % nested["type"]
if items["type"] == "object": tables = nested["tables"]
nested_objects = get_json_schema_object_fields( enum_desc = nested["enum_desc"]
items,
enforce_title=True, if prop.get("enum"):
if len(prop["enum"]) > 1:
prop_type = "enum"
enum_desc = (
"One of: %s" % json.dumps(prop["enum"])
) )
value_id = nested_objects[0]["title"]
value_type = "[%s]" % value_id
tables += nested_objects
else: else:
value_type = items["type"] enum_desc = (
if isinstance(value_type, list): "Must be '%s'." % prop["enum"][0]
value_type = " or ".join(value_type) )
value_id = value_type
value_type = "[%s]" % value_type
array_enums = items.get("enum")
if array_enums:
if len(array_enums) > 1:
value_type = "[enum]"
desc += (
" One of: %s" % json.dumps(array_enums)
)
else:
desc += (
" Must be '%s'." % array_enums[0]
)
else:
value_type = prop_type
value_id = prop_type
if prop.get("enum"):
if len(prop["enum"]) > 1:
value_type = "enum"
if desc:
desc += " "
desc += (
"One of: %s" % json.dumps(prop["enum"])
)
else:
if desc:
desc += " "
desc += (
"Must be '%s'." % prop["enum"][0]
)
if isinstance(value_type, list):
value_type = " or ".join(value_type)
if isinstance(prop_type, list):
prop_type = " or ".join(prop_type)
if required:
desc = "**Required.** " + desc rq = "**Required.**" if required else None
desc = " ".join(x for x in [rq, prop.get("description"), enum_desc] if x)
return { return {
"row": { "type": prop_type,
"key": key_name, "desc": desc,
"type": value_type, "enum_desc": enum_desc,
"id": value_id, "is_object": is_object,
"required": required,
"desc": desc,
},
"tables": tables, "tables": tables,
} }
def deduplicate_tables(tables):
def get_tables_for_schema(schema):
schema = inherit_parents(schema)
tables = get_json_schema_object_fields(schema)
# the result may contain duplicates, if objects are referred to more than # the result may contain duplicates, if objects are referred to more than
# once. Filter them out. # once. Filter them out.
# #
@ -305,32 +274,106 @@ def get_tables_for_schema(schema):
return filtered return filtered
def get_tables_for_schema(schema):
pv = process_data_type(schema, enforce_title=False)
return deduplicate_tables(pv["tables"])
def get_tables_for_response(schema):
pv = process_data_type(schema, enforce_title=False)
tables = deduplicate_tables(pv["tables"])
# make up the first table, with just the 'body' row in, unless the response
# is an object, in which case there's little point in having one.
if not pv["is_object"]:
tables = [{
"title": None,
"rows": [{
"key": "<body>",
"type": pv["type"],
"desc": pv["desc"],
}]
}] + tables
logger.debug("response: %r" % tables)
return tables
def get_example_for_schema(schema): def get_example_for_schema(schema):
"""Returns a python object representing a suitable example for this object""" """Returns a python object representing a suitable example for this object"""
schema = inherit_parents(schema)
if 'example' in schema: if 'example' in schema:
example = schema['example'] example = schema['example']
return example return example
if 'properties' in schema:
res = {} proptype = schema['type']
if proptype == 'object':
if 'properties' not in schema:
raise Exception('"object" property has neither properties nor example')
res = OrderedDict()
for prop_name, prop in schema['properties'].iteritems(): for prop_name, prop in schema['properties'].iteritems():
logger.debug("Parsing property %r" % prop_name) logger.debug("Parsing property %r" % prop_name)
prop_example = get_example_for_schema(prop) prop_example = get_example_for_schema(prop)
res[prop_name] = prop_example res[prop_name] = prop_example
return res return res
if 'items' in schema:
if proptype == 'array':
if 'items' not in schema:
raise Exception('"array" property has neither items nor example')
return [get_example_for_schema(schema['items'])] return [get_example_for_schema(schema['items'])]
return schema.get('type', '??')
if proptype == 'integer':
return 0
if proptype == 'string':
return proptype
raise Exception("Don't know to make an example %s" % proptype)
def get_example_for_param(param): def get_example_for_param(param):
"""Returns a stringified example for a parameter"""
if 'x-example' in param: if 'x-example' in param:
return param['x-example'] return param['x-example']
schema = param.get('schema') schema = param.get('schema')
if not schema: if not schema:
return None return None
# allow examples for the top-level object to be in formatted json
exampleobj = None
if 'example' in schema: if 'example' in schema:
return schema['example'] exampleobj = schema['example']
return json.dumps(get_example_for_schema(param['schema']), if isinstance(exampleobj, basestring):
indent=2) return exampleobj
if exampleobj is None:
exampleobj = get_example_for_schema(schema)
return json.dumps(exampleobj, indent=2)
def get_example_for_response(response):
"""Returns a stringified example for a response"""
exampleobj = None
if 'examples' in response:
exampleobj = response["examples"].get("application/json")
# the openapi spec suggests that examples in the 'examples' section should
# be formatted as raw objects rather than json-formatted strings, but we
# have lots of the latter in our spec, which work with the swagger UI,
# so grandfather them in.
if isinstance(exampleobj, basestring):
return exampleobj
if exampleobj is None:
schema = response.get('schema')
if schema:
if schema['type'] == 'file':
# no example for 'file' responses
return None
exampleobj = get_example_for_schema(schema)
if exampleobj is None:
return None
return json.dumps(exampleobj, indent=2)
class MatrixUnits(Units): class MatrixUnits(Units):
def _load_swagger_meta(self, api, group_name): def _load_swagger_meta(self, api, group_name):
@ -349,40 +392,68 @@ class MatrixUnits(Units):
"rate_limited": 429 in single_api.get("responses", {}), "rate_limited": 429 in single_api.get("responses", {}),
"req_param_by_loc": {}, "req_param_by_loc": {},
"req_body_tables": [], "req_body_tables": [],
"res_headers": [],
"res_tables": [], "res_tables": [],
"responses": [], "responses": [],
"example": { "example": {
"req": "", "req": "",
} }
} }
self.log(" ------- Endpoint: %s %s ------- " % (method, path)) logger.info(" ------- Endpoint: %s %s ------- " % (method, path))
for param in single_api.get("parameters", []):
param_loc = param["in"]
if param_loc == "body":
self._handle_body_param(param, endpoint)
continue
path_template = api.get("basePath", "").rstrip("/") + path
example_query_params = []
example_body = ""
for param in single_api.get("parameters", []):
# even body params should have names, otherwise the active docs don't work.
param_name = param["name"] param_name = param["name"]
# description try:
desc = param.get("description", "") param_loc = param["in"]
if param.get("required"):
desc = "**Required.** " + desc
# assign value expected for this param if param_loc == "body":
val_type = param.get("type") # integer/string self._handle_body_param(param, endpoint)
example_body = get_example_for_param(param)
continue
if param.get("enum"): # description
val_type = "enum" desc = param.get("description", "")
desc += ( if param.get("required"):
" One of: %s" % json.dumps(param.get("enum")) desc = "**Required.** " + desc
)
endpoint["req_param_by_loc"].setdefault(param_loc, []).append({ # assign value expected for this param
"key": param_name, val_type = param.get("type") # integer/string
"type": val_type,
"desc": desc if param.get("enum"):
}) val_type = "enum"
desc += (
" One of: %s" % json.dumps(param.get("enum"))
)
endpoint["req_param_by_loc"].setdefault(param_loc, []).append({
"key": param_name,
"type": val_type,
"desc": desc
})
example = get_example_for_param(param)
if example is None:
continue
if param_loc == "path":
path_template = path_template.replace(
"{%s}" % param_name, urllib.quote(example)
)
elif param_loc == "query":
if type(example) == list:
for value in example:
example_query_params.append((param_name, value))
else:
example_query_params.append((param_name, example))
except Exception, e:
raise Exception("Error handling parameter %s" % param_name, e)
# endfor[param] # endfor[param]
good_response = None good_response = None
@ -391,111 +462,39 @@ class MatrixUnits(Units):
if not good_response and code == 200: if not good_response and code == 200:
good_response = res good_response = res
description = res.get("description", "") description = res.get("description", "")
example = res.get("examples", {}).get("application/json", "") example = get_example_for_response(res)
endpoint["responses"].append({ endpoint["responses"].append({
"code": code, "code": code,
"description": description, "description": description,
"example": example, "example": example,
}) })
path_template = api.get("basePath", "").rstrip("/") + path # add response params if this API has any.
qps = [] if good_response:
body = "" if "schema" in good_response:
for param in single_api.get("parameters", []): endpoint["res_tables"] = get_tables_for_response(
try: good_response["schema"]
example = get_example_for_param(param) )
if "headers" in good_response:
if not example: headers = []
self.log( for (header_name, header) in good_response["headers"].iteritems():
"The parameter %s is missing an example." % headers.append({
param["name"]) "key": header_name,
continue "type": header["type"],
"desc": header["description"],
if param["in"] == "path": })
path_template = path_template.replace( endpoint["res_headers"] = headers
"{%s}" % param["name"], urllib.quote(example)
) query_string = "" if len(example_query_params) == 0 else "?"+urllib.urlencode(example_query_params)
elif param["in"] == "body": if example_body:
body = example
elif param["in"] == "query":
if type(example) == list:
for value in example:
qps.append((param["name"], value))
else:
qps.append((param["name"], example))
except Exception, e:
raise Exception("Error handling parameter %s" % param["name"],
e)
query_string = "" if len(qps) == 0 else "?"+urllib.urlencode(qps)
if body:
endpoint["example"]["req"] = "%s %s%s HTTP/1.1\nContent-Type: application/json\n\n%s" % ( endpoint["example"]["req"] = "%s %s%s HTTP/1.1\nContent-Type: application/json\n\n%s" % (
method.upper(), path_template, query_string, body method.upper(), path_template, query_string, example_body
) )
else: else:
endpoint["example"]["req"] = "%s %s%s HTTP/1.1\n\n" % ( endpoint["example"]["req"] = "%s %s%s HTTP/1.1\n\n" % (
method.upper(), path_template, query_string method.upper(), path_template, query_string
) )
# add response params if this API has any.
if good_response:
self.log("Found a 200 response for this API")
res_type = Units.prop(good_response, "schema/type")
res_name = Units.prop(good_response, "schema/name")
if res_type and res_type not in ["object", "array"]:
# response is a raw string or something like that
good_table = {
"title": None,
"rows": [{
"key": "<" + res_type + ">" if not res_name else res_name,
"type": res_type,
"desc": res.get("description", ""),
"req_str": ""
}]
}
if good_response.get("headers"):
for (header_name, header) in good_response.get("headers").iteritems():
good_table["rows"].append({
"key": header_name,
"type": "Header<" + header["type"] + ">",
"desc": header["description"],
"req_str": ""
})
endpoint["res_tables"].append(good_table)
elif res_type and Units.prop(good_response, "schema/properties"):
# response is an object:
schema = good_response["schema"]
res_tables = get_tables_for_schema(schema)
endpoint["res_tables"].extend(res_tables)
elif res_type and Units.prop(good_response, "schema/items"):
# response is an array:
# FIXME: Doesn't recurse at all.
schema = good_response["schema"]
array_type = Units.prop(schema, "items/type")
if Units.prop(schema, "items/allOf"):
array_type = (
Units.prop(schema, "items/title")
)
endpoint["res_tables"].append({
"title": schema.get("title", ""),
"rows": [{
"key": "N/A",
"type": ("[%s]" % array_type),
"desc": schema.get("description", ""),
"req_str": ""
}]
})
for response_table in endpoint["res_tables"]:
self.log("Response: %s" % response_table["title"])
for r in response_table["rows"]:
self.log("Row: %s" % r)
if len(endpoint["res_tables"]) == 0:
self.log(
"This API appears to have no response table. Are you " +
"sure this API returns no parameters?"
)
endpoints.append(endpoint) endpoints.append(endpoint)
return { return {
@ -512,22 +511,34 @@ class MatrixUnits(Units):
:param dict endpoint_data dictionary of endpoint data to be updated :param dict endpoint_data dictionary of endpoint data to be updated
""" """
try: try:
req_body_tables = get_tables_for_schema(param["schema"]) schema = inherit_parents(param["schema"])
except Exception, e: if schema["type"] != "object":
logger.warning("Error decoding body of API endpoint %s %s" % logger.warn(
(endpoint_data["method"], endpoint_data["path"]), "Unsupported body type %s for %s %s", schema["type"],
exc_info=1) endpoint_data["method"], endpoint_data["path"]
return )
return
# put the top-level parameters into 'req_param_by_loc', and the others req_body_tables = get_tables_for_schema(schema)
# into 'req_body_tables'
body_params = endpoint_data['req_param_by_loc'].setdefault("JSON body",[])
body_params.extend(req_body_tables[0]["rows"])
body_tables = req_body_tables[1:] if req_body_tables == []:
# TODO: remove this when PR #255 has landed # no fields defined for the body.
body_tables = (t for t in body_tables if not t.get('no-table')) return
endpoint_data['req_body_tables'].extend(body_tables)
# put the top-level parameters into 'req_param_by_loc', and the others
# into 'req_body_tables'
body_params = endpoint_data['req_param_by_loc'].setdefault("JSON body",[])
body_params.extend(req_body_tables[0]["rows"])
body_tables = req_body_tables[1:]
endpoint_data['req_body_tables'].extend(body_tables)
except Exception, e:
e2 = Exception(
"Error decoding body of API endpoint %s %s: %s" %
(endpoint_data["method"], endpoint_data["path"], e)
)
raise e2, None, sys.exc_info()[2]
def load_swagger_apis(self): def load_swagger_apis(self):
@ -536,13 +547,13 @@ class MatrixUnits(Units):
for filename in os.listdir(path): for filename in os.listdir(path):
if not filename.endswith(".yaml"): if not filename.endswith(".yaml"):
continue continue
self.log("Reading swagger API: %s" % filename) logger.info("Reading swagger API: %s" % filename)
filepath = os.path.join(path, filename) filepath = os.path.join(path, filename)
with open(filepath, "r") as f: with open(filepath, "r") as f:
# strip .yaml # strip .yaml
group_name = filename[:-5].replace("-", "_") group_name = filename[:-5].replace("-", "_")
group_name = "%s_%s" % (group_name, suffix) group_name = "%s_%s" % (group_name, suffix)
api = yaml.load(f.read()) api = yaml.load(f.read(), OrderedLoader)
api = resolve_references(filepath, api) api = resolve_references(filepath, api)
api["__meta"] = self._load_swagger_meta( api["__meta"] = self._load_swagger_meta(
api, group_name api, group_name
@ -563,7 +574,7 @@ class MatrixUnits(Units):
filepath = os.path.join(root, filename) filepath = os.path.join(root, filename)
with open(filepath) as f: with open(filepath) as f:
try: try:
event_info = yaml.load(f) event_info = yaml.load(f, OrderedLoader)
except Exception as e: except Exception as e:
raise ValueError( raise ValueError(
"Error reading file %r" % (filepath,), e "Error reading file %r" % (filepath,), e
@ -653,13 +664,13 @@ class MatrixUnits(Units):
return schemata return schemata
def read_event_schema(self, filepath): def read_event_schema(self, filepath):
self.log("Reading %s" % filepath) logger.info("Reading %s" % filepath)
with open(filepath, "r") as f: with open(filepath, "r") as f:
json_schema = yaml.load(f) json_schema = yaml.load(f, OrderedLoader)
schema = { schema = {
"typeof": None, "typeof": "",
"typeof_info": "", "typeof_info": "",
"type": None, "type": None,
"title": None, "title": None,
@ -682,11 +693,9 @@ class MatrixUnits(Units):
STATE_EVENT: "State Event" STATE_EVENT: "State Event"
} }
if type(json_schema.get("allOf")) == list: if type(json_schema.get("allOf")) == list:
schema["typeof"] = base_defs.get( firstRef = json_schema["allOf"][0]["$ref"]
json_schema["allOf"][0].get("$ref") if firstRef in base_defs:
) schema["typeof"] = base_defs[firstRef]
elif json_schema.get("title"):
schema["typeof"] = json_schema["title"]
json_schema = resolve_references(filepath, json_schema) json_schema = resolve_references(filepath, json_schema)

Loading…
Cancel
Save