From 2f039a114201887f7b2da8b4f8a334bcd9f8f515 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 24 Sep 2015 15:39:17 +0100 Subject: [PATCH 01/68] Add swagger docs for the v2 filter POST API --- api/client-server/v2_alpha/filter.yaml | 188 +++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 api/client-server/v2_alpha/filter.yaml diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml new file mode 100644 index 00000000..99dc0ebc --- /dev/null +++ b/api/client-server/v2_alpha/filter.yaml @@ -0,0 +1,188 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v2 filter API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https +basePath: /_matrix/client/api/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +definitions: + EventFilter: + title: + type: object + properties: + types: + type: array + description: |- + A list of event types to include. + If this list is absent then all event types are included. + items: + type: string + not_types: + type: array + description: |- + A list of event types to exclude. + If this list is absent then no event types are excluded. + items: + type: string + senders: + type: array + description: |- + A list of senders IDs to include. + If this list is absent then all senders are included. + items: + type: string + not_senders: + type: array + description: |- + A list of sender IDs to exclude + If this list is absent then no senders are excluded. + items: + type: string + rooms: + type: array + description: |- + A list of room IDs to include. + If this list is absent then all rooms are included. + items: + type: string + not_rooms: + type: array + description: |- + A list of room IDs to exclude + If this list is absent then no rooms are excluded. + items: + type: string + SyncFilter: + room: + type: object + properties: + state: + description: + The state events to include for rooms. + allOf: + - $ref: "#/definitions/EventFilter" + events: + description: + The message and state update events to include for rooms. + allOf: + - $ref: "#/definitions/EventFilter" + ephemeral: + description: |- + The events that aren't recorded in the permenant history, e.g. + typing and receipts, to include for rooms. + allOf: + - $ref: "#/definitions/EventFilter" + public_user_data: + description: |- + The public user data, e.g. profile and presence, to include. + allOf: + - $ref: "#/definitions/EventFilter" + private_user_data: + description: |- + Events that are private to a user but shared amoungst their devices, + e.g. notification settings, to include. + allOf: + - $ref: "#/definitions/EventFilter" + event_format: + description: |- + The format to use for events. "client" will return the events in a + format suitable for clients. "federation" will return the raw event + as receieved over federation. The default is "client". + type: string + event_fields: + type: array + description: |- + List of event fields to include. If this list is absent then all fields + are included. The entries may include "." charaters to indicate + sub-fields. So ["content.body"] will include the "body" field of the + "content" object. A server may include more fields than were requested. + items: + string +paths: + "/user/{userId}/filter": + post: + summary: Upload a new filter. + description: |- + Uploads a new filter definition to the homeserver. + Returns a filter ID that may be used in /sync requests to + retrict which events are returned to the client. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + required: true + description: + The id of the user uploading the filter. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: body + name: filter + required: true + description: The filter to upload. + schema: + type: object + allOf: + - $ref: "#/definitions/SyncFilter" + example: + type: object + example: |- + { + "room": { + "state": { + "types": ["m.room.*"], + "not_rooms": ["!726s6s6q:example.com"], + }, + "events": { + "types": ["m.room.message"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + }, + "emphemeral": { + "types": ["m.receipt", "m.typing"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + } + }, + "public_user_data": { + "types": ["m.presence"] + }, + "private_user_data": { + "types": [] + }, + "server_data": { + "types": [] + }, + "event_format": "client", + "event_fields": ["type", "content", "sender"] + } + responses: + 200: + description: The filter was created. + examples: + application/json: |- + { + "filter_id": "66696p746572" + } + schema: + type: object + properties: + filter_id: + type: string + description: + The ID of the filter that was created. + + + From 883105eae69d51e8672f63b9aa48946b6ae5347b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 24 Sep 2015 16:25:03 +0100 Subject: [PATCH 02/68] Document the v2 filter GET API --- .../v2_alpha/definitions/definitions | 1 + .../v2_alpha/definitions/event_filter.json | 53 ++++++ .../v2_alpha/definitions/sync_filter.json | 48 ++++++ api/client-server/v2_alpha/filter.yaml | 158 +++++++----------- 4 files changed, 159 insertions(+), 101 deletions(-) create mode 120000 api/client-server/v2_alpha/definitions/definitions create mode 100644 api/client-server/v2_alpha/definitions/event_filter.json create mode 100644 api/client-server/v2_alpha/definitions/sync_filter.json diff --git a/api/client-server/v2_alpha/definitions/definitions b/api/client-server/v2_alpha/definitions/definitions new file mode 120000 index 00000000..945c9b46 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/definitions @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/event_filter.json b/api/client-server/v2_alpha/definitions/event_filter.json new file mode 100644 index 00000000..b8ab1352 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/event_filter.json @@ -0,0 +1,53 @@ +{ + "type": "object", + "properties": { + "types": { + "type": "array", + "description": + "A list of event types to include. If this list is absent then all event types are included.", + "items": { + "type": "string" + } + }, + "not_types": { + "type": "array", + "description": + "A list of event types to exclude. If this list is absent then no event types are excluded.", + "items": { + "type": "string" + } + }, + "senders": { + "type": "array", + "description": + "A list of senders IDs to include. If this list is absent then all senders are included.", + "items": { + "type": "string" + } + }, + "not_senders": { + "type": "array", + "description": + "A list of sender IDs to exclude. If this list is absent then no senders are excluded.", + "items": { + "type": "string" + } + }, + "rooms": { + "type": "array", + "description": + "A list of room IDs to include. If this list is absent then all rooms are included.", + "items": { + "type": "string" + } + }, + "not_rooms": { + "type": "array", + "description": + "A list of room IDs to exclude. If this list is absent then no rooms are excluded.", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/v2_alpha/definitions/sync_filter.json new file mode 100644 index 00000000..c9c47646 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/sync_filter.json @@ -0,0 +1,48 @@ +{ + "type": "object", + "properties": { + "room": { + "type": "object", + "properties": { + "state": { + "description": + "The state events to include for rooms.", + "allOf": [{"$ref": "definitions/event_filter.json"}] + }, + "events": { + "description": + "The message and state update events to include for rooms.", + "allOf": [{"$ref": "definitions/event_filter.json"}] + }, + "ephemeral": { + "description": + "The events that aren't recorded in the room history, e.g. typing and receipts, to include for rooms.", + "allOf": [{"$ref": "definitions/event_filter.json"}] + } + } + }, + "public_user_data": { + "description": + "The public user data, e.g. profile and presence, to include.", + "allOf": [{"$ref": "definitions/event_filter.json"}] + }, + "private_user_data": { + "description": + "Events that are private to a user but shared amoungst their devices, e.g. notification settings, to include.", + "allOf": [{"$ref": "definitions/event_filter.json"}] + }, + "event_format": { + "description": + "The format to use for events. 'client' will return the events in a format suitable for clients. 'federation' will return the raw event as receieved over federation. The default is 'client'.", + "type": "string" + }, + "event_fields": { + "type": "array", + "description": + "List of event fields to include. If this list is absent then all fields are included. The entries may include '.' charaters to indicate sub-fields. So ['content.body'] will include the 'body' field of the 'content' object. A server may include more fields than were requested.", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml index 99dc0ebc..115ef230 100644 --- a/api/client-server/v2_alpha/filter.yaml +++ b/api/client-server/v2_alpha/filter.yaml @@ -16,103 +16,10 @@ securityDefinitions: description: The user_id or application service access_token name: access_token in: query -definitions: - EventFilter: - title: - type: object - properties: - types: - type: array - description: |- - A list of event types to include. - If this list is absent then all event types are included. - items: - type: string - not_types: - type: array - description: |- - A list of event types to exclude. - If this list is absent then no event types are excluded. - items: - type: string - senders: - type: array - description: |- - A list of senders IDs to include. - If this list is absent then all senders are included. - items: - type: string - not_senders: - type: array - description: |- - A list of sender IDs to exclude - If this list is absent then no senders are excluded. - items: - type: string - rooms: - type: array - description: |- - A list of room IDs to include. - If this list is absent then all rooms are included. - items: - type: string - not_rooms: - type: array - description: |- - A list of room IDs to exclude - If this list is absent then no rooms are excluded. - items: - type: string - SyncFilter: - room: - type: object - properties: - state: - description: - The state events to include for rooms. - allOf: - - $ref: "#/definitions/EventFilter" - events: - description: - The message and state update events to include for rooms. - allOf: - - $ref: "#/definitions/EventFilter" - ephemeral: - description: |- - The events that aren't recorded in the permenant history, e.g. - typing and receipts, to include for rooms. - allOf: - - $ref: "#/definitions/EventFilter" - public_user_data: - description: |- - The public user data, e.g. profile and presence, to include. - allOf: - - $ref: "#/definitions/EventFilter" - private_user_data: - description: |- - Events that are private to a user but shared amoungst their devices, - e.g. notification settings, to include. - allOf: - - $ref: "#/definitions/EventFilter" - event_format: - description: |- - The format to use for events. "client" will return the events in a - format suitable for clients. "federation" will return the raw event - as receieved over federation. The default is "client". - type: string - event_fields: - type: array - description: |- - List of event fields to include. If this list is absent then all fields - are included. The entries may include "." charaters to indicate - sub-fields. So ["content.body"] will include the "body" field of the - "content" object. A server may include more fields than were requested. - items: - string paths: - "/user/{userId}/filter": - post: - summary: Upload a new filter. + "/user/{userId}/filter": + post: + summary: Upload a new filter. description: |- Uploads a new filter definition to the homeserver. Returns a filter ID that may be used in /sync requests to @@ -135,7 +42,7 @@ paths: schema: type: object allOf: - - $ref: "#/definitions/SyncFilter" + - $ref: "definitions/sync_filter.json" example: type: object example: |- @@ -181,8 +88,57 @@ paths: properties: filter_id: type: string - description: + description: |- The ID of the filter that was created. - - - + "/user/{userId}/filter/{filterId}": + get: + summary: Download a filter + parameters: + - in: path + name: userId + type: string + description: |- + The user ID to download a filter for. + x-example: "@alice:example.com" + - in: path + name: filterId + description: |- + The filter ID to download. + x-example: "66696p746572" + responses: + 200: + examples: + application/json: |- + { + "room": { + "state": { + "types": ["m.room.*"], + "not_rooms": ["!726s6s6q:example.com"] + }, + "events": { + "types": ["m.room.message"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + }, + "emphemeral": { + "types": ["m.receipt", "m.typing"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + } + }, + "public_user_data": { + "types": ["m.presence"] + }, + "private_user_data": { + "types": [] + }, + "server_data": { + "types": [] + }, + "event_format": "client", + "event_fields": ["type", "content", "sender"] + } + schema: + type: object + allOf: + - $ref: "definitions/sync_filter.json" From 69298b96121182e44a47a29eab3c69de5e91f37a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 24 Sep 2015 16:29:51 +0100 Subject: [PATCH 03/68] Check "v2_alpha" in jenkins --- api/client-server/v2_alpha/filter.yaml | 5 +++++ jenkins.sh | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml index 115ef230..f53ef2ba 100644 --- a/api/client-server/v2_alpha/filter.yaml +++ b/api/client-server/v2_alpha/filter.yaml @@ -100,13 +100,18 @@ paths: description: |- The user ID to download a filter for. x-example: "@alice:example.com" + required: true - in: path name: filterId + type: string description: |- The filter ID to download. x-example: "66696p746572" + required: true responses: 200: + description: |- + "The filter defintion" examples: application/json: |- { diff --git a/jenkins.sh b/jenkins.sh index e1043644..0936de9d 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -5,5 +5,5 @@ set -ex (cd event-schemas/ && ./check_examples.py) (cd api && ./check_examples.py) (cd scripts && ./gendoc.py) -(cd api && npm install && node validator.js -s "client-server/v1") +(cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") (cd event-schemas/ && ./check.sh) From a31a446661262872ba668c2c3192c5cf0a180387 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 24 Sep 2015 16:48:00 +0100 Subject: [PATCH 04/68] Fix the POST example for the v2 filter API --- api/client-server/v2_alpha/filter.yaml | 58 +++++++++++++------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml index f53ef2ba..5296f11a 100644 --- a/api/client-server/v2_alpha/filter.yaml +++ b/api/client-server/v2_alpha/filter.yaml @@ -43,38 +43,36 @@ paths: type: object allOf: - $ref: "definitions/sync_filter.json" - example: - type: object - example: |- - { - "room": { - "state": { - "types": ["m.room.*"], - "not_rooms": ["!726s6s6q:example.com"], - }, - "events": { - "types": ["m.room.message"], - "not_rooms": ["!726s6s6q:example.com"], - "not_senders": ["@spam:example.com"] - }, - "emphemeral": { - "types": ["m.receipt", "m.typing"], - "not_rooms": ["!726s6s6q:example.com"], - "not_senders": ["@spam:example.com"] - } - }, - "public_user_data": { - "types": ["m.presence"] - }, - "private_user_data": { - "types": [] + example: |- + { + "room": { + "state": { + "types": ["m.room.*"], + "not_rooms": ["!726s6s6q:example.com"] }, - "server_data": { - "types": [] + "events": { + "types": ["m.room.message"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] }, - "event_format": "client", - "event_fields": ["type", "content", "sender"] - } + "emphemeral": { + "types": ["m.receipt", "m.typing"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + } + }, + "public_user_data": { + "types": ["m.presence"] + }, + "private_user_data": { + "types": [] + }, + "server_data": { + "types": [] + }, + "event_format": "client", + "event_fields": ["type", "content", "sender"] + } responses: 200: description: The filter was created. From 078dd0165ffe045229b0efee92ec8f425cfd28df Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 25 Sep 2015 11:58:47 +0100 Subject: [PATCH 05/68] Update the room creation API spec to include new keys: 'preset' and 'initial_state' --- specification/1-client_server_api.rst | 33 ++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 93e3cb90..e3dbde5a 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -750,10 +750,41 @@ options which can be set when creating a room: This will tell the server to invite everyone in the list to the newly created room. +``preset`` + Type: + String + Optional: + Yes + Value: + ``private_chat`` or ``public_chat`` + Description: + Convenience parameter for setting various default state events based on a + preset. + + Two presets are defined: + + - ``private_chat``: Sets the ``join_rules`` to ``invite`` and + ``history_visibility`` to ``shared`` + - ``public_chat``: Sets the ``join_rules`` to ``public`` and + ``history_visibility`` to ``shared`` + +``initial_state`` + Type: + List + Optional: + Yes + Value: + A list of state events to set in the new room. + Description: + Allows the user to override the default state events set in the new room. + + The expected format of the state events are an object with ``type``, + ``state_key`` and ``content`` keys set. + Example:: { - "visibility": "public", + "preset": "public_chat", "room_alias_name": "thepub", "name": "The Grand Duke Pub", "topic": "All about happy hour" From 18dc7784df09c3ad6c3f4d844b87d046538bd5a5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 25 Sep 2015 14:34:06 +0100 Subject: [PATCH 06/68] Mention precedence --- specification/1-client_server_api.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index e3dbde5a..c5453931 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -781,6 +781,9 @@ options which can be set when creating a room: The expected format of the state events are an object with ``type``, ``state_key`` and ``content`` keys set. + Takes precedence over events set by ``presets``, but gets overriden by + ``name`` and ``topic`` keys. + Example:: { From e1f73f523318951d2303e2d538b26fb3478abeda Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 25 Sep 2015 18:09:17 +0100 Subject: [PATCH 07/68] Add a limit to filters --- api/client-server/v2_alpha/definitions/event_filter.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/client-server/v2_alpha/definitions/event_filter.json b/api/client-server/v2_alpha/definitions/event_filter.json index b8ab1352..76d8be64 100644 --- a/api/client-server/v2_alpha/definitions/event_filter.json +++ b/api/client-server/v2_alpha/definitions/event_filter.json @@ -1,6 +1,11 @@ { "type": "object", "properties": { + "limit": { + "type": "integer", + "description": + "The maximum number of events to return." + }, "types": { "type": "array", "description": From a0068e1adaa028d29bc00d5ad62dd5a5b6371b1b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 25 Sep 2015 18:10:28 +0100 Subject: [PATCH 08/68] Draft documention for the v2 sync api --- .../v2_alpha/definitions/event_batch.json | 12 ++ .../v2_alpha/definitions/timeline_batch.json | 14 ++ api/client-server/v2_alpha/sync.yaml | 167 ++++++++++++++++++ 3 files changed, 193 insertions(+) create mode 100644 api/client-server/v2_alpha/definitions/event_batch.json create mode 100644 api/client-server/v2_alpha/definitions/timeline_batch.json create mode 100644 api/client-server/v2_alpha/sync.yaml diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/v2_alpha/definitions/event_batch.json new file mode 100644 index 00000000..c7cdbfb3 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/event_batch.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "properties": { + "events": { + "type": "array", + "description": "List of indicies into an events array", + "items": { + "type": "integer" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/v2_alpha/definitions/timeline_batch.json new file mode 100644 index 00000000..dc62724b --- /dev/null +++ b/api/client-server/v2_alpha/definitions/timeline_batch.json @@ -0,0 +1,14 @@ +{ + "type": "object", + "allOf": [{"$ref":"definitions/event_batch.json"}], + "properties": { + "limited": { + "type": "boolean", + "description": "Whether there are more events on the server" + }, + "prev_batch": { + "type": "string", + "description": "If the batch was limited then this is a token that can be supplied to the server to retrieve more events" + } + } +} diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml new file mode 100644 index 00000000..c857d272 --- /dev/null +++ b/api/client-server/v2_alpha/sync.yaml @@ -0,0 +1,167 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v2 sync API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https +basePath: /_matrix/client/api/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/sync": + get: + summary: Synchronise the client's state and receive new messages. + description: |- + Synchronise the client's state with the latest state on the server. + Client's use this API when they first log in to get an initial snapshot + of the state on the server, and then continue to call this API to get + incremental deltas to the state, and to receive new messages. + security: + - accessToken: [] + parameters: + - in: query + name: filter + type: string + description: |- + The ID of a filter created using the filter API. + - in: query + name: since + type: string + description: |- + A point in time to continue a sync from. + - in: query + name: timeout + type: integer + description: |- + The maximum time to poll in milliseconds before returning this + request. + responses: + 200: + description: + The initial snapshot or delta for the client to use to update their + state. + schema: + type: object + properties: + next_batch: + type: string + description: |- + The batch token to supply in the ``since`` param of the next + ``/sync`` request. + events: + type: array + description: |- + A list of event objects that are referred to by index in the + rest of the ``sync`` response. + items: + type: object + rooms: + type: array + description: |- + A list of rooms that the client needs to update. + items: + type: object + properties: + room_id: + type: string + description: |- + The ID of the room. + state: + description: |- + The state updates for the room. + allOf: + - $ref: "definitions/event_batch.json" + timeline: + description: |- + The timeline of messages and state changes in the room. + allOf: + - $ref: "definitions/timeline_batch.json" + ephemeral: + description: |- + The ephemeral events in the room that aren't recorded + in the timeline or state of the room. E.g. typing. + allOf: + - $ref: "definitions/event_batch.json" + public_user_data: + description: |- + The updates to publically visible user data. + allOf: + - $ref: "definitions/event_batch.json" + private_user_data: + description: |- + Updates to the data which is private to the user but shared + amoungst their devices. + allOf: + - $ref: "definitions/event_batch.json" + examples: + application/json: |- + { + "next_batch": "s72595_4483_1934", + "events": [ + { + "sender": "@bob:example.com", + "type": "com.example.weird.setting", + "content": {"setting1": true, "setting2": false} + }, + { + "sender": "@alice:example.com", + "type": "m.profile.display_name", + "content": {"display_name": "Alice"} + }, + { + "sender": "@alice:example.com", + "type": "m.presence", + "content": {"presence": "online"} + }, + { + "room_id": "!726s6s6q:example.com", + "type": "m.typing", + "content": {"user_ids": ["@alice:example.com"]} + }, + { + "sender": "@alice:example.com", + "room_id": "!726s6s6q:example.com", + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + { + "sender": "@bob:example.com", + "room_id": "!726s6s6q:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + { + "sender": "@alice:example.com", + "room_id": "!726s6s6q:example.com", + "type": "m.room.message", + "unsigned": {"age": "124524", "txn_id": "1234"}, + "content": {"body": "I am a fish", "msgtype": "m.text"}, + "origin_server_ts": 1417731086797 + } + ], + "private_user_data": {"events": [0]}, + "public_user_data": {"events": [1, 2]}, + "rooms": [{ + "room_id": "!726s6s6q:example.com", + "ephemeral": {"events": [3]}, + "state": {"events": [4, 5]}, + "timeline": { + "events": [5,6], + "limited": true, + "prev_batch": "t34-23535_0_0" + } + }] + } From de07586ab786d8cc8431dfc1b23030e9a77d6b57 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 25 Sep 2015 19:34:55 +0100 Subject: [PATCH 09/68] Rename 'events' to 'timeline' in the sync filter --- api/client-server/v2_alpha/definitions/sync_filter.json | 2 +- api/client-server/v2_alpha/filter.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/v2_alpha/definitions/sync_filter.json index c9c47646..bfc8b51d 100644 --- a/api/client-server/v2_alpha/definitions/sync_filter.json +++ b/api/client-server/v2_alpha/definitions/sync_filter.json @@ -9,7 +9,7 @@ "The state events to include for rooms.", "allOf": [{"$ref": "definitions/event_filter.json"}] }, - "events": { + "timeline": { "description": "The message and state update events to include for rooms.", "allOf": [{"$ref": "definitions/event_filter.json"}] diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml index 5296f11a..3e564f4b 100644 --- a/api/client-server/v2_alpha/filter.yaml +++ b/api/client-server/v2_alpha/filter.yaml @@ -50,7 +50,7 @@ paths: "types": ["m.room.*"], "not_rooms": ["!726s6s6q:example.com"] }, - "events": { + "timeline": { "types": ["m.room.message"], "not_rooms": ["!726s6s6q:example.com"], "not_senders": ["@spam:example.com"] @@ -118,7 +118,7 @@ paths: "types": ["m.room.*"], "not_rooms": ["!726s6s6q:example.com"] }, - "events": { + "timeline": { "types": ["m.room.message"], "not_rooms": ["!726s6s6q:example.com"], "not_senders": ["@spam:example.com"] From c3b3b2df63ed8964d72517d0092b298988009d3f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 25 Sep 2015 19:37:41 +0100 Subject: [PATCH 10/68] Add "set_presence" in the sync parameters --- api/client-server/v2_alpha/sync.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index c857d272..5018080e 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -38,6 +38,11 @@ paths: type: string description: |- A point in time to continue a sync from. + - in: query + name: set_presence + type: string + description: |- + Set the presence status of this client. - in: query name: timeout type: integer From 393d2831399e07316686129418d21b671571bfe7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 11:22:31 +0100 Subject: [PATCH 11/68] Add a "limit" to the timeline key in the example filter --- api/client-server/v2_alpha/filter.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml index 3e564f4b..9bda2359 100644 --- a/api/client-server/v2_alpha/filter.yaml +++ b/api/client-server/v2_alpha/filter.yaml @@ -51,6 +51,7 @@ paths: "not_rooms": ["!726s6s6q:example.com"] }, "timeline": { + "limit": 10, "types": ["m.room.message"], "not_rooms": ["!726s6s6q:example.com"], "not_senders": ["@spam:example.com"] @@ -119,6 +120,7 @@ paths: "not_rooms": ["!726s6s6q:example.com"] }, "timeline": { + "limit": 10, "types": ["m.room.message"], "not_rooms": ["!726s6s6q:example.com"], "not_senders": ["@spam:example.com"] From 1aa916d690bac548625f5a460ad916a0c6fdf43d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 12:52:12 +0100 Subject: [PATCH 12/68] s/indicies/indices/ --- api/client-server/v2_alpha/definitions/event_batch.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/v2_alpha/definitions/event_batch.json index c7cdbfb3..ed2730c8 100644 --- a/api/client-server/v2_alpha/definitions/event_batch.json +++ b/api/client-server/v2_alpha/definitions/event_batch.json @@ -3,7 +3,7 @@ "properties": { "events": { "type": "array", - "description": "List of indicies into an events array", + "description": "List of indices into an events array", "items": { "type": "integer" } From f50e6d4c0ad8648dc601344b6ac67c468f4e25f8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 13:02:52 +0100 Subject: [PATCH 13/68] Add x-example fields for v2 /sync --- api/client-server/v2_alpha/sync.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 5018080e..d307a683 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -33,22 +33,26 @@ paths: type: string description: |- The ID of a filter created using the filter API. + x-example: "66696p746572" - in: query name: since type: string description: |- A point in time to continue a sync from. + x-example: "s72594_4483_1934" - in: query name: set_presence type: string description: |- Set the presence status of this client. + x-example: "online" - in: query name: timeout type: integer description: |- The maximum time to poll in milliseconds before returning this request. + x-example: 30000 responses: 200: description: From 3204c2f2b6cedf748618c594b86c514c2bf0b2b2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 13:04:37 +0100 Subject: [PATCH 14/68] Fix spelling --- api/client-server/v2_alpha/sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index d307a683..318a1ab5 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -108,7 +108,7 @@ paths: private_user_data: description: |- Updates to the data which is private to the user but shared - amoungst their devices. + amongst their devices. allOf: - $ref: "definitions/event_batch.json" examples: From 00fd4aac264619ec6a53afecd0757149ad7c038c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 13:06:04 +0100 Subject: [PATCH 15/68] s/publically/publicly/ --- api/client-server/v2_alpha/sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 318a1ab5..e678d2d8 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -102,7 +102,7 @@ paths: - $ref: "definitions/event_batch.json" public_user_data: description: |- - The updates to publically visible user data. + The updates to publicly visible user data. allOf: - $ref: "definitions/event_batch.json" private_user_data: From 41bc09ea22e8b18d0e509e666e30033dd3143e04 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 13:20:01 +0100 Subject: [PATCH 16/68] Split the event_filter into a event_filter and a room_event_filter that extends it. So that we don't include "rooms" and "not_rooms" keys for the public_user_data and private_user_data filters. --- .../v2_alpha/definitions/event_filter.json | 16 -------------- .../definitions/room_event_filter.json | 21 +++++++++++++++++++ .../v2_alpha/definitions/sync_filter.json | 6 +++--- 3 files changed, 24 insertions(+), 19 deletions(-) create mode 100644 api/client-server/v2_alpha/definitions/room_event_filter.json diff --git a/api/client-server/v2_alpha/definitions/event_filter.json b/api/client-server/v2_alpha/definitions/event_filter.json index 76d8be64..c15b8133 100644 --- a/api/client-server/v2_alpha/definitions/event_filter.json +++ b/api/client-server/v2_alpha/definitions/event_filter.json @@ -37,22 +37,6 @@ "items": { "type": "string" } - }, - "rooms": { - "type": "array", - "description": - "A list of room IDs to include. If this list is absent then all rooms are included.", - "items": { - "type": "string" - } - }, - "not_rooms": { - "type": "array", - "description": - "A list of room IDs to exclude. If this list is absent then no rooms are excluded.", - "items": { - "type": "string" - } } } } diff --git a/api/client-server/v2_alpha/definitions/room_event_filter.json b/api/client-server/v2_alpha/definitions/room_event_filter.json new file mode 100644 index 00000000..c234448e --- /dev/null +++ b/api/client-server/v2_alpha/definitions/room_event_filter.json @@ -0,0 +1,21 @@ +{ + "type": "object", + "properties": { + "rooms": { + "type": "array", + "description": + "A list of room IDs to include. If this list is absent then all rooms are included.", + "items": { + "type": "string" + } + }, + "not_rooms": { + "type": "array", + "description": + "A list of room IDs to exclude. If this list is absent then no rooms are excluded.", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/v2_alpha/definitions/sync_filter.json index bfc8b51d..4ad3aa82 100644 --- a/api/client-server/v2_alpha/definitions/sync_filter.json +++ b/api/client-server/v2_alpha/definitions/sync_filter.json @@ -7,17 +7,17 @@ "state": { "description": "The state events to include for rooms.", - "allOf": [{"$ref": "definitions/event_filter.json"}] + "allOf": [{"$ref": "definitions/room_event_filter.json"}] }, "timeline": { "description": "The message and state update events to include for rooms.", - "allOf": [{"$ref": "definitions/event_filter.json"}] + "allOf": [{"$ref": "definitions/room_event_filter.json"}] }, "ephemeral": { "description": "The events that aren't recorded in the room history, e.g. typing and receipts, to include for rooms.", - "allOf": [{"$ref": "definitions/event_filter.json"}] + "allOf": [{"$ref": "definitions/room_event_filter.json"}] } } }, From 940e22940dbde376db926b1b64fcda411cceffdd Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 13:28:58 +0100 Subject: [PATCH 17/68] Document how the "not_foo" keys interact with the "foo" keys --- api/client-server/v2_alpha/definitions/event_filter.json | 4 ++-- api/client-server/v2_alpha/definitions/room_event_filter.json | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/api/client-server/v2_alpha/definitions/event_filter.json b/api/client-server/v2_alpha/definitions/event_filter.json index c15b8133..269bb5f0 100644 --- a/api/client-server/v2_alpha/definitions/event_filter.json +++ b/api/client-server/v2_alpha/definitions/event_filter.json @@ -17,7 +17,7 @@ "not_types": { "type": "array", "description": - "A list of event types to exclude. If this list is absent then no event types are excluded.", + "A list of event types to exclude. If this list is absent then no event types are excluded. A matching type will be excluded even if it is listed in the 'types' filter", "items": { "type": "string" } @@ -33,7 +33,7 @@ "not_senders": { "type": "array", "description": - "A list of sender IDs to exclude. If this list is absent then no senders are excluded.", + "A list of sender IDs to exclude. If this list is absent then no senders are excluded. A matching sender will be excluded even if it is listed in the 'senders' filter", "items": { "type": "string" } diff --git a/api/client-server/v2_alpha/definitions/room_event_filter.json b/api/client-server/v2_alpha/definitions/room_event_filter.json index c234448e..5be0fcd2 100644 --- a/api/client-server/v2_alpha/definitions/room_event_filter.json +++ b/api/client-server/v2_alpha/definitions/room_event_filter.json @@ -1,5 +1,6 @@ { "type": "object", + "allOf": [{"$ref": "definitions/event_filter.json"}], "properties": { "rooms": { "type": "array", @@ -12,7 +13,7 @@ "not_rooms": { "type": "array", "description": - "A list of room IDs to exclude. If this list is absent then no rooms are excluded.", + "A list of room IDs to exclude. If this list is absent then no rooms are excluded. A matching room will be excluded even if it is listed in the 'rooms' filter", "items": { "type": "string" } From c115b4c2f4cfda43bdb6409edf9f14bfb8004151 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 13:47:04 +0100 Subject: [PATCH 18/68] Document the valid values for the "set_presence" parameter --- api/client-server/v2_alpha/sync.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index e678d2d8..0fe9173d 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -43,9 +43,14 @@ paths: - in: query name: set_presence type: string + enum: ["offline"] description: |- - Set the presence status of this client. - x-example: "online" + Controls whether the client is automatically marked as online by + polling this API. If this parameter is omitted then the client is + automatically marked as online when it uses this API. Otherwise if + the parameter is set to "offline" then the client is not marked as + being online when it uses this API. + x-example: "offline" - in: query name: timeout type: integer From 14b42a41d653f601d7f6a7cc61539a1ccf7f55c7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 13:58:07 +0100 Subject: [PATCH 19/68] Document wildcard's in filters --- api/client-server/v2_alpha/definitions/event_filter.json | 8 ++++---- .../v2_alpha/definitions/room_event_filter.json | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/client-server/v2_alpha/definitions/event_filter.json b/api/client-server/v2_alpha/definitions/event_filter.json index 269bb5f0..1cdcb1f4 100644 --- a/api/client-server/v2_alpha/definitions/event_filter.json +++ b/api/client-server/v2_alpha/definitions/event_filter.json @@ -9,7 +9,7 @@ "types": { "type": "array", "description": - "A list of event types to include. If this list is absent then all event types are included.", + "A list of event types to include. If this list is absent then all event types are included. A '*' can be used as a wildcard to match any sequence of characters.", "items": { "type": "string" } @@ -17,7 +17,7 @@ "not_types": { "type": "array", "description": - "A list of event types to exclude. If this list is absent then no event types are excluded. A matching type will be excluded even if it is listed in the 'types' filter", + "A list of event types to exclude. If this list is absent then no event types are excluded. A matching type will be excluded even if it is listed in the 'types' filter. A '*' can be used as a wildcard to match any sequence of characters.", "items": { "type": "string" } @@ -25,7 +25,7 @@ "senders": { "type": "array", "description": - "A list of senders IDs to include. If this list is absent then all senders are included.", + "A list of senders IDs to include. If this list is absent then all senders are included. A '*' can be used as a wildcard to match any sequence of characters.", "items": { "type": "string" } @@ -33,7 +33,7 @@ "not_senders": { "type": "array", "description": - "A list of sender IDs to exclude. If this list is absent then no senders are excluded. A matching sender will be excluded even if it is listed in the 'senders' filter", + "A list of sender IDs to exclude. If this list is absent then no senders are excluded. A matching sender will be excluded even if it is listed in the 'senders' filter. A '*' can be used as a wildcard to match any sequence of characters.", "items": { "type": "string" } diff --git a/api/client-server/v2_alpha/definitions/room_event_filter.json b/api/client-server/v2_alpha/definitions/room_event_filter.json index 5be0fcd2..86375781 100644 --- a/api/client-server/v2_alpha/definitions/room_event_filter.json +++ b/api/client-server/v2_alpha/definitions/room_event_filter.json @@ -5,7 +5,7 @@ "rooms": { "type": "array", "description": - "A list of room IDs to include. If this list is absent then all rooms are included.", + "A list of room IDs to include. If this list is absent then all rooms are included. A '*' can be used as a wildcard to match any sequence of characters.", "items": { "type": "string" } @@ -13,7 +13,7 @@ "not_rooms": { "type": "array", "description": - "A list of room IDs to exclude. If this list is absent then no rooms are excluded. A matching room will be excluded even if it is listed in the 'rooms' filter", + "A list of room IDs to exclude. If this list is absent then no rooms are excluded. A matching room will be excluded even if it is listed in the 'rooms' filter. A '*' can be used as a wildcard to match any sequence of characters.", "items": { "type": "string" } From 9dd3b073948740b202e7301624eef6beaebd27c0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 14:00:56 +0100 Subject: [PATCH 20/68] Allow '.' characters in event field names to be escaped with '\' so that fields including a '.' can be included in a filter. I considered replacing '.' with '/'. Since '/' was less likely to appear in event field names. However if we used '\' to escape a literal '/' we risk confusing it with the JSON escape '\/'. --- api/client-server/v2_alpha/definitions/sync_filter.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/v2_alpha/definitions/sync_filter.json index 4ad3aa82..b4f87e5f 100644 --- a/api/client-server/v2_alpha/definitions/sync_filter.json +++ b/api/client-server/v2_alpha/definitions/sync_filter.json @@ -39,7 +39,7 @@ "event_fields": { "type": "array", "description": - "List of event fields to include. If this list is absent then all fields are included. The entries may include '.' charaters to indicate sub-fields. So ['content.body'] will include the 'body' field of the 'content' object. A server may include more fields than were requested.", + "List of event fields to include. If this list is absent then all fields are included. The entries may include '.' charaters to indicate sub-fields. So ['content.body'] will include the 'body' field of the 'content' object. A literal '.' character in a field name may be escaped using a '\\'. A server may include more fields than were requested.", "items": { "type": "string" } From 6ad6c40147cf5d80801aa076f76dad2bd54d609e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 28 Sep 2015 14:11:45 +0100 Subject: [PATCH 21/68] List the allowed values for the 'event_format' as an enum --- api/client-server/v2_alpha/definitions/sync_filter.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/v2_alpha/definitions/sync_filter.json index b4f87e5f..b909830b 100644 --- a/api/client-server/v2_alpha/definitions/sync_filter.json +++ b/api/client-server/v2_alpha/definitions/sync_filter.json @@ -34,7 +34,8 @@ "event_format": { "description": "The format to use for events. 'client' will return the events in a format suitable for clients. 'federation' will return the raw event as receieved over federation. The default is 'client'.", - "type": "string" + "type": "string", + "enum": ["client", "federation"] }, "event_fields": { "type": "array", From 4776e0c04ce70a6a9bc11ec24e3b65c38049573d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 28 Sep 2015 14:49:55 +0100 Subject: [PATCH 22/68] Add creation_content key --- specification/1-client_server_api.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 93e3cb90..260832c9 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -750,6 +750,16 @@ options which can be set when creating a room: This will tell the server to invite everyone in the list to the newly created room. +``creation_content`` + Type: + Object + Optional: + Yes + Value: + Extra keys to be added to the content of the ``m.room.create`` + Description: + Allows clients to add keys to the content of ``m.room.create``. + Example:: { From affc2cfc923222da6a10bd6c21dd7d24c8031747 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 28 Sep 2015 16:39:54 +0100 Subject: [PATCH 23/68] Add 'm.federate' flag to 'm.room.create; --- event-schemas/schema/v1/m.room.create | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/event-schemas/schema/v1/m.room.create b/event-schemas/schema/v1/m.room.create index f2e1ee92..bcbb10d2 100644 --- a/event-schemas/schema/v1/m.room.create +++ b/event-schemas/schema/v1/m.room.create @@ -12,6 +12,10 @@ "creator": { "type": "string", "description": "The ``user_id`` of the room creator. This is set by the homeserver." + }, + "m.federate": { + "type": "boolean", + "description": "Whether users on other servers can join this room." } }, "required": ["creator"] From 7b4c8a9f68624e106f3b0ee8aca502d98b88d615 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 29 Sep 2015 16:19:27 +0100 Subject: [PATCH 24/68] Indicate default for m.federate key --- event-schemas/schema/v1/m.room.create | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event-schemas/schema/v1/m.room.create b/event-schemas/schema/v1/m.room.create index bcbb10d2..348b6d86 100644 --- a/event-schemas/schema/v1/m.room.create +++ b/event-schemas/schema/v1/m.room.create @@ -15,7 +15,7 @@ }, "m.federate": { "type": "boolean", - "description": "Whether users on other servers can join this room." + "description": "Whether users on other servers can join this room. Defaults to ``true`` if key does not exist." } }, "required": ["creator"] From 83b9497664f030ac2e1d3bae1a5faddb8425250f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 29 Sep 2015 16:21:10 +0100 Subject: [PATCH 25/68] Add context and example --- specification/1-client_server_api.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 260832c9..dd58b357 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -756,7 +756,8 @@ options which can be set when creating a room: Optional: Yes Value: - Extra keys to be added to the content of the ``m.room.create`` + Extra keys to be added to the content of the ``m.room.create``. The server + will clober certain keys, e.g. ``creator``. Description: Allows clients to add keys to the content of ``m.room.create``. @@ -766,7 +767,10 @@ Example:: "visibility": "public", "room_alias_name": "thepub", "name": "The Grand Duke Pub", - "topic": "All about happy hour" + "topic": "All about happy hour", + "creation_content": { + "m.federate": false + } } The home server will create a ``m.room.create`` event when the room is created, From 0a04672d764e070fc0199c958b8c006498314fbf Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 29 Sep 2015 17:57:44 +0100 Subject: [PATCH 26/68] Start converting the presence module. Add Rationale admonition. --- scripts/nature.css | 6 ++ specification/1-client_server_api.rst | 21 +++++ specification/modules/presence.rst | 114 ++++++++++++++++---------- 3 files changed, 97 insertions(+), 44 deletions(-) diff --git a/scripts/nature.css b/scripts/nature.css index 9d67f689..f1b4edec 100644 --- a/scripts/nature.css +++ b/scripts/nature.css @@ -282,3 +282,9 @@ td[colspan]:not([colspan="1"]) { thead { background: #eeeeee; } + +div.admonition-rationale { + background-color: #efe; + border: 1px solid #ccc; +} + diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 893aec73..4403e0d3 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -1092,6 +1092,27 @@ Profiles {{profile_http_api}} +Events on Change of Profile Information +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Because the profile display name and avatar information are likely to be used in +many places of a client's display, changes to these fields cause an automatic +propagation event to occur, informing likely-interested parties of the new +values. This change is conveyed using two separate mechanisms: + + - a ``m.room.member`` event is sent to every room the user is a member of, + to update the ``displayname`` and ``avatar_url``. + - a ``m.presence`` presence status update is sent, again containing the new + values of the ``displayname`` and ``avatar_url`` keys, in addition to the + required ``presence`` key containing the current presence state of the user. + +Both of these should be done automatically by the home server when a user +successfully changes their display name or avatar URL fields. + +Additionally, when home servers emit room membership events for their own +users, they should include the display name and avatar URL fields in these +events so that clients already have these details to hand, and do not have to +perform extra round trips to query it. + Security -------- diff --git a/specification/modules/presence.rst b/specification/modules/presence.rst index ddd2adff..4c84359a 100644 --- a/specification/modules/presence.rst +++ b/specification/modules/presence.rst @@ -1,63 +1,89 @@ Presence ======== -Each user has the concept of presence information. This encodes the -"availability" of that user, suitable for display on other user's clients. +Each user has presence information associated with them. This encodes the +"availability" of that user, suitable for display on other clients. This is transmitted as an ``m.presence`` event and is one of the few events -which are sent *outside the context of a room*. The basic piece of presence -information is represented by the ``presence`` key, which is an enum of one -of the following: +which are sent *outside the context of a room*. Their presence state is +represented by the ``presence`` key, which is an enum of one of the following: - ``online`` : The default state when the user is connected to an event stream. - - ``unavailable`` : The user is not reachable at this time. - - ``offline`` : The user is not connected to an event stream. + - ``unavailable`` : The user is not reachable at this time e.g. they are + idle. + - ``offline`` : The user is not connected to an event stream or is + explicitly suppressing their profile information from being sent. - ``free_for_chat`` : The user is generally willing to receive messages moreso than default. - - ``hidden`` : Behaves as offline, but allows the user to see the client - state anyway and generally interact with client features. (Not yet - implemented in synapse). - -In addition, the server maintains a timestamp of the last time it saw a -pro-active event from the user; either sending a message to a room, or -changing presence state from a lower to a higher level of availability -(thus: changing state from ``unavailable`` to ``online`` counts as a -proactive event, whereas in the other direction it will not). This timestamp -is presented via a key called ``last_active_ago``, which gives the relative -number of milliseconds since the message is generated/emitted that the user -was last seen active. Events ------ {{presence_events}} -Presence HTTP API ------------------ -.. TODO-spec - - Define how users receive presence invites, and how they accept/decline them +Client behaviour +---------------- + +Clients can manually set/get their presence using the HTTP APIs listed below. {{presence_http_api}} - -Events on Change of Profile Information ---------------------------------------- -Because the profile displayname and avatar information are likely to be used in -many places of a client's display, changes to these fields cause an automatic -propagation event to occur, informing likely-interested parties of the new -values. This change is conveyed using two separate mechanisms: - - - a ``m.room.member`` event is sent to every room the user is a member of, - to update the ``displayname`` and ``avatar_url``. - - a ``m.presence`` presence status update is sent, again containing the new values of the - ``displayname`` and ``avatar_url`` keys, in addition to the required - ``presence`` key containing the current presence state of the user. - -Both of these should be done automatically by the home server when a user -successfully changes their displayname or avatar URL fields. - -Additionally, when home servers emit room membership events for their own -users, they should include the displayname and avatar URL fields in these -events so that clients already have these details to hand, and do not have to -perform extra roundtrips to query it. +Idle timeout +~~~~~~~~~~~~ + +Clients SHOULD implement an "idle timeout". This is a timer which fires after +a period of inactivity on the client. The definition of inactivity varies +depending on the client. For example, web implementations may determine +inactivity to be not moving the mouse for a certain period of time. When this +timer fires it should set the presence state to ``unavailable``. When the user +becomes active again (e.g. by moving the mouse) the client should set the +presence state to ``online``. A timeout value between 1 and 5 minutes is +recommended. + +Server behaviour +---------------- + +Propagating profile information +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Because the profile display name and avatar information are likely to be used in +many places of a client's display, changes to these fields SHOULD cause an +automatic propagation event to occur, informing likely-interested parties of the +new values. One of these change mechanisms SHOULD be via ``m.presence`` events. +These events should set ``displayname`` and ``avatar_url`` to the new values +along with the presence-specific keys. This SHOULD be done automatically by the +home server when a user successfully changes their display name or avatar URL. + +.. admonition:: Rationale + + The intention for sending this information in ``m.presence`` is so that any + "user list" can display the *current* name/presence for a user ID outside the + scope of a room (e.g. a user page which has a "start conversation" button). + This is bundled into a single event to avoid "flickering" on this page which + can occur if you received presence first and then display name later (the + user's name would flicker from their user ID to the display name). + + +Last active ago +~~~~~~~~~~~~~~~ +The server maintains a timestamp of the last time it saw a +pro-active event from the user. A pro-active event may be sending a message to a +room or changing presence state to a higher level of availability. Levels of +availability are defined from low to high as follows: + + - ``offline`` + - ``unavailable`` + - ``online`` + - ``free_for_chat`` + +Based on this list, changing state from ``unavailable`` to ``online`` counts as +a pro-active event, whereas ``online`` to ``unavailable`` does not. This +timestamp is presented via a key called ``last_active_ago`` which gives the +relative number of milliseconds since the pro-active event. + +Security considerations +----------------------- + +Presence information is shared with all users who share a room with the target +user. In large public rooms this could be undesirable. From 218cf94ead2dacce3b3f4443250465ef2f23fd24 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 29 Sep 2015 19:10:31 +0100 Subject: [PATCH 27/68] Replace the events array with events_map inside the room objects. Only use indirection for the state and timeline events. Use event_ids to reference the state and timeline events. --- .../v2_alpha/definitions/event_batch.json | 4 +- .../definitions/room_event_batch.json | 12 ++ .../v2_alpha/definitions/timeline_batch.json | 2 +- api/client-server/v2_alpha/sync.yaml | 134 ++++++++++-------- 4 files changed, 90 insertions(+), 62 deletions(-) create mode 100644 api/client-server/v2_alpha/definitions/room_event_batch.json diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/v2_alpha/definitions/event_batch.json index ed2730c8..75762d75 100644 --- a/api/client-server/v2_alpha/definitions/event_batch.json +++ b/api/client-server/v2_alpha/definitions/event_batch.json @@ -3,9 +3,9 @@ "properties": { "events": { "type": "array", - "description": "List of indices into an events array", + "description": "List of events", "items": { - "type": "integer" + "type": "object" } } } diff --git a/api/client-server/v2_alpha/definitions/room_event_batch.json b/api/client-server/v2_alpha/definitions/room_event_batch.json new file mode 100644 index 00000000..fcf148f3 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/room_event_batch.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "properties": { + "events": { + "type": "array", + "description": "List of event ids", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/v2_alpha/definitions/timeline_batch.json index dc62724b..ddf8d341 100644 --- a/api/client-server/v2_alpha/definitions/timeline_batch.json +++ b/api/client-server/v2_alpha/definitions/timeline_batch.json @@ -1,6 +1,6 @@ { "type": "object", - "allOf": [{"$ref":"definitions/event_batch.json"}], + "allOf": [{"$ref":"definitions/room_event_batch.json"}], "properties": { "limited": { "type": "boolean", diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 0fe9173d..1df7dace 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -71,13 +71,6 @@ paths: description: |- The batch token to supply in the ``since`` param of the next ``/sync`` request. - events: - type: array - description: |- - A list of event objects that are referred to by index in the - rest of the ``sync`` response. - items: - type: object rooms: type: array description: |- @@ -89,11 +82,20 @@ paths: type: string description: |- The ID of the room. + event_map: + type: object + description: |- + A map from event ID to events for this room. The events + are referenced from the ``timeline`` and ``state`` keys + for this room. + additionalProperties: + description: An event object. + type: object state: description: |- The state updates for the room. allOf: - - $ref: "definitions/event_batch.json" + - $ref: "definitions/room_event_batch.json" timeline: description: |- The timeline of messages and state changes in the room. @@ -120,62 +122,76 @@ paths: application/json: |- { "next_batch": "s72595_4483_1934", - "events": [ - { - "sender": "@bob:example.com", - "type": "com.example.weird.setting", - "content": {"setting1": true, "setting2": false} - }, - { - "sender": "@alice:example.com", - "type": "m.profile.display_name", - "content": {"display_name": "Alice"} - }, - { - "sender": "@alice:example.com", - "type": "m.presence", - "content": {"presence": "online"} - }, - { - "room_id": "!726s6s6q:example.com", - "type": "m.typing", - "content": {"user_ids": ["@alice:example.com"]} - }, - { - "sender": "@alice:example.com", - "room_id": "!726s6s6q:example.com", - "type": "m.room.member", - "state_key": "@alice:example.com", - "content": {"membership": "join"}, - "origin_server_ts": 1417731086795 - }, - { - "sender": "@bob:example.com", - "room_id": "!726s6s6q:example.com", - "type": "m.room.member", - "state_key": "@bob:example.com", - "content": {"membership": "join"}, - "origin_server_ts": 1417731086795 - }, - { - "sender": "@alice:example.com", - "room_id": "!726s6s6q:example.com", - "type": "m.room.message", - "unsigned": {"age": "124524", "txn_id": "1234"}, - "content": {"body": "I am a fish", "msgtype": "m.text"}, - "origin_server_ts": 1417731086797 - } - ], - "private_user_data": {"events": [0]}, - "public_user_data": {"events": [1, 2]}, + "private_user_data": { + "events": [ + { + "sender": "@bob:example.com", + "type": "com.example.weird.setting", + "content": {"setting1": true, "setting2": false} + } + ] + }, + "public_user_data": { + "events": [ + { + "sender": "@alice:example.com", + "type": "m.profile.display_name", + "content": {"display_name": "Alice"} + }, + { + "sender": "@alice:example.com", + "type": "m.presence", + "content": {"presence": "online"} + } + ] + }, "rooms": [{ "room_id": "!726s6s6q:example.com", - "ephemeral": {"events": [3]}, - "state": {"events": [4, 5]}, + "event_map": { + "$66697273743031:example.com": { + "sender": "@alice:example.com", + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + "$7365636s6r6432:example.com": { + "sender": "@bob:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + "$74686972643033:example.com": { + "sender": "@alice:example.com", + "type": "m.room.message", + "unsigned": {"age": "124524", "txn_id": "1234"}, + "content": {"body": "I am a fish", "msgtype": "m.text"}, + "origin_server_ts": 1417731086797 + } + }, + "state": { + "events": [ + "$66697273743031:example.com", + "$7365636s6r6432:example.com" + ] + }, "timeline": { - "events": [5,6], + "events": [ + "$7365636s6r6432:example.com", + "$74686972643033:example.com" + ], "limited": true, "prev_batch": "t34-23535_0_0" + }, + "ephemeral": { + "events": [ + { + "room_id": "!726s6s6q:example.com", + "type": "m.typing", + "content": {"user_ids": ["@alice:example.com"]} + } + ] } }] } From 4cb3f78d2bc8205c87e566e16a049732c2f3bad5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 29 Sep 2015 19:48:48 +0100 Subject: [PATCH 28/68] Wrap the "rooms" list inside an object so that we can add keys for pagination later --- api/client-server/v2_alpha/sync.yaml | 165 ++++++++++++++------------- 1 file changed, 85 insertions(+), 80 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 1df7dace..58a2c495 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -72,41 +72,44 @@ paths: The batch token to supply in the ``since`` param of the next ``/sync`` request. rooms: - type: array - description: |- - A list of rooms that the client needs to update. - items: - type: object - properties: - room_id: - type: string - description: |- - The ID of the room. - event_map: + type: object + properties: + roomlist: + type: array + description: |- + A list of rooms that the client needs to update. + items: type: object - description: |- - A map from event ID to events for this room. The events - are referenced from the ``timeline`` and ``state`` keys - for this room. - additionalProperties: - description: An event object. - type: object - state: - description: |- - The state updates for the room. - allOf: - - $ref: "definitions/room_event_batch.json" - timeline: - description: |- - The timeline of messages and state changes in the room. - allOf: - - $ref: "definitions/timeline_batch.json" - ephemeral: - description: |- - The ephemeral events in the room that aren't recorded - in the timeline or state of the room. E.g. typing. - allOf: - - $ref: "definitions/event_batch.json" + properties: + room_id: + type: string + description: |- + The ID of the room. + event_map: + type: object + description: |- + A map from event ID to events for this room. The events + are referenced from the ``timeline`` and ``state`` keys + for this room. + additionalProperties: + description: An event object. + type: object + state: + description: |- + The state updates for the room. + allOf: + - $ref: "definitions/room_event_batch.json" + timeline: + description: |- + The timeline of messages and state changes in the room. + allOf: + - $ref: "definitions/timeline_batch.json" + ephemeral: + description: |- + The ephemeral events in the room that aren't recorded + in the timeline or state of the room. E.g. typing. + allOf: + - $ref: "definitions/event_batch.json" public_user_data: description: |- The updates to publicly visible user data. @@ -145,53 +148,55 @@ paths: } ] }, - "rooms": [{ - "room_id": "!726s6s6q:example.com", - "event_map": { - "$66697273743031:example.com": { - "sender": "@alice:example.com", - "type": "m.room.member", - "state_key": "@alice:example.com", - "content": {"membership": "join"}, - "origin_server_ts": 1417731086795 + "rooms": { + "roomlist": [{ + "room_id": "!726s6s6q:example.com", + "event_map": { + "$66697273743031:example.com": { + "sender": "@alice:example.com", + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + "$7365636s6r6432:example.com": { + "sender": "@bob:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + "$74686972643033:example.com": { + "sender": "@alice:example.com", + "type": "m.room.message", + "unsigned": {"age": "124524", "txn_id": "1234"}, + "content": {"body": "I am a fish", "msgtype": "m.text"}, + "origin_server_ts": 1417731086797 + } }, - "$7365636s6r6432:example.com": { - "sender": "@bob:example.com", - "type": "m.room.member", - "state_key": "@bob:example.com", - "content": {"membership": "join"}, - "origin_server_ts": 1417731086795 + "state": { + "events": [ + "$66697273743031:example.com", + "$7365636s6r6432:example.com" + ] }, - "$74686972643033:example.com": { - "sender": "@alice:example.com", - "type": "m.room.message", - "unsigned": {"age": "124524", "txn_id": "1234"}, - "content": {"body": "I am a fish", "msgtype": "m.text"}, - "origin_server_ts": 1417731086797 + "timeline": { + "events": [ + "$7365636s6r6432:example.com", + "$74686972643033:example.com" + ], + "limited": true, + "prev_batch": "t34-23535_0_0" + }, + "ephemeral": { + "events": [ + { + "room_id": "!726s6s6q:example.com", + "type": "m.typing", + "content": {"user_ids": ["@alice:example.com"]} + } + ] } - }, - "state": { - "events": [ - "$66697273743031:example.com", - "$7365636s6r6432:example.com" - ] - }, - "timeline": { - "events": [ - "$7365636s6r6432:example.com", - "$74686972643033:example.com" - ], - "limited": true, - "prev_batch": "t34-23535_0_0" - }, - "ephemeral": { - "events": [ - { - "room_id": "!726s6s6q:example.com", - "type": "m.typing", - "content": {"user_ids": ["@alice:example.com"]} - } - ] - } - }] + }] + } } From 73b4090f52f324c598db699899b22eed72503a3d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 30 Sep 2015 10:22:12 +0100 Subject: [PATCH 29/68] Add private_chat_shared_power --- specification/1-client_server_api.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index c5453931..329a87dd 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -756,15 +756,18 @@ options which can be set when creating a room: Optional: Yes Value: - ``private_chat`` or ``public_chat`` + ``private_chat``, ``private_chat_shared_power`` or ``public_chat`` Description: Convenience parameter for setting various default state events based on a preset. - Two presets are defined: + Three presets are defined: - ``private_chat``: Sets the ``join_rules`` to ``invite`` and ``history_visibility`` to ``shared`` + - ``private_chat_shared_power``: Set the ``join_rules`` to + ``invite``, ``history_visibility`` to ``shared`` and gives all invitees + the same power level as the creator. - ``public_chat``: Sets the ``join_rules`` to ``public`` and ``history_visibility`` to ``shared`` From 069e4e39f4d326ef9e4188159767a1194db1ae76 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 30 Sep 2015 10:22:51 +0100 Subject: [PATCH 30/68] Move presence specific sections from intro to presence module --- specification/0-intro.rst | 43 ------------------------------ specification/modules/presence.rst | 35 ++++++++++++++++++++---- 2 files changed, 30 insertions(+), 48 deletions(-) diff --git a/specification/0-intro.rst b/specification/0-intro.rst index a5196e99..1b4e8672 100644 --- a/specification/0-intro.rst +++ b/specification/0-intro.rst @@ -338,49 +338,6 @@ Usage of an IS is not required in order for a client application to be part of the Matrix ecosystem. However, without one clients will not be able to look up user IDs using 3PIDs. -Presence -~~~~~~~~ - -Each user has the concept of presence information. This encodes: - - * Whether the user is currently online - * How recently the user was last active (as seen by the server) - * Whether a given client considers the user to be currently idle - * Arbitrary information about the user's current status (e.g. "in a meeting"). - -This information is collated from both per-device (online; idle; last_active) and -per-user (status) data, aggregated by the user's homeserver and transmitted as -an ``m.presence`` event. This is one of the few events which are sent *outside -the context of a room*. Presence events are sent to all users who subscribe to -this user's presence through a presence list or by sharing membership of a room. - -.. TODO - How do we let users hide their presence information? - -.. TODO - The last_active specifics should be moved to the detailed presence event section - -Last activity is tracked by the server maintaining a timestamp of the last time -it saw a pro-active event from the user. Any event which could be triggered by a -human using the application is considered pro-active (e.g. sending an event to a -room). An example of a non-proactive client activity would be a client setting -'idle' presence status, or polling for events. This timestamp is presented via a -key called ``last_active_ago``, which gives the relative number of milliseconds -since the message is generated/emitted that the user was last seen active. - -N.B. in v1 API, status/online/idle state are muxed into a single 'presence' -field on the ``m.presence`` event. - -Presence Lists -~~~~~~~~~~~~~~ - -Each user's home server stores a "presence list". This stores a list of user IDs -whose presence the user wants to follow. - -To be added to this list, the user being added must be invited by the list owner -and accept the invitation. Once accepted, both user's HSes track the -subscription. - Profiles ~~~~~~~~ diff --git a/specification/modules/presence.rst b/specification/modules/presence.rst index 4c84359a..1a359e5a 100644 --- a/specification/modules/presence.rst +++ b/specification/modules/presence.rst @@ -1,5 +1,23 @@ Presence ======== + +Each user has the concept of presence information. This encodes: + + * Whether the user is currently online + * How recently the user was last active (as seen by the server) + * Whether a given client considers the user to be currently idle + * Arbitrary information about the user's current status (e.g. "in a meeting"). + +This information is collated from both per-device (``online``, ``idle``, +``last_active``) and per-user (status) data, aggregated by the user's homeserver +and transmitted as an ``m.presence`` event. This is one of the few events which +are sent *outside the context of a room*. Presence events are sent to all users +who subscribe to this user's presence through a presence list or by sharing +membership of a room. + +A presence list is a list of user IDs whose presence the user wants to follow. +To be added to this list, the user being added must be invited by the list owner +who must accept the invitation. Each user has presence information associated with them. This encodes the "availability" of that user, suitable for display on other clients. @@ -15,6 +33,7 @@ represented by the ``presence`` key, which is an enum of one of the following: explicitly suppressing their profile information from being sent. - ``free_for_chat`` : The user is generally willing to receive messages moreso than default. + Events ------ @@ -24,7 +43,8 @@ Events Client behaviour ---------------- -Clients can manually set/get their presence using the HTTP APIs listed below. +Clients can manually set/get their presence/presence list using the HTTP APIs +listed below. {{presence_http_api}} @@ -43,6 +63,9 @@ recommended. Server behaviour ---------------- +Each user's home server stores a "presence list" per user. Once a user accepts +a presence list, both user's HSes must track the subscription. + Propagating profile information ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -58,10 +81,12 @@ home server when a user successfully changes their display name or avatar URL. The intention for sending this information in ``m.presence`` is so that any "user list" can display the *current* name/presence for a user ID outside the - scope of a room (e.g. a user page which has a "start conversation" button). - This is bundled into a single event to avoid "flickering" on this page which - can occur if you received presence first and then display name later (the - user's name would flicker from their user ID to the display name). + scope of a room e.g. for a user page. This is bundled into a single event for + several reasons. The user's display name can change per room. This + event provides the "canonical" name for the user. In addition, the name is + bundled into a single event for the ease of client implementations. If this + was not done, the client would need to search all rooms for their own + membership event to pull out the display name. Last active ago From 931057accfdd2756686c3c896a22157b3f5faf51 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 30 Sep 2015 10:30:39 +0100 Subject: [PATCH 31/68] Add a top level presence key for the presence events and remove the public_user_data/private_user_data for now --- .../v2_alpha/definitions/sync_filter.json | 9 ++----- api/client-server/v2_alpha/filter.yaml | 22 +++++----------- api/client-server/v2_alpha/sync.yaml | 26 +++---------------- 3 files changed, 11 insertions(+), 46 deletions(-) diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/v2_alpha/definitions/sync_filter.json index b909830b..0cd6a798 100644 --- a/api/client-server/v2_alpha/definitions/sync_filter.json +++ b/api/client-server/v2_alpha/definitions/sync_filter.json @@ -21,14 +21,9 @@ } } }, - "public_user_data": { + "presence": { "description": - "The public user data, e.g. profile and presence, to include.", - "allOf": [{"$ref": "definitions/event_filter.json"}] - }, - "private_user_data": { - "description": - "Events that are private to a user but shared amoungst their devices, e.g. notification settings, to include.", + "The presence updates to include.", "allOf": [{"$ref": "definitions/event_filter.json"}] }, "event_format": { diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml index 9bda2359..30d52844 100644 --- a/api/client-server/v2_alpha/filter.yaml +++ b/api/client-server/v2_alpha/filter.yaml @@ -62,14 +62,9 @@ paths: "not_senders": ["@spam:example.com"] } }, - "public_user_data": { - "types": ["m.presence"] - }, - "private_user_data": { - "types": [] - }, - "server_data": { - "types": [] + "presence": { + "types": ["m.presence"], + "not_senders": ["@alice:example.com"] }, "event_format": "client", "event_fields": ["type", "content", "sender"] @@ -131,14 +126,9 @@ paths: "not_senders": ["@spam:example.com"] } }, - "public_user_data": { - "types": ["m.presence"] - }, - "private_user_data": { - "types": [] - }, - "server_data": { - "types": [] + "presence": { + "types": ["m.presence"], + "not_senders": ["@alice:example.com"] }, "event_format": "client", "event_fields": ["type", "content", "sender"] diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 58a2c495..238fab63 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -110,37 +110,17 @@ paths: in the timeline or state of the room. E.g. typing. allOf: - $ref: "definitions/event_batch.json" - public_user_data: + presence: description: |- - The updates to publicly visible user data. - allOf: - - $ref: "definitions/event_batch.json" - private_user_data: - description: |- - Updates to the data which is private to the user but shared - amongst their devices. + The updates to the presence status of other users. allOf: - $ref: "definitions/event_batch.json" examples: application/json: |- { "next_batch": "s72595_4483_1934", - "private_user_data": { + "presence": { "events": [ - { - "sender": "@bob:example.com", - "type": "com.example.weird.setting", - "content": {"setting1": true, "setting2": false} - } - ] - }, - "public_user_data": { - "events": [ - { - "sender": "@alice:example.com", - "type": "m.profile.display_name", - "content": {"display_name": "Alice"} - }, { "sender": "@alice:example.com", "type": "m.presence", From 2b7e02c0805b81d1c2ad72245f4dc13bdb56a445 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 30 Sep 2015 11:26:02 +0100 Subject: [PATCH 32/68] Add sections for typing. Add swagger, JSON schema and example m.typing event --- api/client-server/v1/typing.yaml | 77 +++++++++++++++++++ event-schemas/examples/v1/m.typing | 7 ++ event-schemas/schema/v1/m.typing | 28 +++++++ .../modules/typing_notifications.rst | 47 +++++------ 4 files changed, 131 insertions(+), 28 deletions(-) create mode 100644 api/client-server/v1/typing.yaml create mode 100644 event-schemas/examples/v1/m.typing create mode 100644 event-schemas/schema/v1/m.typing diff --git a/api/client-server/v1/typing.yaml b/api/client-server/v1/typing.yaml new file mode 100644 index 00000000..737c6928 --- /dev/null +++ b/api/client-server/v1/typing.yaml @@ -0,0 +1,77 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v1 Typing API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/api/v1 +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/rooms/{roomId}/typing/{userId}": + put: + summary: Informs the server that the user has started or stopped typing. + description: |- + This tells the server that the user is typing for the next N + milliseconds where N is the value specified in the ``timeout`` key. + Alternatively, if ``typing`` is ``false``, it tells the server that the + user has stopped typing. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + description: The user who has started to type. + required: true + x-example: "@alice:example.com" + - in: path + type: string + name: roomId + description: The room in which the user is typing. + required: true + x-example: "!wefh3sfukhs:example.com" + - in: body + name: typingState + description: The current typing state. + required: true + schema: + type: object + example: |- + { + "typing": true, + "timeout": 30000 + } + properties: + typing: + type: boolean + description: |- + Whether the user is typing or not. If ``false``, the ``timeout`` + key can be omitted. + timeout: + type: integer + description: The length of time in milliseconds to mark this user as typing. + required: ["typing"] + responses: + 200: + description: The new typing state was set. + examples: + application/json: |- + {} + schema: + type: object # empty json object + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + diff --git a/event-schemas/examples/v1/m.typing b/event-schemas/examples/v1/m.typing new file mode 100644 index 00000000..bd53f6fb --- /dev/null +++ b/event-schemas/examples/v1/m.typing @@ -0,0 +1,7 @@ +{ + "type": "m.typing", + "room_id": "!z0mnsuiwhifuhwwfw:matrix.org", + "content": { + "user_ids": ["@alice:matrix.org", "@bob:example.com"] + } +} \ No newline at end of file diff --git a/event-schemas/schema/v1/m.typing b/event-schemas/schema/v1/m.typing new file mode 100644 index 00000000..b712f6ec --- /dev/null +++ b/event-schemas/schema/v1/m.typing @@ -0,0 +1,28 @@ +{ + "type": "object", + "title": "Typing Event", + "description": "Informs the client of the list of users currently typing.", + "properties": { + "content": { + "type": "object", + "properties": { + "user_ids": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The list of user IDs typing in this room, if any." + } + }, + "required": ["user_ids"] + }, + "type": { + "type": "string", + "enum": ["m.typing"] + }, + "room_id": { + "type": "string" + } + }, + "required": ["type", "room_id", "content"] +} diff --git a/specification/modules/typing_notifications.rst b/specification/modules/typing_notifications.rst index 25b714ab..b32e3411 100644 --- a/specification/modules/typing_notifications.rst +++ b/specification/modules/typing_notifications.rst @@ -1,45 +1,30 @@ Typing Notifications --------------------- +==================== -Client APIs -~~~~~~~~~~~ +Events +------ -To set "I am typing for the next N msec":: +{{m_typing_event}} - PUT .../rooms//typing/ - Content: { "typing": true, "timeout": N } - # timeout is in milliseconds; suggested no more than 20 or 30 seconds +Client behaviour +---------------- + + - suggested no more than 20-30 seconds This should be re-sent by the client to continue informing the server the user is still typing; a safety margin of 5 seconds before the expected timeout runs out is recommended. Just keep declaring a new timeout, it will replace the old one. -To set "I am no longer typing":: - - PUT ../rooms//typing/ - Content: { "typing": false } - -Client Events -~~~~~~~~~~~~~ - -All room members will receive an event on the event stream:: - - { - "type": "m.typing", - "room_id": "!room-id-here:matrix.org", - "content": { - "user_ids": ["list of", "every user", "who is", "currently typing"] - } - } - -The client must use this list to *REPLACE* its knowledge of every user who is +Event: The client must use this list to *REPLACE* its knowledge of every user who is currently typing. The reason for this is that the server DOES NOT remember users who are not currently typing, as that list gets big quickly. The client should mark as not typing, any user ID who is not in that list. -Server APIs -~~~~~~~~~~~ +{{typing_http_api}} + +Server behaviour +---------------- Servers will emit EDUs in the following form:: @@ -59,3 +44,9 @@ originating HSes to ensure they eventually send "stop" notifications. ((This will eventually need addressing, as part of the wider typing/presence timer addition work)) +Security considerations +----------------------- + +Clients may not wish to inform everyone in a room that they are typing and +instead only specific users in the room. + From a82f2ad4ac0b622899fcc039b05d2cd14c26e37c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 30 Sep 2015 11:55:34 +0100 Subject: [PATCH 33/68] Flesh out typing module --- specification/0-intro.rst | 2 + .../modules/typing_notifications.rst | 39 ++++++++++++------- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/specification/0-intro.rst b/specification/0-intro.rst index 93458d97..5fec59c5 100644 --- a/specification/0-intro.rst +++ b/specification/0-intro.rst @@ -180,6 +180,8 @@ of a "Room". Event Graphs ~~~~~~~~~~~~ +.. _sect:event-graph: + Events exchanged in the context of a room are stored in a directed acyclic graph (DAG) called an ``event graph``. The partial ordering of this graph gives the chronological ordering of events within the room. Each event in the graph has a diff --git a/specification/modules/typing_notifications.rst b/specification/modules/typing_notifications.rst index b32e3411..110cd134 100644 --- a/specification/modules/typing_notifications.rst +++ b/specification/modules/typing_notifications.rst @@ -1,6 +1,13 @@ Typing Notifications ==================== +Users often desire to see when another user is typing. This can be achieved +using typing notifications. These are ephemeral events scoped to a ``room_id``. +This means they do not form part of the `Event Graph`_ but still have a +``room_id`` key. + +.. _Event Graph: `sect:event-graph`_ + Events ------ @@ -9,24 +16,28 @@ Events Client behaviour ---------------- - - suggested no more than 20-30 seconds - -This should be re-sent by the client to continue informing the server the user -is still typing; a safety margin of 5 seconds before the expected -timeout runs out is recommended. Just keep declaring a new timeout, it will -replace the old one. - -Event: The client must use this list to *REPLACE* its knowledge of every user who is -currently typing. The reason for this is that the server DOES NOT remember -users who are not currently typing, as that list gets big quickly. The client -should mark as not typing, any user ID who is not in that list. +When a client receives an ``m.typing`` event, it MUST use the user ID list to +**REPLACE** its knowledge of every user who is currently typing. The reason for +this is that the server *does not remember* users who are not currently typing +as that list gets big quickly. The client should mark as not typing any user ID +who is not in that list. + +It is recommended that clients store a ``boolean`` indicating whether the user +is typing or not. Whilst this value is ``true`` a timer should fire periodically +every N seconds to send a typing HTTP request. The value of N is recommended to +be no more than 20-30 seconds. This request should be re-sent by the client to +continue informing the server the user is still typing. As subsequent +requests will replace older requests, a safety margin of 5 seconds before the +expected timeout runs out is recommended. When the user stops typing, the +state change of the ``boolean`` to ``false`` should trigger another HTTP request +to inform the server that the user has stopped typing. {{typing_http_api}} Server behaviour ---------------- -Servers will emit EDUs in the following form:: +Servers MUST emit typing EDUs in the following form:: { "type": "m.typing", @@ -37,8 +48,8 @@ Servers will emit EDUs in the following form:: } } -Server EDUs don't (currently) contain timing information; it is up to -originating HSes to ensure they eventually send "stop" notifications. +This does not contain timing information so it is up to originating homeservers +to ensure they eventually send "stop" notifications. .. TODO ((This will eventually need addressing, as part of the wider typing/presence From 097dc501805891c594dc218d6751fb529d2dd6a0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 30 Sep 2015 15:45:29 +0100 Subject: [PATCH 34/68] Minor tweaks --- specification/modules/typing_notifications.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/specification/modules/typing_notifications.rst b/specification/modules/typing_notifications.rst index 110cd134..74bd76bc 100644 --- a/specification/modules/typing_notifications.rst +++ b/specification/modules/typing_notifications.rst @@ -1,10 +1,10 @@ Typing Notifications ==================== -Users often desire to see when another user is typing. This can be achieved -using typing notifications. These are ephemeral events scoped to a ``room_id``. -This means they do not form part of the `Event Graph`_ but still have a -``room_id`` key. +Users may wish to be informed when another user is typing in a room. This can be +achieved using typing notifications. These are ephemeral events scoped to a +``room_id``. This means they do not form part of the `Event Graph`_ but still +have a ``room_id`` key. .. _Event Graph: `sect:event-graph`_ @@ -37,7 +37,8 @@ to inform the server that the user has stopped typing. Server behaviour ---------------- -Servers MUST emit typing EDUs in the following form:: +Servers MUST emit typing EDUs in a different form to ``m.typing`` events which +are shown to clients. This form looks like:: { "type": "m.typing", From 87b6dd845e0e6c685800bbc13f9459a7d21ad59e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 1 Oct 2015 17:55:16 +0100 Subject: [PATCH 35/68] Flesh out content repo; modify templating to support headers Edit content-repo.yaml to include examples and headers. Restructure content module to conform to the module template. Adjust the HTTP API template to give 1 more char to the response param to fit "Content-Disposition" correctly. Edit the templating system to support displaying enums for swagger APIs (before it was just JSON schema). Also add support for introspecting headers from swagger. Finally, replace - with _ when forming the {{ template_var }} else things whine. --- api/client-server/v1/content-repo.yaml | 49 +++++++++++++++++-- specification/modules/content_repo.rst | 30 ++++++++++-- .../matrix_templates/templates/http-api.tmpl | 12 ++--- templating/matrix_templates/units.py | 22 +++++++-- 4 files changed, 96 insertions(+), 17 deletions(-) diff --git a/api/client-server/v1/content-repo.yaml b/api/client-server/v1/content-repo.yaml index fe3d1dc3..aa47dc18 100644 --- a/api/client-server/v1/content-repo.yaml +++ b/api/client-server/v1/content-repo.yaml @@ -15,16 +15,22 @@ paths: summary: Upload some content to the content repository. produces: ["application/json"] parameters: + - in: header + name: Content-Type + type: string + description: The content type of the file being uploaded + x-example: "Content-Type: audio/mpeg" - in: body - name: content + name: "" description: The content to be uploaded. required: true schema: type: string + example: "" format: byte responses: 200: - description: Information about the uploaded content. + description: The MXC URI for the uploaded content. schema: type: object required: ["content_uri"] @@ -32,6 +38,11 @@ paths: content_uri: type: string description: "The MXC URI to the uploaded content." + examples: + "application/json": |- + { + "content_uri": "mxc://example.com/AQwafuaFswefuhsfAFAgsw" + } "/download/{serverName}/{mediaId}": get: summary: "Download content from the content repository." @@ -40,20 +51,32 @@ paths: - in: path type: string name: serverName + x-example: matrix.org required: true description: | The server name from the ``mxc://`` URI (the authoritory component) - in: path type: string name: mediaId + x-example: ascERGshawAWawugaAcauga required: true description: | The media ID from the ``mxc://`` URI (the path component) responses: 200: - description: "The content downloaded." + description: "The content that was previously uploaded." + headers: + Content-Type: + description: "The content type of the file that was previously uploaded." + x-example: "audio/mpeg" + type: "string" + Content-Disposition: + description: "The name of the file that was previously uploaded, if set." + x-example: "attachment;filename=03-cool.mp3" + type: "string" schema: type: file + name: "" "/thumbnail/{serverName}/{mediaId}": get: summary: "Download a thumbnail of the content from the content repository." @@ -63,31 +86,47 @@ paths: type: string name: serverName required: true + x-example: matrix.org description: | The server name from the ``mxc://`` URI (the authoritory component) - in: path type: string name: mediaId + x-example: ascERGshawAWawugaAcauga required: true description: | The media ID from the ``mxc://`` URI (the path component) - in: query type: integer + x-example: 64 name: width - description: The desired width of the thumbnail. + description: |- + The *desired* width of the thumbnail. The actual thumbnail may not + match the size specified. - in: query type: integer + x-example: 64 name: height - description: The desired height of the thumbnail. + description: |- + The *desired* height of the thumbnail. The actual thumbnail may not + match the size specified. - in: query type: string enum: ["crop", "scale"] name: method + x-example: "scale" description: The desired resizing method. responses: 200: description: "A thumbnail of the requested content." + headers: + Content-Type: + description: "The content type of the thumbnail." + x-example: "image/jpeg" + type: "string" + enum: ["image/jpeg", "image/png"] schema: type: file + name: "" diff --git a/specification/modules/content_repo.rst b/specification/modules/content_repo.rst index 83333f37..de464e20 100644 --- a/specification/modules/content_repo.rst +++ b/specification/modules/content_repo.rst @@ -3,8 +3,23 @@ Content repository .. _module:content: -HTTP API --------- +This module allows users to upload content to their homeserver which is +retrievable from other homeservers. Its' purpose is to allow users to share +attachments in a room. Content locations are represented as Matrix Content (MXC) +URIs. They look like:: + + mxc:/// + + : The name of the homeserver where this content can be found, e.g. matrix.org + : An opaque ID which identifies the content. + +Client behaviour +---------------- + +Clients can upload and download content using the following HTTP APIs. + +{{content_repo_http_api}} + Uploads are POSTed to a resource which returns a token which is used to GET the download. Uploads are POSTed to the sender's local homeserver, but are @@ -49,6 +64,9 @@ width and height are close to the requested size and the aspect matches the requested size. The client should scale the image if it needs to fit within a given rectangle. +Server behaviour +---------------- + Homeservers may generate thumbnails for content uploaded to remote homeservers themselves or may rely on the remote homeserver to thumbnail the content. Homeservers may return thumbnails of a different size to that @@ -58,13 +76,19 @@ Homeservers must never upscale images. Security considerations ----------------------- +The HTTP GET endpoint does not require any authentication. Knowing the URL of +the content is sufficient to retrieve the content, even if the entity isn't in +the room. + +Homeservers have additional concerns: + - Clients may try to upload very large files. Homeservers should not store files that are too large and should not serve them to clients. - Clients may try to upload very large images. Homeservers should not attempt to generate thumbnails for images that are too large. - - Remote homeservers may host very large files or images. Homeserver should not + - Remote homeservers may host very large files or images. Homeservers should not proxy or thumbnail large files or images from remote homeservers. - Clients may try to upload a large number of files. Homeservers should limit the diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index eb3f3e64..472c9d7a 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -31,18 +31,18 @@ Response format: {% for table in endpoint.res_tables -%} {{"``"+table.title+"``" if table.title else "" }} -================== ================= =========================================== +=================== ================= ========================================== Param Type Description -================== ================= =========================================== +=================== ================= ========================================== {% for row in table.rows -%} {# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} +{# Row type needs to prepend spaces to line up with the type column (20 ch) -#} {# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} +{# It also needs to then wrap inside the desc col (42 ch width) -#} {# -#} -{{row.key}}{{row.type|indent(19-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(37)}} +{{row.key}}{{row.type|indent(20-row.key|length)}}{{row.desc|wrap(42,row.req_str | indent(18 - (row.type|length))) |indent_block(38)}} {% endfor -%} -================== ================= =========================================== +=================== ================= ========================================== {% endfor %} {% endif -%} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 50fa784e..04f3bae1 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -151,6 +151,13 @@ class MatrixUnits(Units): # assign value expected for this param val_type = param.get("type") # integer/string + + if param.get("enum"): + val_type = "enum" + desc += ( + " One of: %s" % json.dumps(param.get("enum")) + ) + refType = Units.prop(param, "schema/$ref/") # Error,Event schemaFmt = Units.prop(param, "schema/format") # bytes e.g. uploads if not val_type and refType: @@ -255,7 +262,7 @@ class MatrixUnits(Units): res_type = Units.prop(good_response, "schema/type") if res_type and res_type not in ["object", "array"]: # response is a raw string or something like that - endpoint["res_tables"].append({ + good_table = { "title": None, "rows": [{ "key": good_response["schema"].get("name", ""), @@ -263,7 +270,16 @@ class MatrixUnits(Units): "desc": res.get("description", ""), "req_str": "" }] - }) + } + if good_response.get("headers"): + for (header_name, header) in good_response.get("headers").iteritems(): + good_table["rows"].append({ + "key": header_name, + "type": "Header<" + header["type"] + ">", + "desc": header["description"], + "req_str": "" + }) + endpoint["res_tables"].append(good_table) elif res_type and Units.prop(good_response, "schema/properties"): # response is an object: schema = good_response["schema"] @@ -328,7 +344,7 @@ class MatrixUnits(Units): self.log("Reading swagger API: %s" % filename) with open(os.path.join(path, filename), "r") as f: # strip .yaml - group_name = filename[:-5] + group_name = filename[:-5].replace("-", "_") api = yaml.load(f.read()) api["__meta"] = self._load_swagger_meta(api, group_name) apis[group_name] = api From 8c4d7f50510edb2a8e0932248efed1949bf1715a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 1 Oct 2015 18:03:34 +0100 Subject: [PATCH 36/68] Do not try to parse non-json request examples as json --- api/check_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/check_examples.py b/api/check_examples.py index a0cd0658..f08b2dc1 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -34,7 +34,7 @@ def check_parameter(filepath, request, parameter): example = None try: example_json = schema.get('example') - if example_json: + if example_json and not schema.get("format") == "byte": example = json.loads(example_json) except Exception as e: raise ValueError("Error parsing JSON example request for %r" % ( From 30232f20aa571cbd402ff56eaa970bbc0fc8d34c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 1 Oct 2015 19:13:09 -0500 Subject: [PATCH 37/68] speculator: Merge after fetching, so that /spec/head works --- scripts/speculator/main.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 6ee80150..442400df 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -60,11 +60,10 @@ func (u *User) IsTrusted() bool { } const ( - pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" + pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls" matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git" ) - func gitClone(url string, shared bool) (string, error) { directory := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10)) cmd := exec.Command("git", "clone", url, directory) @@ -80,21 +79,22 @@ func gitClone(url string, shared bool) (string, error) { } func gitCheckout(path, sha string) error { - cmd := exec.Command("git", "checkout", sha) - cmd.Dir = path - err := cmd.Run() - if err != nil { - return fmt.Errorf("error checking out repo: %v", err) + return runGitCommand(path, []string{"checkout", sha}) +} + +func gitFetchAndMerge(path string) error { + if err := runGitCommand(path, []string{"fetch"}); err != nil { + return err } - return nil + return runGitCommand(path, []string{"merge"}) } -func gitFetch(path string) error { - cmd := exec.Command("git", "fetch") +func runGitCommand(path string, args []string) error { + cmd := exec.Command("git", args...) cmd.Dir = path err := cmd.Run() if err != nil { - return fmt.Errorf("error fetching repo: %v", err) + return fmt.Errorf("error running %s: %v", strings.Join(cmd.Args, " "), err) } return nil } @@ -145,7 +145,7 @@ type server struct { // generateAt generates spec from repo at sha. // Returns the path where the generation was done. func (s *server) generateAt(sha string) (dst string, err error) { - err = gitFetch(s.matrixDocCloneURL) + err = gitFetchAndMerge(s.matrixDocCloneURL) if err != nil { return } From a69e03f57778b9ef743cb03430db472e1b0a28a9 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 1 Oct 2015 19:15:30 -0500 Subject: [PATCH 38/68] speculator: Report listening port --- scripts/speculator/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 6ee80150..ece65372 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -362,6 +362,8 @@ func main() { http.HandleFunc("/diff/html/", s.serveHTMLDiff) http.HandleFunc("/healthz", serveText("ok")) http.HandleFunc("/", listPulls) + + fmt.Printf("Listening on port %d\n", *port) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) } From 3d9dbe42e691f446830e348eb680297728dadede Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 2 Oct 2015 10:21:48 +0100 Subject: [PATCH 39/68] Bump to swagger-parser 3.2.1 - remove x- keys on headers Removed x- keys due to https://github.com/BigstickCarpet/swagger-parser/issues/23 --- api/client-server/v1/content-repo.yaml | 5 ----- api/package.json | 2 +- api/validator.js | 10 +++++----- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/api/client-server/v1/content-repo.yaml b/api/client-server/v1/content-repo.yaml index aa47dc18..8e6e8d1a 100644 --- a/api/client-server/v1/content-repo.yaml +++ b/api/client-server/v1/content-repo.yaml @@ -68,15 +68,12 @@ paths: headers: Content-Type: description: "The content type of the file that was previously uploaded." - x-example: "audio/mpeg" type: "string" Content-Disposition: description: "The name of the file that was previously uploaded, if set." - x-example: "attachment;filename=03-cool.mp3" type: "string" schema: type: file - name: "" "/thumbnail/{serverName}/{mediaId}": get: summary: "Download a thumbnail of the content from the content repository." @@ -122,11 +119,9 @@ paths: headers: Content-Type: description: "The content type of the thumbnail." - x-example: "image/jpeg" type: "string" enum: ["image/jpeg", "image/png"] schema: type: file - name: "" diff --git a/api/package.json b/api/package.json index 15193493..84b9dd7b 100644 --- a/api/package.json +++ b/api/package.json @@ -10,6 +10,6 @@ "license": "ISC", "dependencies": { "nopt": "^3.0.2", - "swagger-parser": "^2.4.1" + "swagger-parser": "^3.2.1" } } diff --git a/api/validator.js b/api/validator.js index 3b89a5a3..0d76c09d 100644 --- a/api/validator.js +++ b/api/validator.js @@ -26,11 +26,10 @@ if (!opts.schema) { } -var errFn = function(err, api, metadata) { +var errFn = function(err, api) { if (!err) { return; } - console.log(metadata); console.error(err); process.exit(1); }; @@ -46,11 +45,12 @@ if (isDir) { files.forEach(function(f) { var suffix = ".yaml"; if (f.indexOf(suffix, f.length - suffix.length) > 0) { - parser.parse(path.join(opts.schema, f), function(err, api, metadata) { + parser.validate(path.join(opts.schema, f), function(err, api, metadata) { if (!err) { console.log("%s is valid.", f); } else { + console.error("%s is not valid.", f); errFn(err, api, metadata); } }); @@ -59,12 +59,12 @@ if (isDir) { }); } else{ - parser.parse(opts.schema, function(err, api, metadata) { + parser.validate(opts.schema, function(err, api) { if (!err) { console.log("%s is valid", opts.schema); } else { - errFn(err, api, metadata); + errFn(err, api); } }); }; From dbc72c43aca270d86c128caba1cde434642bcb8a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 2 Oct 2015 10:28:29 +0100 Subject: [PATCH 40/68] s/private_chat_shared_power/trusted_private_chat/ --- specification/1-client_server_api.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index 329a87dd..1c64f021 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -756,7 +756,7 @@ options which can be set when creating a room: Optional: Yes Value: - ``private_chat``, ``private_chat_shared_power`` or ``public_chat`` + ``private_chat``, ``trusted_private_chat`` or ``public_chat`` Description: Convenience parameter for setting various default state events based on a preset. @@ -765,9 +765,9 @@ options which can be set when creating a room: - ``private_chat``: Sets the ``join_rules`` to ``invite`` and ``history_visibility`` to ``shared`` - - ``private_chat_shared_power``: Set the ``join_rules`` to - ``invite``, ``history_visibility`` to ``shared`` and gives all invitees - the same power level as the creator. + - ``trusted_private_chat``: Set the ``join_rules`` to ``invite``, + ``history_visibility`` to ``shared`` and gives all invitees the same + power level as the creator. - ``public_chat``: Sets the ``join_rules`` to ``public`` and ``history_visibility`` to ``shared`` From 912a8ca7605278322cc8f8c921002022ea4e34c7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 2 Oct 2015 10:30:45 +0100 Subject: [PATCH 41/68] Be more explicit about keys that are clobbered --- specification/1-client_server_api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index dd58b357..40b46be0 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -757,7 +757,8 @@ options which can be set when creating a room: Yes Value: Extra keys to be added to the content of the ``m.room.create``. The server - will clober certain keys, e.g. ``creator``. + will clober the following keys: ``creator``. Future versions of this + spec may allow the server to clobber other keys if required. Description: Allows clients to add keys to the content of ``m.room.create``. From 4dabcd112ef787c2f09ddbd61415e145e1b146e3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 2 Oct 2015 10:44:50 +0100 Subject: [PATCH 42/68] Remove redundant info now we have the http api template. Minor tweaks to display of schema with no names but a type --- specification/modules/content_repo.rst | 46 +++++--------------------- templating/matrix_templates/units.py | 3 +- 2 files changed, 11 insertions(+), 38 deletions(-) diff --git a/specification/modules/content_repo.rst b/specification/modules/content_repo.rst index de464e20..9ac5e199 100644 --- a/specification/modules/content_repo.rst +++ b/specification/modules/content_repo.rst @@ -10,9 +10,15 @@ URIs. They look like:: mxc:/// - : The name of the homeserver where this content can be found, e.g. matrix.org + : The name of the homeserver where this content originated, e.g. matrix.org : An opaque ID which identifies the content. +Uploads are POSTed to a resource on the user's local homeserver which returns a +token which is used to GET the download. Content is downloaded from the +recipient's local homeserver, which must first transfer the content from the +origin homeserver using the same API (unless the origin and destination +homeservers are the same). + Client behaviour ---------------- @@ -20,42 +26,8 @@ Clients can upload and download content using the following HTTP APIs. {{content_repo_http_api}} - -Uploads are POSTed to a resource which returns a token which is used to GET -the download. Uploads are POSTed to the sender's local homeserver, but are -downloaded from the recipient's local homeserver, which must thus first transfer -the content from the origin homeserver using the same API (unless the origin -and destination homeservers are the same). The upload/download API is:: - - => POST /_matrix/media/v1/upload HTTP/1.1 - Content-Type: - - - - <= HTTP/1.1 200 OK - Content-Type: application/json - - { "content-uri": "mxc:///" } - - => GET /_matrix/media/v1/download// HTTP/1.1 - - <= HTTP/1.1 200 OK - Content-Type: - Content-Disposition: attachment;filename= - - - -Clients can get thumbnails by supplying a desired width and height and -thumbnailing method:: - - => GET /_matrix/media/v1/thumbnail/ - /?width=&height=&method= HTTP/1.1 - - <= HTTP/1.1 200 OK - Content-Type: image/jpeg or image/png - - - +Thumbnails +~~~~~~~~~~ The thumbnail methods are "crop" and "scale". "scale" tries to return an image where either the width or the height is smaller than the requested size. The client should then scale and letterbox the image if it needs to diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 04f3bae1..1e449b5d 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -260,12 +260,13 @@ class MatrixUnits(Units): if good_response: self.log("Found a 200 response for this API") res_type = Units.prop(good_response, "schema/type") + res_name = Units.prop(good_response, "schema/name") if res_type and res_type not in ["object", "array"]: # response is a raw string or something like that good_table = { "title": None, "rows": [{ - "key": good_response["schema"].get("name", ""), + "key": "<" + res_type + ">" if not res_name else res_name, "type": res_type, "desc": res.get("description", ""), "req_str": "" From a73cc50aa963f05d86783272a1ab105a70a84f34 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 2 Oct 2015 13:03:39 +0100 Subject: [PATCH 43/68] s|client/api/v2_alpha|client/v2_alpha| --- api/client-server/v2_alpha/filter.yaml | 2 +- api/client-server/v2_alpha/sync.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml index 30d52844..37a0a3aa 100644 --- a/api/client-server/v2_alpha/filter.yaml +++ b/api/client-server/v2_alpha/filter.yaml @@ -5,7 +5,7 @@ info: host: localhost:8008 schemes: - https -basePath: /_matrix/client/api/v2_alpha +basePath: /_matrix/client/v2_alpha consumes: - application/json produces: diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 238fab63..7daa33f5 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -5,7 +5,7 @@ info: host: localhost:8008 schemes: - https -basePath: /_matrix/client/api/v2_alpha +basePath: /_matrix/client/v2_alpha consumes: - application/json produces: From 0e8f1b5475f70d8436ec3a1d19886c2fe71ca677 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 2 Oct 2015 07:33:26 -0500 Subject: [PATCH 44/68] Quote args --- scripts/speculator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 442400df..07ffa44e 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -94,7 +94,7 @@ func runGitCommand(path string, args []string) error { cmd.Dir = path err := cmd.Run() if err != nil { - return fmt.Errorf("error running %s: %v", strings.Join(cmd.Args, " "), err) + return fmt.Errorf("error running %q: %v", strings.Join(cmd.Args, " "), err) } return nil } From c8ddf1af09581ca5d54a1e21561d9adce2b3d99d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 2 Oct 2015 16:00:15 +0100 Subject: [PATCH 45/68] Add changelog --- CHANGELOG.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 78aebddc..66f3d0a8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,29 @@ .. in Jenkins. Comments like this are ignored by both RST and the templating .. system. Add the newest release notes beneath this comment. +Specification changes in v0.2.0 (2015-10-02) +============================================ + +This update restructures the specification and begins to aggressively standardise +on using Swagger and JSON Schema to document HTTP endpoints and Events +respectively. It also introduces a number of new concepts to Matrix. + +Additions: + - New section: Feature Profiles. + - New section: Receipts. + - New section: Room history visibility. + - New event: ``m.receipt``. + - New event: ``m.room.canonical_alias`` + - New event: ``m.room.history_visibility`` + - New keys: ``/createRoom`` - allows room "presets" using ``preset`` and + ``initial_state`` keys. + - New endpoint: ``/tokenrefresh`` - Related to refreshing access tokens. + +Modifications: + - Convert most of the older HTTP APIs to Swagger documentation. + - Convert most of the older event formats to JSON Schema. + - Move selected client-server sections to be "Modules". + Specification changes in v0.1.0 (2015-06-01) ============================================ - First numbered release. From bde003fe8618195122dd300b03913dc1b28710c3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 2 Oct 2015 16:14:24 +0100 Subject: [PATCH 46/68] Split the rooms out into a separate top level key. Divide the rooms into separate groups in preparation for adding tag support. Further subdivide the rooms into "joined/invited/archived" based the membership of the user in the room because that membership affects what events the user can view from the room. E.g only users that are joined to a room may see the ephemeral events for the room. --- api/client-server/v2_alpha/sync.yaml | 114 +++++++++++++++++---------- 1 file changed, 74 insertions(+), 40 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 7daa33f5..7a003745 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -5,7 +5,7 @@ info: host: localhost:8008 schemes: - https -basePath: /_matrix/client/v2_alpha +basePath: /_matrix/client/api/v2_alpha consumes: - application/json produces: @@ -73,43 +73,71 @@ paths: ``/sync`` request. rooms: type: object - properties: - roomlist: - type: array - description: |- - A list of rooms that the client needs to update. - items: + description: |- + The updates to rooms, grouped according to the filter. By + default there is a single ``default`` group. + additionalProperties: + joined: + type: array + description: |- + A list of room ids that the user is a member of that + have updates. + items: + type: string. + invited: + type: array + description: |- + A list of room ids that the user has been invited to. + The entries in the room_map will have a ``invite`` key + containing the ``m.room.member`` event of the invite. + items: + type: string. + archived: + type: array + description: |- + A list of room ids that the user has left or been + banned from. The entries in the room_map will have a + ``state`` key and a ``timeline`` key. But will lack the + ``emphemeral`` key. + items: + type: string. + room_map: + description: |- + Map from room id to the updates for that room. The room ids + are referenced from the ``rooms`` key. + type: object + additionalProperties: + type: object + properties: + room_id: + type: string + description: |- + The ID of the room. + event_map: type: object - properties: - room_id: - type: string - description: |- - The ID of the room. - event_map: - type: object - description: |- - A map from event ID to events for this room. The events - are referenced from the ``timeline`` and ``state`` keys - for this room. - additionalProperties: - description: An event object. - type: object - state: - description: |- - The state updates for the room. - allOf: - - $ref: "definitions/room_event_batch.json" - timeline: - description: |- - The timeline of messages and state changes in the room. - allOf: - - $ref: "definitions/timeline_batch.json" - ephemeral: - description: |- - The ephemeral events in the room that aren't recorded - in the timeline or state of the room. E.g. typing. - allOf: - - $ref: "definitions/event_batch.json" + description: |- + A map from event ID to events for this room. The events + are referenced from the ``timeline`` and ``state`` keys + for this room. + additionalProperties: + description: An event object. + type: object + state: + description: |- + The state updates for the room. + allOf: + - $ref: "definitions/room_event_batch.json" + timeline: + description: |- + The timeline of messages and state changes in the room. + allOf: + - $ref: "definitions/timeline_batch.json" + ephemeral: + description: |- + The ephemeral events in the room that aren't recorded + in the timeline or state of the room. E.g. typing. + allOf: + - $ref: "definitions/event_batch.json" presence: description: |- The updates to the presence status of other users. @@ -129,8 +157,14 @@ paths: ] }, "rooms": { - "roomlist": [{ - "room_id": "!726s6s6q:example.com", + "default": { + "invited": [], + "archived": [], + "joined": ["!726s6s6q:example.com"] + } + }, + "room_map": { + "!726s6s6q:example.com": { "event_map": { "$66697273743031:example.com": { "sender": "@alice:example.com", @@ -177,6 +211,6 @@ paths: } ] } - }] + } } } From 047419f2ad9b9b72ed7de144d9c97bcc17e62030 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 2 Oct 2015 16:21:48 +0100 Subject: [PATCH 47/68] Remove the room_id since it is redundent. Remove text about invite event handling till we've thought about it some more. --- api/client-server/v2_alpha/sync.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 7a003745..6b7a1b2a 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -88,8 +88,6 @@ paths: type: array description: |- A list of room ids that the user has been invited to. - The entries in the room_map will have a ``invite`` key - containing the ``m.room.member`` event of the invite. items: type: string. archived: @@ -109,10 +107,6 @@ paths: additionalProperties: type: object properties: - room_id: - type: string - description: |- - The ID of the room. event_map: type: object description: |- From 417c5b53c4c6264f6b91a4bfbaaa4049da6934a0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 2 Oct 2015 16:24:33 +0100 Subject: [PATCH 48/68] Remove duplicate sentences from merge conflicts --- specification/modules/presence.rst | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/specification/modules/presence.rst b/specification/modules/presence.rst index cff4490b..79151c4f 100644 --- a/specification/modules/presence.rst +++ b/specification/modules/presence.rst @@ -21,11 +21,8 @@ A presence list is a list of user IDs whose presence the user wants to follow. To be added to this list, the user being added must be invited by the list owner who must accept the invitation. -Each user has presence information associated with them. This encodes the -"availability" of that user, suitable for display on other clients. -This is transmitted as an ``m.presence`` event and is one of the few events -which are sent *outside the context of a room*. Their presence state is -represented by the ``presence`` key, which is an enum of one of the following: +User's presence state is represented by the ``presence`` key, which is an enum +of one of the following: - ``online`` : The default state when the user is connected to an event stream. @@ -35,7 +32,6 @@ represented by the ``presence`` key, which is an enum of one of the following: explicitly suppressing their profile information from being sent. - ``free_for_chat`` : The user is generally willing to receive messages moreso than default. - Events ------ From 28fd1aa205b28adadee838547aed62e9f2a0a115 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 2 Oct 2015 16:34:06 +0100 Subject: [PATCH 49/68] Go into a bit more detail about feature profiles --- CHANGELOG.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 66f3d0a8..6e3198ef 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,9 +9,14 @@ Specification changes in v0.2.0 (2015-10-02) ============================================ -This update restructures the specification and begins to aggressively standardise -on using Swagger and JSON Schema to document HTTP endpoints and Events -respectively. It also introduces a number of new concepts to Matrix. +This update fundamentally restructures the specification. The specification has +been split into more digestible "modules" which each describe a particular +function (e.g. typing). This was done in order make the specification easier to +maintain and help define which modules are mandatory for certain types +of clients. Types of clients along with the mandatory modules can be found in a +new "Feature Profiles" section. This update also begins to aggressively +standardise on using Swagger and JSON Schema to document HTTP endpoints and +Events respectively. It also introduces a number of new concepts to Matrix. Additions: - New section: Feature Profiles. From 97cdd8106cec02871d58a54f1abcafe7b33481a2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 5 Oct 2015 14:21:16 +0100 Subject: [PATCH 50/68] s|client/api/v2_alpha|/client/v2_alpha| --- api/client-server/v2_alpha/sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 6b7a1b2a..c502d926 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -5,7 +5,7 @@ info: host: localhost:8008 schemes: - https -basePath: /_matrix/client/api/v2_alpha +basePath: /_matrix/client/v2_alpha consumes: - application/json produces: From ed0f6d3ff38848250f830ad8ade02838d9c82f3e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 5 Oct 2015 14:22:50 +0100 Subject: [PATCH 51/68] Typo --- api/client-server/v2_alpha/sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index c502d926..437af251 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -96,7 +96,7 @@ paths: A list of room ids that the user has left or been banned from. The entries in the room_map will have a ``state`` key and a ``timeline`` key. But will lack the - ``emphemeral`` key. + ``ephemeral`` key. items: type: string. room_map: From a7b808c5cd8ec423342a9844241a31bdc339eff8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 5 Oct 2015 15:26:33 +0100 Subject: [PATCH 52/68] Add a invite_state key for holding the state that is bundled with an invite. This is kept separate from the actual state for the room as it may be derived from an incomplete, unverified copy of the state that was bundled with an invite event received over federation. --- api/client-server/v2_alpha/sync.yaml | 38 +++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 437af251..07078823 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -88,6 +88,9 @@ paths: type: array description: |- A list of room ids that the user has been invited to. + The entries in the room_map will have an + ``invite_state`` key. But will lack the ``ephemeral`` + key, the ``timeline`` key and the ``state`` key. items: type: string. archived: @@ -132,6 +135,21 @@ paths: in the timeline or state of the room. E.g. typing. allOf: - $ref: "definitions/event_batch.json" + invite_state: + description: |- + The state of a room that the user has been invited to. + These state events may only have the ``type``, + ``state_key`` and ``content`` keys present. + These events do not replace any state that the client + already has for the room, for example if the client has + archived the room. Instead the client should keep two + separate copies of the state: the one from the + ``invite_state`` and one from the archived ``state``. + If the client joins the room then the current state + will be given as a delta against the archived ``state`` + not the ``invite_state``. + allOf: + - $ref: "definitions/event_batch.json" presence: description: |- The updates to the presence status of other users. @@ -152,7 +170,7 @@ paths: }, "rooms": { "default": { - "invited": [], + "invited": ["!696r7674:example.com"], "archived": [], "joined": ["!726s6s6q:example.com"] } @@ -205,6 +223,24 @@ paths: } ] } + }, + "!696r7674:example.com": { + "invite_state": { + "events": [ + { + "sender": "@alice:example.com", + "type": "m.room.name", + "state_key": "@alice:example.com", + "content": {"name": "My Room Name"} + }, + { + "sender": "@alice:example.com", + "type": "m.room.name", + "state_key": "", + "content": {"membership": "invite"} + } + ] + } } } } From bbb5fa9398447f2196cf7c62ec457c7f717704ce Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 5 Oct 2015 15:36:01 +0100 Subject: [PATCH 53/68] Fix the state_key in the example v2 response, include the "sender" key in the list of keys included with invite_state events --- api/client-server/v2_alpha/sync.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 07078823..49f42c6e 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -138,8 +138,8 @@ paths: invite_state: description: |- The state of a room that the user has been invited to. - These state events may only have the ``type``, - ``state_key`` and ``content`` keys present. + These state events may only have the `sender``, + ``type``, ``state_key`` and ``content`` keys present. These events do not replace any state that the client already has for the room, for example if the client has archived the room. Instead the client should keep two @@ -230,13 +230,13 @@ paths: { "sender": "@alice:example.com", "type": "m.room.name", - "state_key": "@alice:example.com", + "state_key": "", "content": {"name": "My Room Name"} }, { "sender": "@alice:example.com", "type": "m.room.name", - "state_key": "", + "state_key": "@bob:example.com", "content": {"membership": "invite"} } ] From 0e5b00feaaa9c337fd097e13780fa7d06158ac0d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 5 Oct 2015 16:10:25 +0100 Subject: [PATCH 54/68] Fix event type in v2 sync example --- api/client-server/v2_alpha/sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 49f42c6e..1b83d1d9 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -235,7 +235,7 @@ paths: }, { "sender": "@alice:example.com", - "type": "m.room.name", + "type": "m.room.member", "state_key": "@bob:example.com", "content": {"membership": "invite"} } From 0ce533d153e9656818dfdefa565b1356afa4b391 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 6 Oct 2015 13:42:03 +0100 Subject: [PATCH 55/68] Split the room_map into separate sections based on whether they are "joined/invited/archived". Rename the room_map to rooms and remove the grouping indirection. When we want groups then we can add them under a separate key, either at the top-level or as part of the events themselves. --- api/client-server/v2_alpha/sync.yaml | 295 ++++++++++++++------------- 1 file changed, 153 insertions(+), 142 deletions(-) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 1b83d1d9..7d60d639 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -74,82 +74,92 @@ paths: rooms: type: object description: |- - The updates to rooms, grouped according to the filter. By - default there is a single ``default`` group. - additionalProperties: - joined: - type: array - description: |- - A list of room ids that the user is a member of that - have updates. - items: - type: string. - invited: - type: array - description: |- - A list of room ids that the user has been invited to. - The entries in the room_map will have an - ``invite_state`` key. But will lack the ``ephemeral`` - key, the ``timeline`` key and the ``state`` key. - items: - type: string. - archived: - type: array - description: |- - A list of room ids that the user has left or been - banned from. The entries in the room_map will have a - ``state`` key and a ``timeline`` key. But will lack the - ``ephemeral`` key. - items: - type: string. - room_map: - description: |- - Map from room id to the updates for that room. The room ids - are referenced from the ``rooms`` key. - type: object - additionalProperties: - type: object - properties: - event_map: + Updates to rooms. + properties: + joined: + type: object + additionalProperties: + type: object + properties: + event_map: + type: object + description: |- + A map from event ID to events for this room. The + events are referenced from the ``timeline`` and + ``state`` keys for this room. + additionalProperties: + description: An event object. + type: object + state: + description: |- + The state updates for the room. + allOf: + - $ref: "definitions/room_event_batch.json" + timeline: + description: |- + The timeline of messages and state changes in the + room. + allOf: + - $ref: "definitions/timeline_batch.json" + ephemeral: + description: |- + The ephemeral events in the room that aren't + recorded in the timeline or state of the room. + e.g. typing. + allOf: + - $ref: "definitions/event_batch.json" + invited: + type: object + description: |- + The rooms that the user has been invited to. + additionalProperties: type: object - description: |- - A map from event ID to events for this room. The events - are referenced from the ``timeline`` and ``state`` keys - for this room. - additionalProperties: - description: An event object. - type: object - state: - description: |- - The state updates for the room. - allOf: - - $ref: "definitions/room_event_batch.json" - timeline: - description: |- - The timeline of messages and state changes in the room. - allOf: - - $ref: "definitions/timeline_batch.json" - ephemeral: - description: |- - The ephemeral events in the room that aren't recorded - in the timeline or state of the room. E.g. typing. - allOf: - - $ref: "definitions/event_batch.json" - invite_state: - description: |- - The state of a room that the user has been invited to. - These state events may only have the `sender``, - ``type``, ``state_key`` and ``content`` keys present. - These events do not replace any state that the client - already has for the room, for example if the client has - archived the room. Instead the client should keep two - separate copies of the state: the one from the - ``invite_state`` and one from the archived ``state``. - If the client joins the room then the current state - will be given as a delta against the archived ``state`` - not the ``invite_state``. - allOf: - - $ref: "definitions/event_batch.json" + properties: + invite_state: + description: |- + The state of a room that the user has been invited + to. These state events may only have the `sender``, + ``type``, ``state_key`` and ``content`` keys + present. These events do not replace any state that + the client already has for the room, for example if + the client has archived the room. Instead the + client should keep two separate copies of the + state: the one from the ``invite_state`` and one + from the archived ``state``. If the client joins + the room then the current state will be given as a + delta against the archived ``state`` not the + ``invite_state``. + allOf: + - $ref: "definitions/event_batch.json" + archived: + type: object + description: |- + The rooms that the user has left or been banned from. The + entries in the room_map will lack an ``ephemeral`` key. + additionalProperties: + type: object + properties: + event_map: + type: object + description: |- + A map from event ID to events for this room. The + events are referenced from the ``timeline`` and + ``state`` keys for this room. + additionalProperties: + description: An event object. + type: object + state: + description: |- + The state updates for the room up to the point when + the user left. + allOf: + - $ref: "definitions/room_event_batch.json" + timeline: + description: |- + The timeline of messages and state changes in the + room up to the point when the user left. + allOf: + - $ref: "definitions/timeline_batch.json" presence: description: |- The updates to the presence status of other users. @@ -169,78 +179,79 @@ paths: ] }, "rooms": { - "default": { - "invited": ["!696r7674:example.com"], - "archived": [], - "joined": ["!726s6s6q:example.com"] - } - }, - "room_map": { - "!726s6s6q:example.com": { - "event_map": { - "$66697273743031:example.com": { - "sender": "@alice:example.com", - "type": "m.room.member", - "state_key": "@alice:example.com", - "content": {"membership": "join"}, - "origin_server_ts": 1417731086795 - }, - "$7365636s6r6432:example.com": { - "sender": "@bob:example.com", - "type": "m.room.member", - "state_key": "@bob:example.com", - "content": {"membership": "join"}, - "origin_server_ts": 1417731086795 - }, - "$74686972643033:example.com": { - "sender": "@alice:example.com", - "type": "m.room.message", - "unsigned": {"age": "124524", "txn_id": "1234"}, - "content": {"body": "I am a fish", "msgtype": "m.text"}, - "origin_server_ts": 1417731086797 - } - }, - "state": { - "events": [ - "$66697273743031:example.com", - "$7365636s6r6432:example.com" - ] - }, - "timeline": { - "events": [ - "$7365636s6r6432:example.com", - "$74686972643033:example.com" - ], - "limited": true, - "prev_batch": "t34-23535_0_0" - }, - "ephemeral": { - "events": [ - { - "room_id": "!726s6s6q:example.com", - "type": "m.typing", - "content": {"user_ids": ["@alice:example.com"]} - } - ] - } - }, - "!696r7674:example.com": { - "invite_state": { - "events": [ - { + "joined": { + "!726s6s6q:example.com": { + "event_map": { + "$66697273743031:example.com": { "sender": "@alice:example.com", - "type": "m.room.name", - "state_key": "", - "content": {"name": "My Room Name"} + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 }, - { - "sender": "@alice:example.com", + "$7365636s6r6432:example.com": { + "sender": "@bob:example.com", "type": "m.room.member", "state_key": "@bob:example.com", - "content": {"membership": "invite"} + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + "$74686972643033:example.com": { + "sender": "@alice:example.com", + "type": "m.room.message", + "unsigned": {"age": "124524", "txn_id": "1234"}, + "content": { + "body": "I am a fish", + "msgtype": "m.text" + }, + "origin_server_ts": 1417731086797 } - ] + }, + "state": { + "events": [ + "$66697273743031:example.com", + "$7365636s6r6432:example.com" + ] + }, + "timeline": { + "events": [ + "$7365636s6r6432:example.com", + "$74686972643033:example.com" + ], + "limited": true, + "prev_batch": "t34-23535_0_0" + }, + "ephemeral": { + "events": [ + { + "room_id": "!726s6s6q:example.com", + "type": "m.typing", + "content": {"user_ids": ["@alice:example.com"]} + } + ] + } } - } + }, + "invited": { + "!696r7674:example.com": { + "invite_state": { + "events": [ + { + "sender": "@alice:example.com", + "type": "m.room.name", + "state_key": "", + "content": {"name": "My Room Name"} + }, + { + "sender": "@alice:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "invite"} + } + ] + } + } + }, + "archived": {} } } From 39faccb00d7d22a19f08724d47833dff2e53bec3 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 6 Oct 2015 09:38:46 -0500 Subject: [PATCH 56/68] CS API is not just v1 surely --- specification/1-client_server_api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index c63742d1..59e6b68e 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -1,5 +1,5 @@ -Client-Server API v1 -==================== +Client-Server API +================= Overview -------- From 4f606cf3a7098359f68bd292dab43450d7081602 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 6 Oct 2015 17:31:05 +0100 Subject: [PATCH 57/68] Add the fields required by the templating system to v2 sync --- api/client-server/v2_alpha/sync.yaml | 29 ++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml index 7d60d639..266f27bc 100644 --- a/api/client-server/v2_alpha/sync.yaml +++ b/api/client-server/v2_alpha/sync.yaml @@ -72,36 +72,49 @@ paths: The batch token to supply in the ``since`` param of the next ``/sync`` request. rooms: + title: Rooms type: object description: |- Updates to rooms. properties: joined: + title: Joined type: object additionalProperties: + title: Joined Room type: object properties: event_map: + title: EventMap type: object description: |- A map from event ID to events for this room. The events are referenced from the ``timeline`` and ``state`` keys for this room. additionalProperties: + title: Event description: An event object. type: object + allOf: + - $ref: "core-event-schema/event.json" state: + title: State + type: object description: |- The state updates for the room. allOf: - $ref: "definitions/room_event_batch.json" timeline: + title: Timeline + type: object description: |- The timeline of messages and state changes in the room. allOf: - $ref: "definitions/timeline_batch.json" ephemeral: + title: Ephemeral + type: object description: |- The ephemeral events in the room that aren't recorded in the timeline or state of the room. @@ -109,13 +122,17 @@ paths: allOf: - $ref: "definitions/event_batch.json" invited: + title: Invited type: object description: |- The rooms that the user has been invited to. additionalProperties: + title: Invited Room type: object properties: invite_state: + title: InviteState + type: object description: |- The state of a room that the user has been invited to. These state events may only have the `sender``, @@ -132,35 +149,47 @@ paths: allOf: - $ref: "definitions/event_batch.json" archived: + title: Archived type: object description: |- The rooms that the user has left or been banned from. The entries in the room_map will lack an ``ephemeral`` key. additionalProperties: + title: Archived Room type: object properties: event_map: + title: EventMap type: object description: |- A map from event ID to events for this room. The events are referenced from the ``timeline`` and ``state`` keys for this room. additionalProperties: + title: Event description: An event object. type: object + allOf: + - $ref: "core-event-schema/event.json" state: + title: State + type: object description: |- The state updates for the room up to the point when the user left. allOf: - $ref: "definitions/room_event_batch.json" timeline: + title: Timeline + type: object description: |- The timeline of messages and state changes in the room up to the point when the user left. allOf: - $ref: "definitions/timeline_batch.json" presence: + title: Presence + type: object description: |- The updates to the presence status of other users. allOf: From 509d178d5892f8ed218b5919fcc36a77cb14109a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 6 Oct 2015 17:40:28 +0100 Subject: [PATCH 58/68] Fix the receipts API to be valid swagger --- api/client-server/v2_alpha/definitions/error.yaml | 10 ++++++++++ api/client-server/v2_alpha/receipts.yaml | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 api/client-server/v2_alpha/definitions/error.yaml diff --git a/api/client-server/v2_alpha/definitions/error.yaml b/api/client-server/v2_alpha/definitions/error.yaml new file mode 100644 index 00000000..20312ae4 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/error.yaml @@ -0,0 +1,10 @@ +type: object +description: A Matrix-level Error +properties: + errcode: + type: string + description: An error code. + error: + type: string + description: A human-readable error message. +required: ["errcode"] \ No newline at end of file diff --git a/api/client-server/v2_alpha/receipts.yaml b/api/client-server/v2_alpha/receipts.yaml index 4ef435b0..b60f72e6 100644 --- a/api/client-server/v2_alpha/receipts.yaml +++ b/api/client-server/v2_alpha/receipts.yaml @@ -47,6 +47,7 @@ paths: required: true x-example: "$1924376522eioj:example.com" - in: body + name: receipt description: |- Extra receipt information to attach to ``content`` if any. The server will automatically set the ``ts`` field. @@ -65,4 +66,4 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" \ No newline at end of file + "$ref": "definitions/error.yaml" From 034241eb20cb0958bccb78f939c21d036c95d63f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 6 Oct 2015 17:43:04 +0100 Subject: [PATCH 59/68] Missing symlink for the v2 swagger APIs --- api/client-server/v2_alpha/core-event-schema | 1 + 1 file changed, 1 insertion(+) create mode 120000 api/client-server/v2_alpha/core-event-schema diff --git a/api/client-server/v2_alpha/core-event-schema b/api/client-server/v2_alpha/core-event-schema new file mode 120000 index 00000000..b020e6da --- /dev/null +++ b/api/client-server/v2_alpha/core-event-schema @@ -0,0 +1 @@ +../../../event-schemas/schema/v1/core-event-schema \ No newline at end of file From 7ae25731134fbca4a53bc2a43b8a58ec6b7a2e52 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 6 Oct 2015 13:36:59 -0500 Subject: [PATCH 60/68] fix barefaced lie regarding the passivity of ASes --- specification/3-application_service_api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/3-application_service_api.rst b/specification/3-application_service_api.rst index bdde7789..a6e82137 100644 --- a/specification/3-application_service_api.rst +++ b/specification/3-application_service_api.rst @@ -14,7 +14,8 @@ irrespective of the underlying homeserver implementation. Passive Application Services ---------------------------- -"Passive" application services can only observe events from a given home server. +"Passive" application services can only observe events from a given home server, +and inject events into a room they are participating in. They cannot prevent events from being sent, nor can they modify the content of the event being sent. In order to observe events from a homeserver, the homeserver needs to be configured to pass certain types of traffic to the From f759dc30d2896622a3df31460ed43f08c957b229 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 6 Oct 2015 14:13:15 -0500 Subject: [PATCH 61/68] major refresh of the existing FAQ content (WIP) --- supporting-docs/_posts/2015-08-19-faq.md | 247 ++++++++++++++++------- 1 file changed, 172 insertions(+), 75 deletions(-) diff --git a/supporting-docs/_posts/2015-08-19-faq.md b/supporting-docs/_posts/2015-08-19-faq.md index 517b2ccf..8bfbb0f9 100644 --- a/supporting-docs/_posts/2015-08-19-faq.md +++ b/supporting-docs/_posts/2015-08-19-faq.md @@ -37,59 +37,92 @@ FAQ Content ##### What is Matrix? -Matrix is an ambitious new open standard for open, distributed, -real-time communication over IP. It defines interoperable Instant -Messaging and VoIP, providing pragmatic HTTP APIs and open source -reference implementations for creating and running your own real-time -communication infrastructure. +Matrix is an open standard for interoperable, decentralised, +real-time communication over IP. It can be used to power Instant +Messaging, VoIP/WebRTC signalling, Internet of Things communication - or anywhere +you need a standard HTTP API for publishing and subscribing to +data whilst tracking the conversation history. + +| + +Matrix defines the standard, and provides open source reference implementations +of Matrix-compatible Servers, Clients, Client SDKs and Application Services +to help you create new communication solutions or extend the capabilities +and reach of existing ones. ##### What is Matrix's Mission? -Matrix.org's initial inspiration and goal has been to fix the problem of -fragmented IP communications. But Matrix's real potential and ultimate -mission is to be a generic messaging and data synchronisation system for -the web - allowing people, services and devices to easily communicate -with each other with full history. +Matrix's initial goal is to fix the problem of fragmented IP communications: +letting users message and call each other without having to care what app +the other user is on - making it as easy as sending an email. + +| + +The longer term goal is for Matrix to act as a generic HTTP messaging and data +synchronisation system for the whole web - allowing people, services and devices +to easily communicate with each other, empowering users to own and control their +data and select the services and vendors they want to use. ##### What does Matrix provide? -Today Matrix provides a new [open standard](/docs/spec), -[APIs](/docs/api) to integrate a service to the Matrix ecosystem and -reference [open source -implementations](http://github.com/matrix-org/synapse) of the standard. +Matrix provides: + +- [Open Standard](/docs/spec) HTTP APIs for transferring JSON messages (e.g. instant messages, WebRTC signalling), including: + - [Client\<-\>Server API](/docs/spec#client-server-api-v1) - defines how Matrix compatible clients communicate with Matrix homeservers. + - [Server\<-\>Server API](/docs/spec#federation-api) - defines how Matrix homeservers exchange messages and synchronise history with each other. + - [Application Service API](/docs/spec/#application-service-api) - defines how to extend the functionality of Matrix with 'integrations' and bridge to other networks. + - [Modules](/docs/spec/#modules) - specifies features that must be implemented by particular classes of clients. +- Open source reference implementations of: + - Clients (Web (React), iOS, Android) + - Client SDKs (Javascript, Web (React), iOS, Android) + - Homeservers (Synapse) + - Application Services (bridges to IRC, Slack, Skype, Lync and more...) +- The actual ecosystem and community of everyone running Matrix servers and services +- Loads of 3rd party contributions of clients, SDKs, servers and services. + +You can find the full list of Matrix enabled projects at https://matrix.org/blog/try-matrix-now. ##### What does this mean for users? The aim is to provide an analogous ecosystem to email - one where you can communicate with pretty much anyone, without caring what app or server they are using, using whichever app & server you chose to use, -and a nice neutral identity system like an e-mail address or phone +and use a neutral identity system like an e-mail address or phone number to discover people to talk to. ##### What kind of company is Matrix.org? Matrix is an open initiative which acts as a neutral custodian of the Matrix standard. It's not actually incorporated anywhere at the moment -but we are looking at the best legal structure for the future. We are -committed to keeping the Matrix project open. +but we are looking at the best legal structure for the future (and as +of October 2015 we have hopefully found one). Whatever the legal +structure, we are committed to keeping the Matrix project open. ##### Who is funding Matrix.org? -We have been given permission by our employers, Amdocs, to work on -Matrix as an independent non-profit initiative. +Most of the current core contributors to Matrix work at +[Amdocs](http://amdocs.com), who have kindly given us permission to work +on Matrix as an independent non-profit initiative. Other contributors +are funded by their own employers or donate their own time to the project. ##### Who is building Matrix? -We're a team of ~10 people with decades of experience building custom +The core team is ~10 people with extensive experience in building custom VoIP and Messaging apps for mobile network operators. Most of us have -day jobs at Amdocs or OpenMarket, but we are supported by a mix of -freelancers and volunteers. +day jobs at [Amdocs](http://amdocs.com) or [OpenMarket](http://openmarket.com), +but there are an increasing number of contributors from other companies and +folks all over the internet. ##### Why are you called Matrix? We are called Matrix because we provide a structure in which all communication can be matrixed together. +| + +No, it's nothing to do with the film (although you could go and build virtual +worlds on top of Matrix if you wanted :) + ##### Why have you released this as open source? We believe that any open standard defining interoperable communication @@ -102,16 +135,18 @@ and build on top of it. ##### What do you mean by open? Matrix is an open standard, meaning that we have freely published the -details for how to interface with Matrix compliant servers and clients, -and encourage anyone and everyone to interface with them.  We also +details for how to communicate interoperably using the Matrix set of +HTTP APIs. We encourage anyone and everyone to use the APIs and build +their own projects which implement them and so benefit from +interoperability with the rest of the Matrix ecosystem. We also ensure the standard is not encumbered by any known patent licensing requirements. -| +| Matrix is also open source, meaning that we have released the source -code of the reference servers and clients to the public domain under the -[Apache Licence v2](http://www.apache.org/licenses/LICENSE-2.0.html), to +code of the reference servers, clients and services to the public domain +under the [Apache Licence v2](http://www.apache.org/licenses/LICENSE-2.0.html), to encourage anyone and everyone to run their own servers and clients, and enhance them and contribute their enhancements as they see fit. @@ -120,7 +155,7 @@ enhance them and contribute their enhancements as they see fit. Federation allows separate deployments of a communication service to communicate with each other - for instance a mail server run by Google federates with a mail server run by Microsoft when you send email from -@gmail.com to @outlook.com. +@gmail.com to @hotmail.com. | @@ -145,16 +180,18 @@ VoIP and IM. ##### Why has no-one done this before? There have been several attempts before including SIP, XMPP and RCS. - All of these have had some level of success, but + All of these have had some level of success, but many different technological/usability/economic factors have ended up limiting their -success in providing true open federation. +success. Unfortunately, we've not ended up in a world where everyone +has a SIP URI or Jabber ID on their business card, or a phone that +actually uses RCS. ##### What is the difference between Matrix and IRC? We love IRC.  In fact, as of today the core Matrix team still uses it as our primary communication tool. Between us we've written IRCds, IRC bots and admined dreamforge, UnrealIRCd, epona, ircservices and several -others.  That said, it has some limitations that Matrix seeks to improve +others. That said, it has some limitations that Matrix seeks to improve on: - Text only @@ -163,18 +200,24 @@ on: - No presence support - Fragmented identity model - No open federation -- No standard APIs, just an archaic TCP line protocol +- No standard APIs, just a rather limited TCP line protocol - Non-standardised federation protocol - No built-in end-to-end encryption - Disruptive net-splits - Non-extensible +[IRCv3](http://ircv3.net) exists and is addressing some of issues; +this is great news and we wish them well. It's almost a contradiction +in terms to get competitive between openly interoperable communication +projects - we look forward to increasing the richness of Matrix\<-\>IRC +bridges as the project progresses. + ##### What is the difference between Matrix and XMPP? The Matrix team used XMPP (Openfire, ejabberd, spectrum, asmack, XMPPFramework) for IM before starting to experiment with open HTTP APIs -as an alternative.   The main issues with XMPP that drove us in this -direction were: +as an alternative in around 2012. The main issues with XMPP that +drove us in this direction were: - Not particularly web-friendly - you can't easily speak XMPP from a web browser. (N.B. Nowadays you have options like XMPP-FTW and @@ -182,8 +225,8 @@ direction were: - Single logical server per MUC is a single point of control and availability. (MUCs can be distributed over multiple physical servers, but they still sit behind a single logical JID and domain. - FMUC improves this with a similar approach to Matrix, but at time of - writing there are no open implementations.) + FMUC improves this with a similar approach to Matrix, but as of Oct + 2015 there are no open source implementations.) - History synchronisation is very much a second class citizen feature - Stanzas aren't framed or reliably delivered without extensions. (See [wiki.xmpp.org](http://wiki.xmpp.org/web/Myths#Myth_Four:_XMPP_is_unreliable_without_a_bunch_of_extensions.) @@ -191,7 +234,8 @@ direction were: - Multiple device support is limited. (Apparently Carbons and MAM help with this) - Baseline feature set is so minimal that fragmentation of features - between clients and servers is common + between clients and servers is common, especially as interoperability + profiles for features have fallen behind (as of July 2015) - No strong identity system (i.e. no standard E2E PKI, unless you count X.509 certs, which [are questionable](http://www.thoughtcrime.org/blog/ssl-and-the-future-of-authenticity/)) @@ -199,31 +243,42 @@ direction were: bandwidth-efficient transports. (Since the time of writing a [Push XEP has appeared](http://xmpp.org/extensions/xep-0357.html), and [wiki.xmpp.org](http://wiki.xmpp.org/web/Myths#Myth_Three:_It.27s_too_bandwidth-inefficient_for_mobile.) - claims that XMPP runs fine over a 9600bps + 30s latency link.) + claims that XMPP runs "fine" over a 9600bps + 30s latency link.) The whole subject of XMPP vs Matrix seems to bring out the worst in -people. We think of the standards as being quite different; at its core +people. Rather than fighting over which open interoperable communication +standard works the best, we should just collaborate and bridge everything +together. The more federation and interoperability the better. + +| + +We think of Matrix and XMPP as being quite different; at its core Matrix can be thought of as an eventually consistent global JSON db with an HTTP API and pubsub semantics - whilst XMPP can be thought of as a message passing protocol. You can use them both to build chat systems; you can use them both to build pubsub systems; each comes with different -tradeoffs. Matrix has a 'kitchen sink' baseline of functionality; XMPP -has a deliberately minimal baseline set of functionality. If XMPP does -what you need it to do, then we're genuinely happy for you :) Meanwhile, -rather than competing, an XMPP Bridge like [Skaverat's xmpptrix -beta](https://github.com/SkaveRat/xmpptrix) has potential to let both -environments coexist and make the most of each other's benefits. +tradeoffs. Matrix has a deliberately extensive 'kitchen sink' baseline +of functionality; XMPP has a deliberately minimal baseline set of +functionality. If XMPP does what you need it to do, then we're genuinely +happy for you :) Meanwhile, rather than competing, an XMPP Bridge like +[Skaverat's xmpptrix beta](https://github.com/SkaveRat/xmpptrix) or +[jfred's matrix-xmpp-bridge](https://github.com/jfrederickson/matrix-xmpp-bridge) +or Matrix.org's own [matrix-appservice-purple](https://github.com/matrix-org/matrix-appservice-purple) +has potential to let both environments coexist and make the most of each +other's benefits. ##### What is the difference between Matrix and PSYC? PSYC is a open federated messaging protocol loosely inspired by IRC.  In version 1 it was a standalone protocol, and in version 2 it is being -reutilised as the messaging layer on top of GNUnet.  We honestly don't +reutilised as a messaging layer on top of GNUnet.  We honestly don't know that much about it, beyond trying to use psycd as an XMPP\<-\>IRC bridge in 2010. Matrix differentiates primarily by providing simple HTTP APIs rather than the more exotic compact line protocol in PSYC v1 or the -complicated GNUnet stack in v2.  Meanwhile, Matrix doesn't provide of -the metadata protection guarantees that GNUnet/PSYC aims for. +comprehensive GNUnet stack in v2, and Matrix focuses more on decentralised +conversation history rather than just decentralised chat servers. +On the other hand, Matrix doesn't provide the metadata protection +guarantees that GNUnet/PSYC aims for. | @@ -233,29 +288,56 @@ PSYC's views on Matrix. ##### What is the difference between Matrix and Tox? Tox.im looks to be a very cool clone of Skype - a fully decentralised -peer-to-peer network.  Matrix is deliberately not peer-to-peer; instead -each user has a well-defined homeserver which stores his data and that -he can depend upon.  Matrix provides HTTP APIs; Tox.im provides C APIs. - We haven't actually played with Tox at all yet. +peer-to-peer network.  Matrix is deliberately not a 'pure' peer-to-peer +system; instead each user has a well-defined homeserver which stores +his data and that he can depend upon.  Matrix provides HTTP APIs; +Tox.im provides C APIs. As of October 2015 Tox doesn't seem to have an +answer yet for decentralised conversation history storage. ##### How does Matrix compare with something like Trillian or Pidgin? Trillian and Pidgin and similar aggregating IM clients merge all your IM -activity into a single user experience.  However, your history and +activity into a single app.  However, your history and identity is still fragmented across the networks.  People can't find you easily, and your history is fragmented (other than on the device where the client runs).   And rather than being able to chose the right app for the job when communicating with people, you are pushed towards relying on a specific aggregation app. +Matrix lets you get the best of both worlds by linking to all the +different networks (XMPP, AIM, ICQ, Lync, Skype etc) on the serverside, +using bridges which can be run by anyone. Matrix then provides a simple +standard HTTP API to access any of these networks, and lets you choose +whichever client you prefer (either as a 'native' Matrix client or using +a non-Matrix client from one of the networks which has been bridged in). + ##### What Matrix compliant apps are there? -None yet, other than our examples.  It's early days :) +Quite a few, ranging from the glossy mass-market to the geeky command-line. There's even an emacs macro. Check out [https://matrix.org/blog/try-matrix-now] for the current +list of Matrix enabled projects. -##### Why do you think existing apps will ever join this? +##### What bridges to other networks are available? + +The number of 'bridges' which integrate existing communication networks into +Matrix are growing on a daily basis - both written by the Matrix core team +and contributed by the wider community. The full list can be seen at +https://matrix.org/blog/try-matrix-now, but the core ones as of Oct 2015 include: + + * [matrix-appservice-irc](https://github.com/matrix-org/matrix-appservice-irc) - an increasingly comprehensive Matrix\<-\>IRC bridge + * [matrix-appservice-verto](https://github.com/matrix-org/matrix-appservice-verto) - links from Matrix to FreeSWITCH via the Verto protocol + * [matrix-appservice-slack](https://github.com/matrix-org/matrix-appservice-slack) - a basic bridge to Slack + * [matrix-appservice-purple](https://github.com/matrix-org/matrix-appservice-purple) - lets you access any of the 20+ protocols supported by + [libpurple](https://developer.pidgin.im/wiki/WhatIsLibpurple), including + Skype, Lync, + * [matrix-appservice-bridge](https://github.com/matrix-org/matrix-appservice-bridge) - a general NodeJS framework for writing bridges + +Writing new bridges is incredibly fun and easy - see the [matrix-appservice-bridge HOWTO](https://github.com/matrix-org/matrix-appservice-bridge/blob/master/HOWTO.md) +for an example of how to write a fully functional Slack bridge in less than 100 lines of code! + +##### Why do you think existing apps will ever join this officially? We firmly believe it is what is right for the consumer. As people begin -to use interoperable communications tools service providers will see the +to use interoperable communications tools, service providers will see the benefit and compete on quality of service, security and features rather than relying on locking people into their walled garden. We believe as soon as users see the availability and benefits of interoperable @@ -264,9 +346,9 @@ services they will demand it. ##### Why aren't you doing this through the IETF? or W3C? or 3GPP? We do recognise the advantages of working with existing standards -bodies. We have been focused on writing code and getting it out. As -Matrix matures it may well be appropriate to work with an official -standard body. +bodies. We have been focused on writing code and getting it out, and the standard has been evolving rapidly since initial release in September 2014. +Once the standard has matured sufficiently it may well be appropriate to work with an official +standard body to maintain it going forwards. | @@ -274,18 +356,19 @@ standard body. ##### How do I get an account and get started? -The quickest way is to just jump to the demo webclient at -[http://matrix.org/beta](http://matrix.org/beta) and sign up.  Please note that you can point the -webclient to access any homeserver - you don't have to use matrix.org, +The quickest way is to pick a client from https://matrix.org/blog/try-matrix-now and sign up. +Please note that you can point clients to access any homeserver - you don't have to use matrix.org, although as of day 1, matrix.org is the only communal homeserver available. ##### What can I actually do with this? -The demo webclient provides a simple chatroom interface to Matrix - +A typical client provides a simple chatroom interface to Matrix - letting the user interact with users and rooms anywhere within the Matrix federation.  Text and image messages are supported, and basic voice-only VoIP calling via WebRTC is supported in one-to-one rooms. +(As of October 2015, experimental multi-way calling is also available +on Vector.im). ##### How do I connect my homeserver to the public Matrix network? @@ -295,11 +378,21 @@ for details ##### How do I Matrix-enable my existing app? -See the [Client-Server API -HOWTO](http://matrix.org/docs/howtos/client-server.html) for an example -of how to use Matrix's client-server API to let your app communicate -with users via Matrix.  We're currently working out the best way to -integrate your application's existing identity system with Matrix. +If your app doesn't have any communication capability already, you'll want +to use one of the Matrix client SDKs to add it in. These come in different +levels of sophistication - ranging from a simple HTTP API wrapper (like matrix-js-sdk, matrix-ios-sdk or matrix-android-sdk) +through to reusable UI components (like matrix-react-sdk and matrix-ios-kit). Pick +the one for your platform, or a 3rd party one if none of the above work for you, +and get plugging it in. You'll probably also want to read the [Client-Server API +HOWTO](http://matrix.org/docs/howtos/client-server.html) too. + +If you already have communication infrastructure set up (XMPP, custom HTTP, or whatever), +then you'll want to run a bridge to expose it to the wider Matrix ecosystem. +See [matrix-appservice-bridge HOWTO](https://github.com/matrix-org/matrix-appservice-bridge/blob/master/HOWTO.md) for a +guide of how to write bridges using the matrix-appservice-bridge framework, or co-opt one +from the list at https://matrix.org/blog/try-matrix-now. +[Application Service API](/docs/spec/#application-service-api) gives the details of the API +that bridges have to implement. ##### How can I write a client on Matrix? @@ -308,16 +401,20 @@ HOWTO](http://matrix.org/docs/howtos/client-server.html) and the [API docs](/docs/api) and [the Spec](/docs/spec) for all the details you need to write a client. -##### *How can I help out with this?* +##### How can I help out with this? + +Come say hi on #matrix:matrix.org! Install synapse and tell us how you get on. Critique the spec.  Write +clients. Write bridges! Run bridges! Nose around in [Jira](https://matrix.org/jira) and +send us some pull requests on github to fix some bugs or add some features! You could even +try to write a homeserver (but be warned, Matrix's architecture makes homeservers orders of +magnitude harder than clients or bridges.) -Install synapse and tell us how you get on. Critique the spec.  Write -clients.  Just come say hi on [\#matrix:matrix.org](/alpha) or the -[mailing lists](/mailman/listinfo/matrix-users)! +See [CONTRIBUTING.rst](http://github.com/matrix-org/synapse/tree/master/CONTRIBUTING.rst) for +full details on how to contribute to the project. All are welcome! ##### Where can I get support? -[\#matrix:matrix.org](/alpha), \#matrix on irc.freenode.net or -the [mailing lists](/mailman/listinfo/matrix-users) are your best bets. +\#matrix:matrix.org aka \#matrix on irc.freenode.is your best bet. ##### How do I register custom matrix event types? @@ -328,7 +425,7 @@ use the [mailing list](/mailman/listinfo/matrix-users) for now. ##### How mature is this? We started working on Matrix in July 2014, and have opened it to the -public in September 2014.  It's early days, and under no circumstances +public in September 2014. It's early days, and under no circumstances should you use Matrix or Synapse for anything other than experimentation and learning at this point.  Obviously the spec and apps are maturing rapidly, but as of the time of writing APIs are not frozen and the apps From 6e6585cebca0d8c6e72b6d0964c6e429f253284b Mon Sep 17 00:00:00 2001 From: Oddvar Lovaas Date: Tue, 6 Oct 2015 20:21:49 +0100 Subject: [PATCH 62/68] FAQ draft --- supporting-docs/_posts/2015-08-19-faq.md | 72 ++++++++++++++++++------ 1 file changed, 55 insertions(+), 17 deletions(-) diff --git a/supporting-docs/_posts/2015-08-19-faq.md b/supporting-docs/_posts/2015-08-19-faq.md index 517b2ccf..dc7dc723 100644 --- a/supporting-docs/_posts/2015-08-19-faq.md +++ b/supporting-docs/_posts/2015-08-19-faq.md @@ -31,17 +31,13 @@ FAQ Content * TOC -{:toc} +{:toc .toc} ### General ##### What is Matrix? -Matrix is an ambitious new open standard for open, distributed, -real-time communication over IP. It defines interoperable Instant -Messaging and VoIP, providing pragmatic HTTP APIs and open source -reference implementations for creating and running your own real-time -communication infrastructure. +Matrix is an ambitious new open standard for open, distributed, real-time communication over IP. It defines interoperable Instant Messaging and VoIP, providing pragmatic HTTP APIs and open source reference implementations for creating and running your own real-time communication infrastructure. ##### What is Matrix's Mission? @@ -268,8 +264,6 @@ bodies. We have been focused on writing code and getting it out. As Matrix matures it may well be appropriate to work with an official standard body. -| - ### Quick Start ##### How do I get an account and get started? @@ -308,7 +302,7 @@ HOWTO](http://matrix.org/docs/howtos/client-server.html) and the [API docs](/docs/api) and [the Spec](/docs/spec) for all the details you need to write a client. -##### *How can I help out with this?* +##### How can I help out with this? Install synapse and tell us how you get on. Critique the spec.  Write clients.  Just come say hi on [\#matrix:matrix.org](/alpha) or the @@ -341,45 +335,75 @@ soon!* *In the mean time, don't hesitate to get in touch on [\#matrix:matrix.org](/alpha) or the [mailing lists](/mailman/listinfo/matrix-users)!* -| - ### Standard ##### What is a home server? -##### What is an identity sever? +**Users in Matrix use one or more clients to communicate. This could be a web client, a command line client, a mobile client - or multiple of these being used simultaneously by the same user. The clients are registered to a single homeserver, which stores the communication history and account information, and shares data with the wider Matrix ecosystem by synchronising communication history with other homeservers.** + +##### What is an identity server? + +**Users in Matrix are identified via their matrix user ID (MXID). However, existing 3rd party ID namespaces can also be used in order to identify Matrix users. A Matrix "Identity" describes both the user ID and any other existing IDs from third party namespaces linked to their account.** + +| + +**Matrix users can link third-party IDs (3PIDs) such as email addresses, social network accounts and phone numbers to their user ID. Linking 3PIDs creates a mapping from a 3PID to a user ID. This mapping can then be used by Matrix users in order to discover the MXIDs of their contacts.** + +| + +**In order to ensure that the mapping from 3PID to user ID is genuine, a globally federated cluster of trusted "Identity Servers" (IS) are used to verify the 3PID and persist and replicate the mappings. +Usage of an IS is not required in order for a client application to be part of the Matrix ecosystem. However, without one clients will not be able to look up user IDs using 3PIDs.** ##### Where do my conversations get stored? +**Each homeserver stores the communication history and account information for all of its clients, and shares data with the wider Matrix ecosystem by synchronising communication history with other homeservers and their clients. Clients typically communicate with each other by emitting events in the context of a virtual room. Room data is replicated across all of the homeservers *whose users are participating in a given room*.** + ##### What is a 3PID? +**Third-party IDs (3PIDs) are IDs from other systems or contexts, such as email addresses, social network accounts and phone numbers.** + ##### How do you do VoIP calls on Matrix? +**Voice (and video) over Matrix is built on the WebRTC 1.0 standard. Call events are sent to a room, like any other event. This means that clients must only send call events to rooms with exactly two participants as currently the WebRTC standard is based around two-party communication. Group calls are on the to-do list, though!** + ##### Can I log into other homeservers with my username and password? +**Currently, no. We are looking at options for enabling multi-server access for users, and might add this feature at a later stage.** + ##### Why Apache Licence? +**The Apache Licence is a permissive licence. We want the Matrix protocol itself to be free and open, but people are free to create both free and commercial apps and services that uses the protocol. In our opinion, any Matrix-service only enhances the Matrix ecosystem.** + ##### Can I write a Matrix homeserver? +**Yes. Matrix is just a spec, so implementations of the spec are very welcome! It should be noted that at the moment, changes are still being made to the spec, so if you want to write a Matrix homeserver, it is strongly recommended that you chat to the Matrix.org devs in [\#matrix:matrix.org](https://matrix.org/beta/#/room/%23matrix:matrix.org) first! You can also read about the [Federation API here]( https://github.com/matrix-org/matrix-doc/blob/master/specification/30_server_server_api.rst).** + ##### How secure is this? +**Server-server traffic is mandatorily TLS from the outset. Server-client traffic mandates transport layer encryption other than for tinkering. Clients that support PKI publish their public keys, and may encrypt +and sign their messages for E2E security. "Well behaved" clients should participate in key escrow servers to allow private key submission for law enforcement. End-to-end encryption for group chat is supported through a per-room encryption key which is shared 1:1 between participating members.** + ##### Why aren't you using an ORM layer like SqlAlchemy?   -| ### APIs ##### How do I join the global Matrix federation? +**You can download and run one of the available Matrix servers - please see [this guide](http://matrix.org/docs/guides/getting_involved.html#run) for details!** + ##### What ports do I have to open up to join the global Matrix federation? -| +**That is up to you! Look at ["Setting up Federation"](https://github.com/matrix-org/synapse#setting-up-federation) in the Synapse readme file for details.** ### Reference Implementations ##### What is Matrix built on - and why? -##### How do I run my own home server? + +##### How do I run my own homeserver? + +**Follow the instructions for the homeserver you want to run. If you want to run Synapse, the homeserver created by Matrix.org, follow [these instructions](https://github.com/matrix-org/synapse#synapse-installation).** ##### Can I run my own identity server? @@ -396,18 +420,32 @@ addresses) to matrix IDs to aid user discovery**. ##### What is Synapse? +**Synapse is a reference "homeserver" implementation of Matrix from the core development team at matrix.org, written in Python/Twisted for clarity and simplicity. It is intended to showcase the concept of Matrix and let folks see the spec in the context of a codebase and let you run your own homeserver and generally help bootstrap the ecosystem.** + ##### Why is Synapse in Python/Twisted? ##### What are Synapse's platform requirements? + + ##### What are the Synapse webclient's requirements? ##### Where is the mobile app? +**The mobile apps can be downloaded from the [Google Play store](https://play.google.com/store/apps/details?id=org.matrix.androidsdk.alpha) + and [Apple store](https://itunes.apple.com/gb/app/matrix-console/id970074271).** + +| + +**For the Android app, you can also install the latest development version +built by [Jenkins](http://www.matrix.org/jenkins/job/AndroidConsoleDevelop/lastBuild/artifact/console/build/outputs/apk/console-alpha-debug.apk).** + ##### What decides the room member order on the webclient? +**The members are ordered by their *last active time*.** + | -Any other question? Please contact us on -[\#matrix:matrix.org](/alpha) or the [mailing +Any other questions? Please contact us in +[\#matrix:matrix.org](https://matrix.org/beta/#/room/%23matrix:matrix.org) or the [mailing lists](/mailman/listinfo/matrix-users)! From 2621c22956e300c32ab1f274805121dbc42a3fff Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 6 Oct 2015 13:36:59 -0500 Subject: [PATCH 63/68] fix barefaced lie regarding the passivity of ASes --- specification/3-application_service_api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/3-application_service_api.rst b/specification/3-application_service_api.rst index bdde7789..a6e82137 100644 --- a/specification/3-application_service_api.rst +++ b/specification/3-application_service_api.rst @@ -14,7 +14,8 @@ irrespective of the underlying homeserver implementation. Passive Application Services ---------------------------- -"Passive" application services can only observe events from a given home server. +"Passive" application services can only observe events from a given home server, +and inject events into a room they are participating in. They cannot prevent events from being sent, nor can they modify the content of the event being sent. In order to observe events from a homeserver, the homeserver needs to be configured to pass certain types of traffic to the From ed9028f0fcad707cc51446fcbc510d28d7fb9346 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 6 Oct 2015 14:46:13 -0500 Subject: [PATCH 64/68] merge again... --- supporting-docs/_posts/2015-08-19-faq.md | 241 ++++++++++++++++------- 1 file changed, 171 insertions(+), 70 deletions(-) diff --git a/supporting-docs/_posts/2015-08-19-faq.md b/supporting-docs/_posts/2015-08-19-faq.md index dc7dc723..87e10f2c 100644 --- a/supporting-docs/_posts/2015-08-19-faq.md +++ b/supporting-docs/_posts/2015-08-19-faq.md @@ -37,55 +37,92 @@ FAQ Content ##### What is Matrix? -Matrix is an ambitious new open standard for open, distributed, real-time communication over IP. It defines interoperable Instant Messaging and VoIP, providing pragmatic HTTP APIs and open source reference implementations for creating and running your own real-time communication infrastructure. +Matrix is an open standard for interoperable, decentralised, +real-time communication over IP. It can be used to power Instant +Messaging, VoIP/WebRTC signalling, Internet of Things communication - or anywhere +you need a standard HTTP API for publishing and subscribing to +data whilst tracking the conversation history. + +| + +Matrix defines the standard, and provides open source reference implementations +of Matrix-compatible Servers, Clients, Client SDKs and Application Services +to help you create new communication solutions or extend the capabilities +and reach of existing ones. ##### What is Matrix's Mission? -Matrix.org's initial inspiration and goal has been to fix the problem of -fragmented IP communications. But Matrix's real potential and ultimate -mission is to be a generic messaging and data synchronisation system for -the web - allowing people, services and devices to easily communicate -with each other with full history. +Matrix's initial goal is to fix the problem of fragmented IP communications: +letting users message and call each other without having to care what app +the other user is on - making it as easy as sending an email. + +| + +The longer term goal is for Matrix to act as a generic HTTP messaging and data +synchronisation system for the whole web - allowing people, services and devices +to easily communicate with each other, empowering users to own and control their +data and select the services and vendors they want to use. ##### What does Matrix provide? -Today Matrix provides a new [open standard](/docs/spec), -[APIs](/docs/api) to integrate a service to the Matrix ecosystem and -reference [open source -implementations](http://github.com/matrix-org/synapse) of the standard. +Matrix provides: + +- [Open Standard](/docs/spec) HTTP APIs for transferring JSON messages (e.g. instant messages, WebRTC signalling), including: + - [Client\<-\>Server API](/docs/spec#client-server-api-v1) - defines how Matrix compatible clients communicate with Matrix homeservers. + - [Server\<-\>Server API](/docs/spec#federation-api) - defines how Matrix homeservers exchange messages and synchronise history with each other. + - [Application Service API](/docs/spec/#application-service-api) - defines how to extend the functionality of Matrix with 'integrations' and bridge to other networks. + - [Modules](/docs/spec/#modules) - specifies features that must be implemented by particular classes of clients. +- Open source reference implementations of: + - Clients (Web (React), iOS, Android) + - Client SDKs (Javascript, Web (React), iOS, Android) + - Homeservers (Synapse) + - Application Services (bridges to IRC, Slack, Skype, Lync and more...) +- The actual ecosystem and community of everyone running Matrix servers and services +- Loads of 3rd party contributions of clients, SDKs, servers and services. + +You can find the full list of Matrix enabled projects at https://matrix.org/blog/try-matrix-now. ##### What does this mean for users? The aim is to provide an analogous ecosystem to email - one where you can communicate with pretty much anyone, without caring what app or server they are using, using whichever app & server you chose to use, -and a nice neutral identity system like an e-mail address or phone +and use a neutral identity system like an e-mail address or phone number to discover people to talk to. ##### What kind of company is Matrix.org? Matrix is an open initiative which acts as a neutral custodian of the Matrix standard. It's not actually incorporated anywhere at the moment -but we are looking at the best legal structure for the future. We are -committed to keeping the Matrix project open. +but we are looking at the best legal structure for the future (and as +of October 2015 we have hopefully found one). Whatever the legal +structure, we are committed to keeping the Matrix project open. ##### Who is funding Matrix.org? -We have been given permission by our employers, Amdocs, to work on -Matrix as an independent non-profit initiative. +Most of the current core contributors to Matrix work at +[Amdocs](http://amdocs.com), who have kindly given us permission to work +on Matrix as an independent non-profit initiative. Other contributors +are funded by their own employers or donate their own time to the project. ##### Who is building Matrix? -We're a team of ~10 people with decades of experience building custom +The core team is ~10 people with extensive experience in building custom VoIP and Messaging apps for mobile network operators. Most of us have -day jobs at Amdocs or OpenMarket, but we are supported by a mix of -freelancers and volunteers. +day jobs at [Amdocs](http://amdocs.com) or [OpenMarket](http://openmarket.com), +but there are an increasing number of contributors from other companies and +folks all over the internet. ##### Why are you called Matrix? We are called Matrix because we provide a structure in which all communication can be matrixed together. +| + +No, it's nothing to do with the film (although you could go and build virtual +worlds on top of Matrix if you wanted :) + ##### Why have you released this as open source? We believe that any open standard defining interoperable communication @@ -98,16 +135,18 @@ and build on top of it. ##### What do you mean by open? Matrix is an open standard, meaning that we have freely published the -details for how to interface with Matrix compliant servers and clients, -and encourage anyone and everyone to interface with them.  We also +details for how to communicate interoperably using the Matrix set of +HTTP APIs. We encourage anyone and everyone to use the APIs and build +their own projects which implement them and so benefit from +interoperability with the rest of the Matrix ecosystem. We also ensure the standard is not encumbered by any known patent licensing requirements. -| +| Matrix is also open source, meaning that we have released the source -code of the reference servers and clients to the public domain under the -[Apache Licence v2](http://www.apache.org/licenses/LICENSE-2.0.html), to +code of the reference servers, clients and services to the public domain +under the [Apache Licence v2](http://www.apache.org/licenses/LICENSE-2.0.html), to encourage anyone and everyone to run their own servers and clients, and enhance them and contribute their enhancements as they see fit. @@ -116,7 +155,7 @@ enhance them and contribute their enhancements as they see fit. Federation allows separate deployments of a communication service to communicate with each other - for instance a mail server run by Google federates with a mail server run by Microsoft when you send email from -@gmail.com to @outlook.com. +@gmail.com to @hotmail.com. | @@ -141,16 +180,18 @@ VoIP and IM. ##### Why has no-one done this before? There have been several attempts before including SIP, XMPP and RCS. - All of these have had some level of success, but + All of these have had some level of success, but many different technological/usability/economic factors have ended up limiting their -success in providing true open federation. +success. Unfortunately, we've not ended up in a world where everyone +has a SIP URI or Jabber ID on their business card, or a phone that +actually uses RCS. ##### What is the difference between Matrix and IRC? We love IRC.  In fact, as of today the core Matrix team still uses it as our primary communication tool. Between us we've written IRCds, IRC bots and admined dreamforge, UnrealIRCd, epona, ircservices and several -others.  That said, it has some limitations that Matrix seeks to improve +others. That said, it has some limitations that Matrix seeks to improve on: - Text only @@ -159,18 +200,24 @@ on: - No presence support - Fragmented identity model - No open federation -- No standard APIs, just an archaic TCP line protocol +- No standard APIs, just a rather limited TCP line protocol - Non-standardised federation protocol - No built-in end-to-end encryption - Disruptive net-splits - Non-extensible +[IRCv3](http://ircv3.net) exists and is addressing some of issues; +this is great news and we wish them well. It's almost a contradiction +in terms to get competitive between openly interoperable communication +projects - we look forward to increasing the richness of Matrix\<-\>IRC +bridges as the project progresses. + ##### What is the difference between Matrix and XMPP? The Matrix team used XMPP (Openfire, ejabberd, spectrum, asmack, XMPPFramework) for IM before starting to experiment with open HTTP APIs -as an alternative.   The main issues with XMPP that drove us in this -direction were: +as an alternative in around 2012. The main issues with XMPP that +drove us in this direction were: - Not particularly web-friendly - you can't easily speak XMPP from a web browser. (N.B. Nowadays you have options like XMPP-FTW and @@ -178,8 +225,8 @@ direction were: - Single logical server per MUC is a single point of control and availability. (MUCs can be distributed over multiple physical servers, but they still sit behind a single logical JID and domain. - FMUC improves this with a similar approach to Matrix, but at time of - writing there are no open implementations.) + FMUC improves this with a similar approach to Matrix, but as of Oct + 2015 there are no open source implementations.) - History synchronisation is very much a second class citizen feature - Stanzas aren't framed or reliably delivered without extensions. (See [wiki.xmpp.org](http://wiki.xmpp.org/web/Myths#Myth_Four:_XMPP_is_unreliable_without_a_bunch_of_extensions.) @@ -187,7 +234,8 @@ direction were: - Multiple device support is limited. (Apparently Carbons and MAM help with this) - Baseline feature set is so minimal that fragmentation of features - between clients and servers is common + between clients and servers is common, especially as interoperability + profiles for features have fallen behind (as of July 2015) - No strong identity system (i.e. no standard E2E PKI, unless you count X.509 certs, which [are questionable](http://www.thoughtcrime.org/blog/ssl-and-the-future-of-authenticity/)) @@ -195,31 +243,42 @@ direction were: bandwidth-efficient transports. (Since the time of writing a [Push XEP has appeared](http://xmpp.org/extensions/xep-0357.html), and [wiki.xmpp.org](http://wiki.xmpp.org/web/Myths#Myth_Three:_It.27s_too_bandwidth-inefficient_for_mobile.) - claims that XMPP runs fine over a 9600bps + 30s latency link.) + claims that XMPP runs "fine" over a 9600bps + 30s latency link.) The whole subject of XMPP vs Matrix seems to bring out the worst in -people. We think of the standards as being quite different; at its core +people. Rather than fighting over which open interoperable communication +standard works the best, we should just collaborate and bridge everything +together. The more federation and interoperability the better. + +| + +We think of Matrix and XMPP as being quite different; at its core Matrix can be thought of as an eventually consistent global JSON db with an HTTP API and pubsub semantics - whilst XMPP can be thought of as a message passing protocol. You can use them both to build chat systems; you can use them both to build pubsub systems; each comes with different -tradeoffs. Matrix has a 'kitchen sink' baseline of functionality; XMPP -has a deliberately minimal baseline set of functionality. If XMPP does -what you need it to do, then we're genuinely happy for you :) Meanwhile, -rather than competing, an XMPP Bridge like [Skaverat's xmpptrix -beta](https://github.com/SkaveRat/xmpptrix) has potential to let both -environments coexist and make the most of each other's benefits. +tradeoffs. Matrix has a deliberately extensive 'kitchen sink' baseline +of functionality; XMPP has a deliberately minimal baseline set of +functionality. If XMPP does what you need it to do, then we're genuinely +happy for you :) Meanwhile, rather than competing, an XMPP Bridge like +[Skaverat's xmpptrix beta](https://github.com/SkaveRat/xmpptrix) or +[jfred's matrix-xmpp-bridge](https://github.com/jfrederickson/matrix-xmpp-bridge) +or Matrix.org's own [matrix-appservice-purple](https://github.com/matrix-org/matrix-appservice-purple) +has potential to let both environments coexist and make the most of each +other's benefits. ##### What is the difference between Matrix and PSYC? PSYC is a open federated messaging protocol loosely inspired by IRC.  In version 1 it was a standalone protocol, and in version 2 it is being -reutilised as the messaging layer on top of GNUnet.  We honestly don't +reutilised as a messaging layer on top of GNUnet.  We honestly don't know that much about it, beyond trying to use psycd as an XMPP\<-\>IRC bridge in 2010. Matrix differentiates primarily by providing simple HTTP APIs rather than the more exotic compact line protocol in PSYC v1 or the -complicated GNUnet stack in v2.  Meanwhile, Matrix doesn't provide of -the metadata protection guarantees that GNUnet/PSYC aims for. +comprehensive GNUnet stack in v2, and Matrix focuses more on decentralised +conversation history rather than just decentralised chat servers. +On the other hand, Matrix doesn't provide the metadata protection +guarantees that GNUnet/PSYC aims for. | @@ -229,29 +288,56 @@ PSYC's views on Matrix. ##### What is the difference between Matrix and Tox? Tox.im looks to be a very cool clone of Skype - a fully decentralised -peer-to-peer network.  Matrix is deliberately not peer-to-peer; instead -each user has a well-defined homeserver which stores his data and that -he can depend upon.  Matrix provides HTTP APIs; Tox.im provides C APIs. - We haven't actually played with Tox at all yet. +peer-to-peer network.  Matrix is deliberately not a 'pure' peer-to-peer +system; instead each user has a well-defined homeserver which stores +his data and that he can depend upon.  Matrix provides HTTP APIs; +Tox.im provides C APIs. As of October 2015 Tox doesn't seem to have an +answer yet for decentralised conversation history storage. ##### How does Matrix compare with something like Trillian or Pidgin? Trillian and Pidgin and similar aggregating IM clients merge all your IM -activity into a single user experience.  However, your history and +activity into a single app.  However, your history and identity is still fragmented across the networks.  People can't find you easily, and your history is fragmented (other than on the device where the client runs).   And rather than being able to chose the right app for the job when communicating with people, you are pushed towards relying on a specific aggregation app. +Matrix lets you get the best of both worlds by linking to all the +different networks (XMPP, AIM, ICQ, Lync, Skype etc) on the serverside, +using bridges which can be run by anyone. Matrix then provides a simple +standard HTTP API to access any of these networks, and lets you choose +whichever client you prefer (either as a 'native' Matrix client or using +a non-Matrix client from one of the networks which has been bridged in). + ##### What Matrix compliant apps are there? -None yet, other than our examples.  It's early days :) +Quite a few, ranging from the glossy mass-market to the geeky command-line. There's even an emacs macro. Check out [https://matrix.org/blog/try-matrix-now] for the current +list of Matrix enabled projects. -##### Why do you think existing apps will ever join this? +##### What bridges to other networks are available? + +The number of 'bridges' which integrate existing communication networks into +Matrix are growing on a daily basis - both written by the Matrix core team +and contributed by the wider community. The full list can be seen at +https://matrix.org/blog/try-matrix-now, but the core ones as of Oct 2015 include: + + * [matrix-appservice-irc](https://github.com/matrix-org/matrix-appservice-irc) - an increasingly comprehensive Matrix\<-\>IRC bridge + * [matrix-appservice-verto](https://github.com/matrix-org/matrix-appservice-verto) - links from Matrix to FreeSWITCH via the Verto protocol + * [matrix-appservice-slack](https://github.com/matrix-org/matrix-appservice-slack) - a basic bridge to Slack + * [matrix-appservice-purple](https://github.com/matrix-org/matrix-appservice-purple) - lets you access any of the 20+ protocols supported by + [libpurple](https://developer.pidgin.im/wiki/WhatIsLibpurple), including + Skype, Lync, + * [matrix-appservice-bridge](https://github.com/matrix-org/matrix-appservice-bridge) - a general NodeJS framework for writing bridges + +Writing new bridges is incredibly fun and easy - see the [matrix-appservice-bridge HOWTO](https://github.com/matrix-org/matrix-appservice-bridge/blob/master/HOWTO.md) +for an example of how to write a fully functional Slack bridge in less than 100 lines of code! + +##### Why do you think existing apps will ever join this officially? We firmly believe it is what is right for the consumer. As people begin -to use interoperable communications tools service providers will see the +to use interoperable communications tools, service providers will see the benefit and compete on quality of service, security and features rather than relying on locking people into their walled garden. We believe as soon as users see the availability and benefits of interoperable @@ -260,26 +346,27 @@ services they will demand it. ##### Why aren't you doing this through the IETF? or W3C? or 3GPP? We do recognise the advantages of working with existing standards -bodies. We have been focused on writing code and getting it out. As -Matrix matures it may well be appropriate to work with an official -standard body. +bodies. We have been focused on writing code and getting it out, and the standard has been evolving rapidly since initial release in September 2014. +Once the standard has matured sufficiently it may well be appropriate to work with an official +standard body to maintain it going forwards. ### Quick Start ##### How do I get an account and get started? -The quickest way is to just jump to the demo webclient at -[http://matrix.org/beta](http://matrix.org/beta) and sign up.  Please note that you can point the -webclient to access any homeserver - you don't have to use matrix.org, +The quickest way is to pick a client from https://matrix.org/blog/try-matrix-now and sign up. +Please note that you can point clients to access any homeserver - you don't have to use matrix.org, although as of day 1, matrix.org is the only communal homeserver available. ##### What can I actually do with this? -The demo webclient provides a simple chatroom interface to Matrix - +A typical client provides a simple chatroom interface to Matrix - letting the user interact with users and rooms anywhere within the Matrix federation.  Text and image messages are supported, and basic voice-only VoIP calling via WebRTC is supported in one-to-one rooms. +(As of October 2015, experimental multi-way calling is also available +on Vector.im). ##### How do I connect my homeserver to the public Matrix network? @@ -289,11 +376,21 @@ for details ##### How do I Matrix-enable my existing app? -See the [Client-Server API -HOWTO](http://matrix.org/docs/howtos/client-server.html) for an example -of how to use Matrix's client-server API to let your app communicate -with users via Matrix.  We're currently working out the best way to -integrate your application's existing identity system with Matrix. +If your app doesn't have any communication capability already, you'll want +to use one of the Matrix client SDKs to add it in. These come in different +levels of sophistication - ranging from a simple HTTP API wrapper (like matrix-js-sdk, matrix-ios-sdk or matrix-android-sdk) +through to reusable UI components (like matrix-react-sdk and matrix-ios-kit). Pick +the one for your platform, or a 3rd party one if none of the above work for you, +and get plugging it in. You'll probably also want to read the [Client-Server API +HOWTO](http://matrix.org/docs/howtos/client-server.html) too. + +If you already have communication infrastructure set up (XMPP, custom HTTP, or whatever), +then you'll want to run a bridge to expose it to the wider Matrix ecosystem. +See [matrix-appservice-bridge HOWTO](https://github.com/matrix-org/matrix-appservice-bridge/blob/master/HOWTO.md) for a +guide of how to write bridges using the matrix-appservice-bridge framework, or co-opt one +from the list at https://matrix.org/blog/try-matrix-now. +[Application Service API](/docs/spec/#application-service-api) gives the details of the API +that bridges have to implement. ##### How can I write a client on Matrix? @@ -304,14 +401,18 @@ to write a client. ##### How can I help out with this? -Install synapse and tell us how you get on. Critique the spec.  Write -clients.  Just come say hi on [\#matrix:matrix.org](/alpha) or the -[mailing lists](/mailman/listinfo/matrix-users)! +Come say hi on #matrix:matrix.org! Install synapse and tell us how you get on. Critique the spec.  Write +clients. Write bridges! Run bridges! Nose around in [Jira](https://matrix.org/jira) and +send us some pull requests on github to fix some bugs or add some features! You could even +try to write a homeserver (but be warned, Matrix's architecture makes homeservers orders of +magnitude harder than clients or bridges.) + +See [CONTRIBUTING.rst](http://github.com/matrix-org/synapse/tree/master/CONTRIBUTING.rst) for +full details on how to contribute to the project. All are welcome! ##### Where can I get support? -[\#matrix:matrix.org](/alpha), \#matrix on irc.freenode.net or -the [mailing lists](/mailman/listinfo/matrix-users) are your best bets. +\#matrix:matrix.org aka \#matrix on irc.freenode.is your best bet. ##### How do I register custom matrix event types? @@ -322,7 +423,7 @@ use the [mailing list](/mailman/listinfo/matrix-users) for now. ##### How mature is this? We started working on Matrix in July 2014, and have opened it to the -public in September 2014.  It's early days, and under no circumstances +public in September 2014. It's early days, and under no circumstances should you use Matrix or Synapse for anything other than experimentation and learning at this point.  Obviously the spec and apps are maturing rapidly, but as of the time of writing APIs are not frozen and the apps From 91fb2ae723b3b0b17008f970dd8b1654ef64bfd9 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 6 Oct 2015 16:38:29 -0500 Subject: [PATCH 65/68] edit oddvar's FAQ entries and add a bunch more questions --- supporting-docs/_posts/2015-08-19-faq.md | 117 ++++++++++++++++------- 1 file changed, 81 insertions(+), 36 deletions(-) diff --git a/supporting-docs/_posts/2015-08-19-faq.md b/supporting-docs/_posts/2015-08-19-faq.md index df981421..6bc6db92 100644 --- a/supporting-docs/_posts/2015-08-19-faq.md +++ b/supporting-docs/_posts/2015-08-19-faq.md @@ -328,7 +328,7 @@ https://matrix.org/blog/try-matrix-now, but the core ones as of Oct 2015 include * [matrix-appservice-slack](https://github.com/matrix-org/matrix-appservice-slack) - a basic bridge to Slack * [matrix-appservice-purple](https://github.com/matrix-org/matrix-appservice-purple) - lets you access any of the 20+ protocols supported by [libpurple](https://developer.pidgin.im/wiki/WhatIsLibpurple), including - Skype, Lync, + Skype, Lync, XMPP, etc) * [matrix-appservice-bridge](https://github.com/matrix-org/matrix-appservice-bridge) - a general NodeJS framework for writing bridges Writing new bridges is incredibly fun and easy - see the [matrix-appservice-bridge HOWTO](https://github.com/matrix-org/matrix-appservice-bridge/blob/master/HOWTO.md) @@ -435,80 +435,95 @@ nice features before we really declare it ready for production. ### Standard +##### What is a client? + +Users in Matrix use one or more clients to communicate. This could be any combination of a web client, a command line client, a mobile client - or embedded clients built into existing apps. It could even be a piece of hardware (e.g. a drone) that is Matrix enabled. + +##### Can I use Matrix without installing a Matrix client? + +Sure. An ever increasing number of protocols are being bridged into Matrix, so if you use something like IRC on Freenode you may well be indirectly benefiting from Matrix, as others may be connected into the IRC channel via Matrix. + ##### What is a home server? -**Users in Matrix use one or more clients to communicate. This could be a web client, a command line client, a mobile client - or multiple of these being used simultaneously by the same user. The clients are registered to a single homeserver, which stores the communication history and account information, and shares data with the wider Matrix ecosystem by synchronising communication history with other homeservers.** +A user's clients connect to a single homeserver, which stores the communication history and account information for that user, and shares data with the wider Matrix ecosystem by synchronising communication history with other homeservers. ##### What is an identity server? -**Users in Matrix are identified via their matrix user ID (MXID). However, existing 3rd party ID namespaces can also be used in order to identify Matrix users. A Matrix "Identity" describes both the user ID and any other existing IDs from third party namespaces linked to their account.** +Users in Matrix are identified internally via their matrix user ID (MXID). However, existing 3rd party ID (3PID) namespaces such as email addresses or phone numbers should be used publically to identify Matrix users, at least for invitation purposes. A Matrix "Identity" describes both the user ID and any other existing IDs from third party namespaces linked to their account. | -**Matrix users can link third-party IDs (3PIDs) such as email addresses, social network accounts and phone numbers to their user ID. Linking 3PIDs creates a mapping from a 3PID to a user ID. This mapping can then be used by Matrix users in order to discover the MXIDs of their contacts.** +Matrix users can link third-party IDs (3PIDs) to their user ID. Linking 3PIDs creates a mapping from a 3PID to a user ID. This mapping can then be used by Matrix users in order to discover the MXIDs of their contacts. | -**In order to ensure that the mapping from 3PID to user ID is genuine, a globally federated cluster of trusted "Identity Servers" (IS) are used to verify the 3PID and persist and replicate the mappings. -Usage of an IS is not required in order for a client application to be part of the Matrix ecosystem. However, without one clients will not be able to look up user IDs using 3PIDs.** +In order to ensure that the mapping from 3PID to user ID is genuine, a globally federated cluster of trusted "Identity Servers" (IS) are used to verify the 3PID and persist and replicate the mappings. +Usage of an IS is not required in order for a client application to be part of the Matrix ecosystem. However, without one clients will not be able to look up user IDs using 3PIDs. + +| + +The precise architecture of identity servers is currently in flux and subject to change as we work to fully decentralise them. ##### Where do my conversations get stored? -**Each homeserver stores the communication history and account information for all of its clients, and shares data with the wider Matrix ecosystem by synchronising communication history with other homeservers and their clients. Clients typically communicate with each other by emitting events in the context of a virtual room. Room data is replicated across all of the homeservers *whose users are participating in a given room*.** +Each homeserver stores the communication history and account information for all of its clients, and shares data with the wider Matrix ecosystem by synchronising communication history with other homeservers and their clients. Clients typically communicate with each other by emitting events in the context of a virtual room. Room data is replicated across all of the homeservers *whose users are participating in a given room*. ##### What is a 3PID? -**Third-party IDs (3PIDs) are IDs from other systems or contexts, such as email addresses, social network accounts and phone numbers.** +Third-party IDs (3PIDs) are IDs from other systems or contexts, such as email addresses, social network accounts and phone numbers. ##### How do you do VoIP calls on Matrix? -**Voice (and video) over Matrix is built on the WebRTC 1.0 standard. Call events are sent to a room, like any other event. This means that clients must only send call events to rooms with exactly two participants as currently the WebRTC standard is based around two-party communication. Group calls are on the to-do list, though!** +Voice (and video) over Matrix uses the WebRTC 1.0 standard to transfer call media (i.e. the actual voice and video traffic). Matrix is used to signal the establishment and termination of the call by sending call events, like any other event. Currently calls are only supported in rooms with exactly two participants - however, one of those participants may be a conferencing bridge. We're looking at better ways to do group calling. ##### Can I log into other homeservers with my username and password? -**Currently, no. We are looking at options for enabling multi-server access for users, and might add this feature at a later stage.** +Currently, no. We are looking at options for decentralising or migrating user accounts between multiple servers, and might add this feature at a later stage. ##### Why Apache Licence? -**The Apache Licence is a permissive licence. We want the Matrix protocol itself to be free and open, but people are free to create both free and commercial apps and services that uses the protocol. In our opinion, any Matrix-service only enhances the Matrix ecosystem.** +The Apache Licence is a permissive licence. We want the Matrix protocol itself to be free and open, but people are free to create both free and commercial apps and services that uses the protocol. In our opinion, any Matrix-service only enhances the Matrix ecosystem. ##### Can I write a Matrix homeserver? -**Yes. Matrix is just a spec, so implementations of the spec are very welcome! It should be noted that at the moment, changes are still being made to the spec, so if you want to write a Matrix homeserver, it is strongly recommended that you chat to the Matrix.org devs in [\#matrix:matrix.org](https://matrix.org/beta/#/room/%23matrix:matrix.org) first! You can also read about the [Federation API here]( https://github.com/matrix-org/matrix-doc/blob/master/specification/30_server_server_api.rst).** +Yes. Matrix is just a spec, so implementations of the spec are very welcome! It should be noted that as of October 2015 the server, changes are still being made to the spec, so if you want to write a Matrix homeserver, it is strongly recommended that you chat to the Matrix.org devs in #matrix:matrix.org first! You can also read about the [Federation API here]( https://github.com/matrix-org/matrix-doc/blob/master/specification/30_server_server_api.rst). ##### How secure is this? -**Server-server traffic is mandatorily TLS from the outset. Server-client traffic mandates transport layer encryption other than for tinkering. Clients that support PKI publish their public keys, and may encrypt -and sign their messages for E2E security. "Well behaved" clients should participate in key escrow servers to allow private key submission for law enforcement. End-to-end encryption for group chat is supported through a per-room encryption key which is shared 1:1 between participating members.** +Server-server traffic is mandatorily TLS from the outset. Server-client traffic mandates transport layer encryption other than for tinkering. Servers maintain a public/private key pair, and sign the integrity of all messages in the context of the historical conversation, preventing tampering. Server keys are distributed using a PERSPECTIVES-style system. + +End-to-end encryption is coming shortly to clients for both 1:1 and group chats to protect user data stored on servers, using the [Olm](https://matrix.org/git/olm) cryptographic ratchet implementation. As of October 2015 this is blocked on implementing the necessary key distribution and fingerprint management. -##### Why aren't you using an ORM layer like SqlAlchemy?   +Privacy of metadata is not currently protected from server administrators - a malicious homeserver administrator can see who is talking to who and when, but not what is being said (once E2E encryption is enabled). See [this presentation from Jardin Entropique](http://matrix.org/~matthew/2015-06-26%20Matrix%20Jardin%20Entropique.pdf) for a more comprehensive discussion of privacy in Matrix. +### Implementations -### APIs +##### What is Synapse? + +Synapse is a reference "homeserver" implementation of Matrix from the core development team at matrix.org, written in Python 2/Twisted. It is intended to showcase the concept of Matrix and let folks see the spec in the context of a codebase and let you run your own homeserver and generally help bootstrap the ecosystem. ##### How do I join the global Matrix federation? -**You can download and run one of the available Matrix servers - please see [this guide](http://matrix.org/docs/guides/getting_involved.html#run) for details!** +You can download and run one of the available Matrix servers - please see [this guide](http://matrix.org/docs/guides/getting_involved.html#run) for details! ##### What ports do I have to open up to join the global Matrix federation? -**That is up to you! Look at ["Setting up Federation"](https://github.com/matrix-org/synapse#setting-up-federation) in the Synapse readme file for details.** - -### Reference Implementations +We recommend servers use port 8448 for server\<-\>server HTTPS traffic. Look at ["Setting up Federation"](https://github.com/matrix-org/synapse#setting-up-federation) in the Synapse readme file for details. -##### What is Matrix built on - and why? +Client\<-\>Server traffic can talk directly to Synapse via port 8448, but as by default Synapse creates a self-signed TLS certificate this can cause problems for clients which can't easily trust self-signed certificates (e.g. most web browsers). Instead, you can proxy access to Synapse's HTTP listener on port 8008 via an existing HTTPS proxy with a valid certificate (e.g. an nginx listening on port 443), or you can point Synapse at a valid X.509 signed TLS certificate. In future, Synapse will probably use letsencrypt to autogenerate valid certificates rather than self-signed ones during installation, simplifying this process enormously. +You can also put Synapse entirely behind an existing TLS load balancer and not expose port 8448 at all. In this situation, Synapse will need to be configured to share the same *public* TLS certificate as the load balancer (as Synapse uses the public certificate for identity in other areas too, and it has to match the certificate that other servers see when they connect). ##### How do I run my own homeserver? -**Follow the instructions for the homeserver you want to run. If you want to run Synapse, the homeserver created by Matrix.org, follow [these instructions](https://github.com/matrix-org/synapse#synapse-installation).** +Follow the instructions for the homeserver you want to run. If you want to run Synapse, the reference homeserver from Matrix.org, follow [these instructions](https://github.com/matrix-org/synapse#synapse-installation). ##### Can I run my own identity server? Yes - the reference implementation is [sydent](https://github.com/matrix-org/sydent) and you can run your own -ID server cluster that tracks 3rd party to Matrix ID mappings. If you -want your server to participate in the global replicated Matrix ID +ID server cluster that tracks 3rd party to Matrix ID mappings. This won't be very useful right now, though, and we don't recommend it. +If you want your server to participate in the global replicated Matrix ID service then please get in touch with us. Meanwhile, we are looking at ways of decentralising the 'official' Matrix identity service so that identity servers are 100% decentralised and can openly federate with @@ -516,31 +531,61 @@ each other. **N.B. that you can use Matrix without ever using the identity service - it exists only to map 3rd party IDs (e.g. email addresses) to matrix IDs to aid user discovery**. -##### What is Synapse? +##### What are Synapse's platform requirements? -**Synapse is a reference "homeserver" implementation of Matrix from the core development team at matrix.org, written in Python/Twisted for clarity and simplicity. It is intended to showcase the concept of Matrix and let folks see the spec in the context of a codebase and let you run your own homeserver and generally help bootstrap the ecosystem.** +Synapse will use as much RAM as you give it in order to cache conversations in RAM to avoid hitting the database. For small deployments (<50 active users) around 512MB of RAM is probably okay. You can configure the amount of RAM used by synapse with the event_cache_size config parameter - the more events in the cache, the more RAM required. Synapse itself requires relatively little diskspace other than for logging (which as of October 2015 is quite verbose for debugging purposes), but as it caches the content of all the file attachments (images, videos etc) viewed by its users, you may need to size storage appropriately. Synapse is currently effectively single threaded, and will never use more than 1 core. -##### Why is Synapse in Python/Twisted? +| -##### What are Synapse's platform requirements? +For better performance, one should back Synapse with a Postgres database rather than the default SQLite - see https://github.com/matrix-org/synapse/tree/master/README.rst#using-postgresql for details. +##### Why is Synapse in Python/Twisted? + +This is because both provide a mature and well known event-driven async IO framework for writing serverside code. Whilst this has been okay for our initial experimentation and proof of concept, it's likely that future homeserver work will be written in a more strongly typed language (e.g. Go). +##### Why aren't you using an ORM layer like SqlAlchemy in Synapse? -##### What are the Synapse webclient's requirements? +Synapse is *very* database dependent (as of Oct 2015; this is improving in the near future however), and we like having the flexibility to sculpt our own queries. -##### Where is the mobile app? +##### Where can I find a mobile app? -**The mobile apps can be downloaded from the [Google Play store](https://play.google.com/store/apps/details?id=org.matrix.androidsdk.alpha) - and [Apple store](https://itunes.apple.com/gb/app/matrix-console/id970074271).** +The "Matrix Console" reference apps (ugly, geeky and powerful - intended for early adopter powerusers) can be downloaded from the [Google Play store](https://play.google.com/store/apps/details?id=org.matrix.androidsdk.alpha) + and [Apple store](https://itunes.apple.com/gb/app/matrix-console/id970074271). | -**For the Android app, you can also install the latest development version -built by [Jenkins](http://www.matrix.org/jenkins/job/AndroidConsoleDevelop/lastBuild/artifact/console/build/outputs/apk/console-alpha-debug.apk).** +For the Android app, you can also install the latest development version +built by [Jenkins](http://www.matrix.org/jenkins/job/AndroidConsoleDevelop/lastBuild/artifact/console/build/outputs/apk/console-alpha-debug.apk). + +##### Where can I find a web app? -##### What decides the room member order on the webclient? +As of Oct 2015, the best web app options are to use https://vector.im - a glossy web client written on top of matrix-react-sdk, or the original AngularJS based client at https://matrix.org/beta, which has serious performance problems and is not currently being maintained. In future a "Matrix Console" reference web app built on matrix-react-sdk will be released by matrix.org to complement the mobile apps above. + +| -**The members are ordered by their *last active time*.** +### QUESTIONS TO BE ANSWERED! + +This FAQ is a constant work in progress - patches and pull requests are *very* welcome to help us improve it. Some of the frequent questions where we need to write an answer include: + + * How do I rename servers? + * How do I change the TLS key of my server? + * How do I maintain my synapse's DB (e.g. prune old conversations)? + * How do I maintain my synapse's content repository (e.g. prune old content)? + * What are redactions? + * Why is the spec so big, especially relative to the XMPP baseline spec? + * How do I contribute to the spec? + * What is the privacy policy on Matrix.org? + * How precisely does E2E work? + * How does Matrix actually work architecturally? + * What IOT use cases are there for Matrix? + * Why is are the Matrix reference implementations written in so many different languages? + * How does push work? + * What's on the roadmap? + * How can I use Matrix to talk on Freenode or other IRC networks? + * Where can I learn more about Matrix? (link to PDFs of other presentations etc) + * WHy HTTP? Doesn't HTTP suck? + * Why don't you use websockets? + * \[your question goes here...\] | From dffe9b3e4c2f0c12cae91346ad02b261183b5408 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 6 Oct 2015 21:33:32 -0500 Subject: [PATCH 66/68] more questions --- supporting-docs/_posts/2015-08-19-faq.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/supporting-docs/_posts/2015-08-19-faq.md b/supporting-docs/_posts/2015-08-19-faq.md index 6bc6db92..402021e1 100644 --- a/supporting-docs/_posts/2015-08-19-faq.md +++ b/supporting-docs/_posts/2015-08-19-faq.md @@ -583,8 +583,9 @@ This FAQ is a constant work in progress - patches and pull requests are *very* w * What's on the roadmap? * How can I use Matrix to talk on Freenode or other IRC networks? * Where can I learn more about Matrix? (link to PDFs of other presentations etc) - * WHy HTTP? Doesn't HTTP suck? + * Why HTTP? Doesn't HTTP suck? * Why don't you use websockets? + * Why is synapse so resource intensive immediately after federating for the first time? * \[your question goes here...\] | From 7805ca87dff536d98ec7de16f5dfc94d45680028 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 7 Oct 2015 10:34:29 +0100 Subject: [PATCH 67/68] If build.py throws, actually print stdout. Also run jenkins.sh in verbose mode. --- jenkins.sh | 2 +- scripts/gendoc.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/jenkins.sh b/jenkins.sh index 0936de9d..0b217e58 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -4,6 +4,6 @@ set -ex (cd event-schemas/ && ./check_examples.py) (cd api && ./check_examples.py) -(cd scripts && ./gendoc.py) +(cd scripts && ./gendoc.py -v) (cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") (cd event-schemas/ && ./check.sh) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 1655d6f0..8db604c4 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -261,6 +261,7 @@ def run_through_template(input, set_verbose): cwd="../templating" ) except subprocess.CalledProcessError as e: + print e.output with open(tmpfile, 'r') as f: sys.stderr.write(f.read() + "\n") raise From 2ab9372b167f50046a16f856a5ab62ddc2529cac Mon Sep 17 00:00:00 2001 From: Oddvar Lovaas Date: Wed, 7 Oct 2015 18:05:50 +0100 Subject: [PATCH 68/68] new nav bar. todo: get animation working. --- supporting-docs/_includes/footer.html | 24 ++++++++ supporting-docs/_includes/head.html | 15 +++++ supporting-docs/_includes/nav.html | 55 +++++++++++-------- supporting-docs/_layouts/default.html | 7 ++- .../_posts/2015-08-14-getting_involved.md | 5 ++ supporting-docs/_posts/2015-08-19-faq.md | 8 +-- supporting-docs/css/site_overrides.css | 5 ++ 7 files changed, 90 insertions(+), 29 deletions(-) diff --git a/supporting-docs/_includes/footer.html b/supporting-docs/_includes/footer.html index fc43ec97..3e020c64 100644 --- a/supporting-docs/_includes/footer.html +++ b/supporting-docs/_includes/footer.html @@ -1 +1,25 @@ + + + + + + + + diff --git a/supporting-docs/_includes/head.html b/supporting-docs/_includes/head.html index ddb4d988..213e1796 100644 --- a/supporting-docs/_includes/head.html +++ b/supporting-docs/_includes/head.html @@ -12,5 +12,20 @@ + + + + + + + diff --git a/supporting-docs/_includes/nav.html b/supporting-docs/_includes/nav.html index b9f13762..b7e2d629 100644 --- a/supporting-docs/_includes/nav.html +++ b/supporting-docs/_includes/nav.html @@ -1,22 +1,33 @@ - +
+
+ + + +
+ + + +
+
+
diff --git a/supporting-docs/_layouts/default.html b/supporting-docs/_layouts/default.html index 00ea25ce..da18483e 100644 --- a/supporting-docs/_layouts/default.html +++ b/supporting-docs/_layouts/default.html @@ -4,9 +4,12 @@ {% include head.html %} - -
+ + +
{% include nav.html %} +
+
{{ content }} diff --git a/supporting-docs/_posts/2015-08-14-getting_involved.md b/supporting-docs/_posts/2015-08-14-getting_involved.md index 9a37a0ea..162ee7b8 100644 --- a/supporting-docs/_posts/2015-08-14-getting_involved.md +++ b/supporting-docs/_posts/2015-08-14-getting_involved.md @@ -33,6 +33,8 @@ You can use multiple clients with the same user, so you might want to also look ## Run a server and/or client yourself +You can clone our open source projects and run clients and servers yourself. Here is how: + ### Running your own client: You can run your own Matrix client; there are [numerous clients available](https://matrix.org/blog/try-matrix-now/). You can take Matrix.org's [reference client](https://github.com/matrix-org/matrix-angular-sdk) and use it as-is - or modify it any way you want! Since it's written in JavaScript, running a client is [really easy](https://github.com/matrix-org/matrix-angular-sdk#running)! @@ -98,6 +100,7 @@ curl -XGET "http://localhost:8008/_matrix/client/api/v1/events?access_token=YOUR "end": "s39_18_0", "start": "s39_18_0" } + 7. Even if there are no new events (as in the example above), there will be some pagination stream response keys. The client should make subsequent requests using the value of the "end" key (in this case s39_18_0) as the from query parameter e.g. ``` http://localhost:8008/_matrix/client/api/v1/events?access _token=YOUR_ACCESS_TOKEN&from=s39_18_0 @@ -105,6 +108,8 @@ http://localhost:8008/_matrix/client/api/v1/events?access _token=YOUR_ACCESS_TOK 8. This ensures that you only get new events. Now you have initial rooms and presence, and a stream of events - a good client should be able to process all these events and present them to the user. And potentially you might want to add functionality to generate events as well (such as messages from the user, for example) - again please consult the [client-server API spec](http://matrix.org/docs/howtos/client-server.html)! +| + ### Write your own server: We are still working on the server-server spec, so the best thing to do if you are interested in writing a server, is to come talk to us in [#matrix:matrix.org](https://matrix.org/beta/#/room/%23matrix:matrix.org). diff --git a/supporting-docs/_posts/2015-08-19-faq.md b/supporting-docs/_posts/2015-08-19-faq.md index 402021e1..ce58c158 100644 --- a/supporting-docs/_posts/2015-08-19-faq.md +++ b/supporting-docs/_posts/2015-08-19-faq.md @@ -19,9 +19,7 @@ Categories [Standard](#standard) -[APIs](#apis) - -[Reference Implementations](#reference-implementations) +[Implementations](#implementations) | @@ -486,7 +484,7 @@ The Apache Licence is a permissive licence. We want the Matrix protocol itself t ##### Can I write a Matrix homeserver? -Yes. Matrix is just a spec, so implementations of the spec are very welcome! It should be noted that as of October 2015 the server, changes are still being made to the spec, so if you want to write a Matrix homeserver, it is strongly recommended that you chat to the Matrix.org devs in #matrix:matrix.org first! You can also read about the [Federation API here]( https://github.com/matrix-org/matrix-doc/blob/master/specification/30_server_server_api.rst). +Yes. Matrix is just a spec, so implementations of the spec are very welcome! It should be noted that as of October 2015, changes are still being made to the spec, so if you want to write a Matrix homeserver, it is strongly recommended that you chat to the Matrix.org devs in #matrix:matrix.org first! You can also read about the [Federation API here]( https://github.com/matrix-org/matrix-doc/blob/master/specification/30_server_server_api.rst). ##### How secure is this? @@ -559,7 +557,7 @@ built by [Jenkins](http://www.matrix.org/jenkins/job/AndroidConsoleDevelop/lastB ##### Where can I find a web app? -As of Oct 2015, the best web app options are to use https://vector.im - a glossy web client written on top of matrix-react-sdk, or the original AngularJS based client at https://matrix.org/beta, which has serious performance problems and is not currently being maintained. In future a "Matrix Console" reference web app built on matrix-react-sdk will be released by matrix.org to complement the mobile apps above. +As of Oct 2015, the best web app options are to use [Vector.im](https://vector.im) - a glossy web client written on top of [matrix-react-sdk](https://github.com/matrix-org/matrix-react-sdk), or the original [AngularJS based client](https://matrix.org/beta), which has serious performance problems and is not currently being maintained. In future a "Matrix Console" reference web app built on matrix-react-sdk will be released by matrix.org to complement the mobile apps above. | diff --git a/supporting-docs/css/site_overrides.css b/supporting-docs/css/site_overrides.css index 0f9a2c80..f7640e1c 100644 --- a/supporting-docs/css/site_overrides.css +++ b/supporting-docs/css/site_overrides.css @@ -53,5 +53,10 @@ h1,h2,h3,h4,h5 { #document { margin-top: 10px; /* We want a little whitespace before the page content starts */ + width: 1080px; +} + +#wrapper { + max-width: 1080px; }