diff --git a/api/client-server/v2_alpha/definitions/definitions b/api/client-server/v2_alpha/definitions/definitions new file mode 120000 index 000000000..945c9b46d --- /dev/null +++ b/api/client-server/v2_alpha/definitions/definitions @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/api/client-server/v2_alpha/definitions/event_batch.json b/api/client-server/v2_alpha/definitions/event_batch.json new file mode 100644 index 000000000..75762d758 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/event_batch.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "properties": { + "events": { + "type": "array", + "description": "List of events", + "items": { + "type": "object" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/event_filter.json b/api/client-server/v2_alpha/definitions/event_filter.json new file mode 100644 index 000000000..1cdcb1f41 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/event_filter.json @@ -0,0 +1,42 @@ +{ + "type": "object", + "properties": { + "limit": { + "type": "integer", + "description": + "The maximum number of events to return." + }, + "types": { + "type": "array", + "description": + "A list of event types to include. If this list is absent then all event types are included. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + }, + "not_types": { + "type": "array", + "description": + "A list of event types to exclude. If this list is absent then no event types are excluded. A matching type will be excluded even if it is listed in the 'types' filter. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + }, + "senders": { + "type": "array", + "description": + "A list of senders IDs to include. If this list is absent then all senders are included. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + }, + "not_senders": { + "type": "array", + "description": + "A list of sender IDs to exclude. If this list is absent then no senders are excluded. A matching sender will be excluded even if it is listed in the 'senders' filter. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/room_event_batch.json b/api/client-server/v2_alpha/definitions/room_event_batch.json new file mode 100644 index 000000000..fcf148f36 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/room_event_batch.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "properties": { + "events": { + "type": "array", + "description": "List of event ids", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/room_event_filter.json b/api/client-server/v2_alpha/definitions/room_event_filter.json new file mode 100644 index 000000000..86375781c --- /dev/null +++ b/api/client-server/v2_alpha/definitions/room_event_filter.json @@ -0,0 +1,22 @@ +{ + "type": "object", + "allOf": [{"$ref": "definitions/event_filter.json"}], + "properties": { + "rooms": { + "type": "array", + "description": + "A list of room IDs to include. If this list is absent then all rooms are included. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + }, + "not_rooms": { + "type": "array", + "description": + "A list of room IDs to exclude. If this list is absent then no rooms are excluded. A matching room will be excluded even if it is listed in the 'rooms' filter. A '*' can be used as a wildcard to match any sequence of characters.", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/sync_filter.json b/api/client-server/v2_alpha/definitions/sync_filter.json new file mode 100644 index 000000000..0cd6a7986 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/sync_filter.json @@ -0,0 +1,44 @@ +{ + "type": "object", + "properties": { + "room": { + "type": "object", + "properties": { + "state": { + "description": + "The state events to include for rooms.", + "allOf": [{"$ref": "definitions/room_event_filter.json"}] + }, + "timeline": { + "description": + "The message and state update events to include for rooms.", + "allOf": [{"$ref": "definitions/room_event_filter.json"}] + }, + "ephemeral": { + "description": + "The events that aren't recorded in the room history, e.g. typing and receipts, to include for rooms.", + "allOf": [{"$ref": "definitions/room_event_filter.json"}] + } + } + }, + "presence": { + "description": + "The presence updates to include.", + "allOf": [{"$ref": "definitions/event_filter.json"}] + }, + "event_format": { + "description": + "The format to use for events. 'client' will return the events in a format suitable for clients. 'federation' will return the raw event as receieved over federation. The default is 'client'.", + "type": "string", + "enum": ["client", "federation"] + }, + "event_fields": { + "type": "array", + "description": + "List of event fields to include. If this list is absent then all fields are included. The entries may include '.' charaters to indicate sub-fields. So ['content.body'] will include the 'body' field of the 'content' object. A literal '.' character in a field name may be escaped using a '\\'. A server may include more fields than were requested.", + "items": { + "type": "string" + } + } + } +} diff --git a/api/client-server/v2_alpha/definitions/timeline_batch.json b/api/client-server/v2_alpha/definitions/timeline_batch.json new file mode 100644 index 000000000..ddf8d3416 --- /dev/null +++ b/api/client-server/v2_alpha/definitions/timeline_batch.json @@ -0,0 +1,14 @@ +{ + "type": "object", + "allOf": [{"$ref":"definitions/room_event_batch.json"}], + "properties": { + "limited": { + "type": "boolean", + "description": "Whether there are more events on the server" + }, + "prev_batch": { + "type": "string", + "description": "If the batch was limited then this is a token that can be supplied to the server to retrieve more events" + } + } +} diff --git a/api/client-server/v2_alpha/filter.yaml b/api/client-server/v2_alpha/filter.yaml new file mode 100644 index 000000000..37a0a3aaa --- /dev/null +++ b/api/client-server/v2_alpha/filter.yaml @@ -0,0 +1,139 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v2 filter API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/user/{userId}/filter": + post: + summary: Upload a new filter. + description: |- + Uploads a new filter definition to the homeserver. + Returns a filter ID that may be used in /sync requests to + retrict which events are returned to the client. + security: + - accessToken: [] + parameters: + - in: path + type: string + name: userId + required: true + description: + The id of the user uploading the filter. The access token must be + authorized to make requests for this user id. + x-example: "@alice:example.com" + - in: body + name: filter + required: true + description: The filter to upload. + schema: + type: object + allOf: + - $ref: "definitions/sync_filter.json" + example: |- + { + "room": { + "state": { + "types": ["m.room.*"], + "not_rooms": ["!726s6s6q:example.com"] + }, + "timeline": { + "limit": 10, + "types": ["m.room.message"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + }, + "emphemeral": { + "types": ["m.receipt", "m.typing"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + } + }, + "presence": { + "types": ["m.presence"], + "not_senders": ["@alice:example.com"] + }, + "event_format": "client", + "event_fields": ["type", "content", "sender"] + } + responses: + 200: + description: The filter was created. + examples: + application/json: |- + { + "filter_id": "66696p746572" + } + schema: + type: object + properties: + filter_id: + type: string + description: |- + The ID of the filter that was created. + "/user/{userId}/filter/{filterId}": + get: + summary: Download a filter + parameters: + - in: path + name: userId + type: string + description: |- + The user ID to download a filter for. + x-example: "@alice:example.com" + required: true + - in: path + name: filterId + type: string + description: |- + The filter ID to download. + x-example: "66696p746572" + required: true + responses: + 200: + description: |- + "The filter defintion" + examples: + application/json: |- + { + "room": { + "state": { + "types": ["m.room.*"], + "not_rooms": ["!726s6s6q:example.com"] + }, + "timeline": { + "limit": 10, + "types": ["m.room.message"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + }, + "emphemeral": { + "types": ["m.receipt", "m.typing"], + "not_rooms": ["!726s6s6q:example.com"], + "not_senders": ["@spam:example.com"] + } + }, + "presence": { + "types": ["m.presence"], + "not_senders": ["@alice:example.com"] + }, + "event_format": "client", + "event_fields": ["type", "content", "sender"] + } + schema: + type: object + allOf: + - $ref: "definitions/sync_filter.json" diff --git a/api/client-server/v2_alpha/sync.yaml b/api/client-server/v2_alpha/sync.yaml new file mode 100644 index 000000000..7d60d6392 --- /dev/null +++ b/api/client-server/v2_alpha/sync.yaml @@ -0,0 +1,257 @@ +swagger: '2.0' +info: + title: "Matrix Client-Server v2 sync API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https +basePath: /_matrix/client/v2_alpha +consumes: + - application/json +produces: + - application/json +securityDefinitions: + accessToken: + type: apiKey + description: The user_id or application service access_token + name: access_token + in: query +paths: + "/sync": + get: + summary: Synchronise the client's state and receive new messages. + description: |- + Synchronise the client's state with the latest state on the server. + Client's use this API when they first log in to get an initial snapshot + of the state on the server, and then continue to call this API to get + incremental deltas to the state, and to receive new messages. + security: + - accessToken: [] + parameters: + - in: query + name: filter + type: string + description: |- + The ID of a filter created using the filter API. + x-example: "66696p746572" + - in: query + name: since + type: string + description: |- + A point in time to continue a sync from. + x-example: "s72594_4483_1934" + - in: query + name: set_presence + type: string + enum: ["offline"] + description: |- + Controls whether the client is automatically marked as online by + polling this API. If this parameter is omitted then the client is + automatically marked as online when it uses this API. Otherwise if + the parameter is set to "offline" then the client is not marked as + being online when it uses this API. + x-example: "offline" + - in: query + name: timeout + type: integer + description: |- + The maximum time to poll in milliseconds before returning this + request. + x-example: 30000 + responses: + 200: + description: + The initial snapshot or delta for the client to use to update their + state. + schema: + type: object + properties: + next_batch: + type: string + description: |- + The batch token to supply in the ``since`` param of the next + ``/sync`` request. + rooms: + type: object + description: |- + Updates to rooms. + properties: + joined: + type: object + additionalProperties: + type: object + properties: + event_map: + type: object + description: |- + A map from event ID to events for this room. The + events are referenced from the ``timeline`` and + ``state`` keys for this room. + additionalProperties: + description: An event object. + type: object + state: + description: |- + The state updates for the room. + allOf: + - $ref: "definitions/room_event_batch.json" + timeline: + description: |- + The timeline of messages and state changes in the + room. + allOf: + - $ref: "definitions/timeline_batch.json" + ephemeral: + description: |- + The ephemeral events in the room that aren't + recorded in the timeline or state of the room. + e.g. typing. + allOf: + - $ref: "definitions/event_batch.json" + invited: + type: object + description: |- + The rooms that the user has been invited to. + additionalProperties: + type: object + properties: + invite_state: + description: |- + The state of a room that the user has been invited + to. These state events may only have the `sender``, + ``type``, ``state_key`` and ``content`` keys + present. These events do not replace any state that + the client already has for the room, for example if + the client has archived the room. Instead the + client should keep two separate copies of the + state: the one from the ``invite_state`` and one + from the archived ``state``. If the client joins + the room then the current state will be given as a + delta against the archived ``state`` not the + ``invite_state``. + allOf: + - $ref: "definitions/event_batch.json" + archived: + type: object + description: |- + The rooms that the user has left or been banned from. The + entries in the room_map will lack an ``ephemeral`` key. + additionalProperties: + type: object + properties: + event_map: + type: object + description: |- + A map from event ID to events for this room. The + events are referenced from the ``timeline`` and + ``state`` keys for this room. + additionalProperties: + description: An event object. + type: object + state: + description: |- + The state updates for the room up to the point when + the user left. + allOf: + - $ref: "definitions/room_event_batch.json" + timeline: + description: |- + The timeline of messages and state changes in the + room up to the point when the user left. + allOf: + - $ref: "definitions/timeline_batch.json" + presence: + description: |- + The updates to the presence status of other users. + allOf: + - $ref: "definitions/event_batch.json" + examples: + application/json: |- + { + "next_batch": "s72595_4483_1934", + "presence": { + "events": [ + { + "sender": "@alice:example.com", + "type": "m.presence", + "content": {"presence": "online"} + } + ] + }, + "rooms": { + "joined": { + "!726s6s6q:example.com": { + "event_map": { + "$66697273743031:example.com": { + "sender": "@alice:example.com", + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + "$7365636s6r6432:example.com": { + "sender": "@bob:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "join"}, + "origin_server_ts": 1417731086795 + }, + "$74686972643033:example.com": { + "sender": "@alice:example.com", + "type": "m.room.message", + "unsigned": {"age": "124524", "txn_id": "1234"}, + "content": { + "body": "I am a fish", + "msgtype": "m.text" + }, + "origin_server_ts": 1417731086797 + } + }, + "state": { + "events": [ + "$66697273743031:example.com", + "$7365636s6r6432:example.com" + ] + }, + "timeline": { + "events": [ + "$7365636s6r6432:example.com", + "$74686972643033:example.com" + ], + "limited": true, + "prev_batch": "t34-23535_0_0" + }, + "ephemeral": { + "events": [ + { + "room_id": "!726s6s6q:example.com", + "type": "m.typing", + "content": {"user_ids": ["@alice:example.com"]} + } + ] + } + } + }, + "invited": { + "!696r7674:example.com": { + "invite_state": { + "events": [ + { + "sender": "@alice:example.com", + "type": "m.room.name", + "state_key": "", + "content": {"name": "My Room Name"} + }, + { + "sender": "@alice:example.com", + "type": "m.room.member", + "state_key": "@bob:example.com", + "content": {"membership": "invite"} + } + ] + } + } + }, + "archived": {} + } + } diff --git a/jenkins.sh b/jenkins.sh index e10436449..0b217e58a 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -4,6 +4,6 @@ set -ex (cd event-schemas/ && ./check_examples.py) (cd api && ./check_examples.py) -(cd scripts && ./gendoc.py) -(cd api && npm install && node validator.js -s "client-server/v1") +(cd scripts && ./gendoc.py -v) +(cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha") (cd event-schemas/ && ./check.sh) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 1655d6f0a..8db604c4e 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -261,6 +261,7 @@ def run_through_template(input, set_verbose): cwd="../templating" ) except subprocess.CalledProcessError as e: + print e.output with open(tmpfile, 'r') as f: sys.stderr.write(f.read() + "\n") raise diff --git a/specification/1-client_server_api.rst b/specification/1-client_server_api.rst index c63742d17..59e6b68ee 100644 --- a/specification/1-client_server_api.rst +++ b/specification/1-client_server_api.rst @@ -1,5 +1,5 @@ -Client-Server API v1 -==================== +Client-Server API +================= Overview -------- diff --git a/specification/3-application_service_api.rst b/specification/3-application_service_api.rst index bdde77894..a6e82137e 100644 --- a/specification/3-application_service_api.rst +++ b/specification/3-application_service_api.rst @@ -14,7 +14,8 @@ irrespective of the underlying homeserver implementation. Passive Application Services ---------------------------- -"Passive" application services can only observe events from a given home server. +"Passive" application services can only observe events from a given home server, +and inject events into a room they are participating in. They cannot prevent events from being sent, nor can they modify the content of the event being sent. In order to observe events from a homeserver, the homeserver needs to be configured to pass certain types of traffic to the diff --git a/supporting-docs/_posts/2015-08-19-faq.md b/supporting-docs/_posts/2015-08-19-faq.md index 517b2ccf7..402021e10 100644 --- a/supporting-docs/_posts/2015-08-19-faq.md +++ b/supporting-docs/_posts/2015-08-19-faq.md @@ -31,65 +31,98 @@ FAQ Content * TOC -{:toc} +{:toc .toc} ### General ##### What is Matrix? -Matrix is an ambitious new open standard for open, distributed, -real-time communication over IP. It defines interoperable Instant -Messaging and VoIP, providing pragmatic HTTP APIs and open source -reference implementations for creating and running your own real-time -communication infrastructure. +Matrix is an open standard for interoperable, decentralised, +real-time communication over IP. It can be used to power Instant +Messaging, VoIP/WebRTC signalling, Internet of Things communication - or anywhere +you need a standard HTTP API for publishing and subscribing to +data whilst tracking the conversation history. + +| + +Matrix defines the standard, and provides open source reference implementations +of Matrix-compatible Servers, Clients, Client SDKs and Application Services +to help you create new communication solutions or extend the capabilities +and reach of existing ones. ##### What is Matrix's Mission? -Matrix.org's initial inspiration and goal has been to fix the problem of -fragmented IP communications. But Matrix's real potential and ultimate -mission is to be a generic messaging and data synchronisation system for -the web - allowing people, services and devices to easily communicate -with each other with full history. +Matrix's initial goal is to fix the problem of fragmented IP communications: +letting users message and call each other without having to care what app +the other user is on - making it as easy as sending an email. + +| + +The longer term goal is for Matrix to act as a generic HTTP messaging and data +synchronisation system for the whole web - allowing people, services and devices +to easily communicate with each other, empowering users to own and control their +data and select the services and vendors they want to use. ##### What does Matrix provide? -Today Matrix provides a new [open standard](/docs/spec), -[APIs](/docs/api) to integrate a service to the Matrix ecosystem and -reference [open source -implementations](http://github.com/matrix-org/synapse) of the standard. +Matrix provides: + +- [Open Standard](/docs/spec) HTTP APIs for transferring JSON messages (e.g. instant messages, WebRTC signalling), including: + - [Client\<-\>Server API](/docs/spec#client-server-api-v1) - defines how Matrix compatible clients communicate with Matrix homeservers. + - [Server\<-\>Server API](/docs/spec#federation-api) - defines how Matrix homeservers exchange messages and synchronise history with each other. + - [Application Service API](/docs/spec/#application-service-api) - defines how to extend the functionality of Matrix with 'integrations' and bridge to other networks. + - [Modules](/docs/spec/#modules) - specifies features that must be implemented by particular classes of clients. +- Open source reference implementations of: + - Clients (Web (React), iOS, Android) + - Client SDKs (Javascript, Web (React), iOS, Android) + - Homeservers (Synapse) + - Application Services (bridges to IRC, Slack, Skype, Lync and more...) +- The actual ecosystem and community of everyone running Matrix servers and services +- Loads of 3rd party contributions of clients, SDKs, servers and services. + +You can find the full list of Matrix enabled projects at https://matrix.org/blog/try-matrix-now. ##### What does this mean for users? The aim is to provide an analogous ecosystem to email - one where you can communicate with pretty much anyone, without caring what app or server they are using, using whichever app & server you chose to use, -and a nice neutral identity system like an e-mail address or phone +and use a neutral identity system like an e-mail address or phone number to discover people to talk to. ##### What kind of company is Matrix.org? Matrix is an open initiative which acts as a neutral custodian of the Matrix standard. It's not actually incorporated anywhere at the moment -but we are looking at the best legal structure for the future. We are -committed to keeping the Matrix project open. +but we are looking at the best legal structure for the future (and as +of October 2015 we have hopefully found one). Whatever the legal +structure, we are committed to keeping the Matrix project open. ##### Who is funding Matrix.org? -We have been given permission by our employers, Amdocs, to work on -Matrix as an independent non-profit initiative. +Most of the current core contributors to Matrix work at +[Amdocs](http://amdocs.com), who have kindly given us permission to work +on Matrix as an independent non-profit initiative. Other contributors +are funded by their own employers or donate their own time to the project. ##### Who is building Matrix? -We're a team of ~10 people with decades of experience building custom +The core team is ~10 people with extensive experience in building custom VoIP and Messaging apps for mobile network operators. Most of us have -day jobs at Amdocs or OpenMarket, but we are supported by a mix of -freelancers and volunteers. +day jobs at [Amdocs](http://amdocs.com) or [OpenMarket](http://openmarket.com), +but there are an increasing number of contributors from other companies and +folks all over the internet. ##### Why are you called Matrix? We are called Matrix because we provide a structure in which all communication can be matrixed together. +| + +No, it's nothing to do with the film (although you could go and build virtual +worlds on top of Matrix if you wanted :) + ##### Why have you released this as open source? We believe that any open standard defining interoperable communication @@ -102,16 +135,18 @@ and build on top of it. ##### What do you mean by open? Matrix is an open standard, meaning that we have freely published the -details for how to interface with Matrix compliant servers and clients, -and encourage anyone and everyone to interface with them.  We also +details for how to communicate interoperably using the Matrix set of +HTTP APIs. We encourage anyone and everyone to use the APIs and build +their own projects which implement them and so benefit from +interoperability with the rest of the Matrix ecosystem. We also ensure the standard is not encumbered by any known patent licensing requirements. -| +| Matrix is also open source, meaning that we have released the source -code of the reference servers and clients to the public domain under the -[Apache Licence v2](http://www.apache.org/licenses/LICENSE-2.0.html), to +code of the reference servers, clients and services to the public domain +under the [Apache Licence v2](http://www.apache.org/licenses/LICENSE-2.0.html), to encourage anyone and everyone to run their own servers and clients, and enhance them and contribute their enhancements as they see fit. @@ -120,7 +155,7 @@ enhance them and contribute their enhancements as they see fit. Federation allows separate deployments of a communication service to communicate with each other - for instance a mail server run by Google federates with a mail server run by Microsoft when you send email from -@gmail.com to @outlook.com. +@gmail.com to @hotmail.com. | @@ -145,16 +180,18 @@ VoIP and IM. ##### Why has no-one done this before? There have been several attempts before including SIP, XMPP and RCS. - All of these have had some level of success, but + All of these have had some level of success, but many different technological/usability/economic factors have ended up limiting their -success in providing true open federation. +success. Unfortunately, we've not ended up in a world where everyone +has a SIP URI or Jabber ID on their business card, or a phone that +actually uses RCS. ##### What is the difference between Matrix and IRC? We love IRC.  In fact, as of today the core Matrix team still uses it as our primary communication tool. Between us we've written IRCds, IRC bots and admined dreamforge, UnrealIRCd, epona, ircservices and several -others.  That said, it has some limitations that Matrix seeks to improve +others. That said, it has some limitations that Matrix seeks to improve on: - Text only @@ -163,18 +200,24 @@ on: - No presence support - Fragmented identity model - No open federation -- No standard APIs, just an archaic TCP line protocol +- No standard APIs, just a rather limited TCP line protocol - Non-standardised federation protocol - No built-in end-to-end encryption - Disruptive net-splits - Non-extensible +[IRCv3](http://ircv3.net) exists and is addressing some of issues; +this is great news and we wish them well. It's almost a contradiction +in terms to get competitive between openly interoperable communication +projects - we look forward to increasing the richness of Matrix\<-\>IRC +bridges as the project progresses. + ##### What is the difference between Matrix and XMPP? The Matrix team used XMPP (Openfire, ejabberd, spectrum, asmack, XMPPFramework) for IM before starting to experiment with open HTTP APIs -as an alternative.   The main issues with XMPP that drove us in this -direction were: +as an alternative in around 2012. The main issues with XMPP that +drove us in this direction were: - Not particularly web-friendly - you can't easily speak XMPP from a web browser. (N.B. Nowadays you have options like XMPP-FTW and @@ -182,8 +225,8 @@ direction were: - Single logical server per MUC is a single point of control and availability. (MUCs can be distributed over multiple physical servers, but they still sit behind a single logical JID and domain. - FMUC improves this with a similar approach to Matrix, but at time of - writing there are no open implementations.) + FMUC improves this with a similar approach to Matrix, but as of Oct + 2015 there are no open source implementations.) - History synchronisation is very much a second class citizen feature - Stanzas aren't framed or reliably delivered without extensions. (See [wiki.xmpp.org](http://wiki.xmpp.org/web/Myths#Myth_Four:_XMPP_is_unreliable_without_a_bunch_of_extensions.) @@ -191,7 +234,8 @@ direction were: - Multiple device support is limited. (Apparently Carbons and MAM help with this) - Baseline feature set is so minimal that fragmentation of features - between clients and servers is common + between clients and servers is common, especially as interoperability + profiles for features have fallen behind (as of July 2015) - No strong identity system (i.e. no standard E2E PKI, unless you count X.509 certs, which [are questionable](http://www.thoughtcrime.org/blog/ssl-and-the-future-of-authenticity/)) @@ -199,31 +243,42 @@ direction were: bandwidth-efficient transports. (Since the time of writing a [Push XEP has appeared](http://xmpp.org/extensions/xep-0357.html), and [wiki.xmpp.org](http://wiki.xmpp.org/web/Myths#Myth_Three:_It.27s_too_bandwidth-inefficient_for_mobile.) - claims that XMPP runs fine over a 9600bps + 30s latency link.) + claims that XMPP runs "fine" over a 9600bps + 30s latency link.) The whole subject of XMPP vs Matrix seems to bring out the worst in -people. We think of the standards as being quite different; at its core +people. Rather than fighting over which open interoperable communication +standard works the best, we should just collaborate and bridge everything +together. The more federation and interoperability the better. + +| + +We think of Matrix and XMPP as being quite different; at its core Matrix can be thought of as an eventually consistent global JSON db with an HTTP API and pubsub semantics - whilst XMPP can be thought of as a message passing protocol. You can use them both to build chat systems; you can use them both to build pubsub systems; each comes with different -tradeoffs. Matrix has a 'kitchen sink' baseline of functionality; XMPP -has a deliberately minimal baseline set of functionality. If XMPP does -what you need it to do, then we're genuinely happy for you :) Meanwhile, -rather than competing, an XMPP Bridge like [Skaverat's xmpptrix -beta](https://github.com/SkaveRat/xmpptrix) has potential to let both -environments coexist and make the most of each other's benefits. +tradeoffs. Matrix has a deliberately extensive 'kitchen sink' baseline +of functionality; XMPP has a deliberately minimal baseline set of +functionality. If XMPP does what you need it to do, then we're genuinely +happy for you :) Meanwhile, rather than competing, an XMPP Bridge like +[Skaverat's xmpptrix beta](https://github.com/SkaveRat/xmpptrix) or +[jfred's matrix-xmpp-bridge](https://github.com/jfrederickson/matrix-xmpp-bridge) +or Matrix.org's own [matrix-appservice-purple](https://github.com/matrix-org/matrix-appservice-purple) +has potential to let both environments coexist and make the most of each +other's benefits. ##### What is the difference between Matrix and PSYC? PSYC is a open federated messaging protocol loosely inspired by IRC.  In version 1 it was a standalone protocol, and in version 2 it is being -reutilised as the messaging layer on top of GNUnet.  We honestly don't +reutilised as a messaging layer on top of GNUnet.  We honestly don't know that much about it, beyond trying to use psycd as an XMPP\<-\>IRC bridge in 2010. Matrix differentiates primarily by providing simple HTTP APIs rather than the more exotic compact line protocol in PSYC v1 or the -complicated GNUnet stack in v2.  Meanwhile, Matrix doesn't provide of -the metadata protection guarantees that GNUnet/PSYC aims for. +comprehensive GNUnet stack in v2, and Matrix focuses more on decentralised +conversation history rather than just decentralised chat servers. +On the other hand, Matrix doesn't provide the metadata protection +guarantees that GNUnet/PSYC aims for. | @@ -233,29 +288,56 @@ PSYC's views on Matrix. ##### What is the difference between Matrix and Tox? Tox.im looks to be a very cool clone of Skype - a fully decentralised -peer-to-peer network.  Matrix is deliberately not peer-to-peer; instead -each user has a well-defined homeserver which stores his data and that -he can depend upon.  Matrix provides HTTP APIs; Tox.im provides C APIs. - We haven't actually played with Tox at all yet. +peer-to-peer network.  Matrix is deliberately not a 'pure' peer-to-peer +system; instead each user has a well-defined homeserver which stores +his data and that he can depend upon.  Matrix provides HTTP APIs; +Tox.im provides C APIs. As of October 2015 Tox doesn't seem to have an +answer yet for decentralised conversation history storage. ##### How does Matrix compare with something like Trillian or Pidgin? Trillian and Pidgin and similar aggregating IM clients merge all your IM -activity into a single user experience.  However, your history and +activity into a single app.  However, your history and identity is still fragmented across the networks.  People can't find you easily, and your history is fragmented (other than on the device where the client runs).   And rather than being able to chose the right app for the job when communicating with people, you are pushed towards relying on a specific aggregation app. +Matrix lets you get the best of both worlds by linking to all the +different networks (XMPP, AIM, ICQ, Lync, Skype etc) on the serverside, +using bridges which can be run by anyone. Matrix then provides a simple +standard HTTP API to access any of these networks, and lets you choose +whichever client you prefer (either as a 'native' Matrix client or using +a non-Matrix client from one of the networks which has been bridged in). + ##### What Matrix compliant apps are there? -None yet, other than our examples.  It's early days :) +Quite a few, ranging from the glossy mass-market to the geeky command-line. There's even an emacs macro. Check out [https://matrix.org/blog/try-matrix-now] for the current +list of Matrix enabled projects. -##### Why do you think existing apps will ever join this? +##### What bridges to other networks are available? + +The number of 'bridges' which integrate existing communication networks into +Matrix are growing on a daily basis - both written by the Matrix core team +and contributed by the wider community. The full list can be seen at +https://matrix.org/blog/try-matrix-now, but the core ones as of Oct 2015 include: + + * [matrix-appservice-irc](https://github.com/matrix-org/matrix-appservice-irc) - an increasingly comprehensive Matrix\<-\>IRC bridge + * [matrix-appservice-verto](https://github.com/matrix-org/matrix-appservice-verto) - links from Matrix to FreeSWITCH via the Verto protocol + * [matrix-appservice-slack](https://github.com/matrix-org/matrix-appservice-slack) - a basic bridge to Slack + * [matrix-appservice-purple](https://github.com/matrix-org/matrix-appservice-purple) - lets you access any of the 20+ protocols supported by + [libpurple](https://developer.pidgin.im/wiki/WhatIsLibpurple), including + Skype, Lync, XMPP, etc) + * [matrix-appservice-bridge](https://github.com/matrix-org/matrix-appservice-bridge) - a general NodeJS framework for writing bridges + +Writing new bridges is incredibly fun and easy - see the [matrix-appservice-bridge HOWTO](https://github.com/matrix-org/matrix-appservice-bridge/blob/master/HOWTO.md) +for an example of how to write a fully functional Slack bridge in less than 100 lines of code! + +##### Why do you think existing apps will ever join this officially? We firmly believe it is what is right for the consumer. As people begin -to use interoperable communications tools service providers will see the +to use interoperable communications tools, service providers will see the benefit and compete on quality of service, security and features rather than relying on locking people into their walled garden. We believe as soon as users see the availability and benefits of interoperable @@ -264,28 +346,27 @@ services they will demand it. ##### Why aren't you doing this through the IETF? or W3C? or 3GPP? We do recognise the advantages of working with existing standards -bodies. We have been focused on writing code and getting it out. As -Matrix matures it may well be appropriate to work with an official -standard body. - -| +bodies. We have been focused on writing code and getting it out, and the standard has been evolving rapidly since initial release in September 2014. +Once the standard has matured sufficiently it may well be appropriate to work with an official +standard body to maintain it going forwards. ### Quick Start ##### How do I get an account and get started? -The quickest way is to just jump to the demo webclient at -[http://matrix.org/beta](http://matrix.org/beta) and sign up.  Please note that you can point the -webclient to access any homeserver - you don't have to use matrix.org, +The quickest way is to pick a client from https://matrix.org/blog/try-matrix-now and sign up. +Please note that you can point clients to access any homeserver - you don't have to use matrix.org, although as of day 1, matrix.org is the only communal homeserver available. ##### What can I actually do with this? -The demo webclient provides a simple chatroom interface to Matrix - +A typical client provides a simple chatroom interface to Matrix - letting the user interact with users and rooms anywhere within the Matrix federation.  Text and image messages are supported, and basic voice-only VoIP calling via WebRTC is supported in one-to-one rooms. +(As of October 2015, experimental multi-way calling is also available +on Vector.im). ##### How do I connect my homeserver to the public Matrix network? @@ -295,11 +376,21 @@ for details ##### How do I Matrix-enable my existing app? -See the [Client-Server API -HOWTO](http://matrix.org/docs/howtos/client-server.html) for an example -of how to use Matrix's client-server API to let your app communicate -with users via Matrix.  We're currently working out the best way to -integrate your application's existing identity system with Matrix. +If your app doesn't have any communication capability already, you'll want +to use one of the Matrix client SDKs to add it in. These come in different +levels of sophistication - ranging from a simple HTTP API wrapper (like matrix-js-sdk, matrix-ios-sdk or matrix-android-sdk) +through to reusable UI components (like matrix-react-sdk and matrix-ios-kit). Pick +the one for your platform, or a 3rd party one if none of the above work for you, +and get plugging it in. You'll probably also want to read the [Client-Server API +HOWTO](http://matrix.org/docs/howtos/client-server.html) too. + +If you already have communication infrastructure set up (XMPP, custom HTTP, or whatever), +then you'll want to run a bridge to expose it to the wider Matrix ecosystem. +See [matrix-appservice-bridge HOWTO](https://github.com/matrix-org/matrix-appservice-bridge/blob/master/HOWTO.md) for a +guide of how to write bridges using the matrix-appservice-bridge framework, or co-opt one +from the list at https://matrix.org/blog/try-matrix-now. +[Application Service API](/docs/spec/#application-service-api) gives the details of the API +that bridges have to implement. ##### How can I write a client on Matrix? @@ -308,16 +399,20 @@ HOWTO](http://matrix.org/docs/howtos/client-server.html) and the [API docs](/docs/api) and [the Spec](/docs/spec) for all the details you need to write a client. -##### *How can I help out with this?* +##### How can I help out with this? + +Come say hi on #matrix:matrix.org! Install synapse and tell us how you get on. Critique the spec.  Write +clients. Write bridges! Run bridges! Nose around in [Jira](https://matrix.org/jira) and +send us some pull requests on github to fix some bugs or add some features! You could even +try to write a homeserver (but be warned, Matrix's architecture makes homeservers orders of +magnitude harder than clients or bridges.) -Install synapse and tell us how you get on. Critique the spec.  Write -clients.  Just come say hi on [\#matrix:matrix.org](/alpha) or the -[mailing lists](/mailman/listinfo/matrix-users)! +See [CONTRIBUTING.rst](http://github.com/matrix-org/synapse/tree/master/CONTRIBUTING.rst) for +full details on how to contribute to the project. All are welcome! ##### Where can I get support? -[\#matrix:matrix.org](/alpha), \#matrix on irc.freenode.net or -the [mailing lists](/mailman/listinfo/matrix-users) are your best bets. +\#matrix:matrix.org aka \#matrix on irc.freenode.is your best bet. ##### How do I register custom matrix event types? @@ -327,66 +422,108 @@ use the [mailing list](/mailman/listinfo/matrix-users) for now. ##### How mature is this? -We started working on Matrix in July 2014, and have opened it to the -public in September 2014.  It's early days, and under no circumstances -should you use Matrix or Synapse for anything other than experimentation -and learning at this point.  Obviously the spec and apps are maturing -rapidly, but as of the time of writing APIs are not frozen and the apps -are very much a work in progress. +We started working on Matrix in July 2014, and opened it to the +public in September 2014. We got all the core features in place in December 2014 +and entered beta, and since then have been iterating away on the beta refining the +architecture and APIs, fixing bugs and scalability, and adding new features, clients, +bridges etc. + +As of October 2015 (synapse 0.10) it's good for serious experimentation and +non-production services and can absolutely be used in the real world. However, we're +still in beta and we'll want to freeze the spec and implement clustering and other +nice features before we really declare it ready for production. + +### Standard + +##### What is a client? + +Users in Matrix use one or more clients to communicate. This could be any combination of a web client, a command line client, a mobile client - or embedded clients built into existing apps. It could even be a piece of hardware (e.g. a drone) that is Matrix enabled. + +##### Can I use Matrix without installing a Matrix client? + +Sure. An ever increasing number of protocols are being bridged into Matrix, so if you use something like IRC on Freenode you may well be indirectly benefiting from Matrix, as others may be connected into the IRC channel via Matrix. + +##### What is a home server? + +A user's clients connect to a single homeserver, which stores the communication history and account information for that user, and shares data with the wider Matrix ecosystem by synchronising communication history with other homeservers. + +##### What is an identity server? + +Users in Matrix are identified internally via their matrix user ID (MXID). However, existing 3rd party ID (3PID) namespaces such as email addresses or phone numbers should be used publically to identify Matrix users, at least for invitation purposes. A Matrix "Identity" describes both the user ID and any other existing IDs from third party namespaces linked to their account. | -*Sorry, the FAQ is still work in progress, the rest of it will up -soon!* *In the mean time, don't hesitate to get in touch on -[\#matrix:matrix.org](/alpha) or the [mailing -lists](/mailman/listinfo/matrix-users)!* +Matrix users can link third-party IDs (3PIDs) to their user ID. Linking 3PIDs creates a mapping from a 3PID to a user ID. This mapping can then be used by Matrix users in order to discover the MXIDs of their contacts. | -### Standard +In order to ensure that the mapping from 3PID to user ID is genuine, a globally federated cluster of trusted "Identity Servers" (IS) are used to verify the 3PID and persist and replicate the mappings. +Usage of an IS is not required in order for a client application to be part of the Matrix ecosystem. However, without one clients will not be able to look up user IDs using 3PIDs. -##### What is a home server? +| -##### What is an identity sever? +The precise architecture of identity servers is currently in flux and subject to change as we work to fully decentralise them. ##### Where do my conversations get stored? +Each homeserver stores the communication history and account information for all of its clients, and shares data with the wider Matrix ecosystem by synchronising communication history with other homeservers and their clients. Clients typically communicate with each other by emitting events in the context of a virtual room. Room data is replicated across all of the homeservers *whose users are participating in a given room*. + ##### What is a 3PID? +Third-party IDs (3PIDs) are IDs from other systems or contexts, such as email addresses, social network accounts and phone numbers. + ##### How do you do VoIP calls on Matrix? +Voice (and video) over Matrix uses the WebRTC 1.0 standard to transfer call media (i.e. the actual voice and video traffic). Matrix is used to signal the establishment and termination of the call by sending call events, like any other event. Currently calls are only supported in rooms with exactly two participants - however, one of those participants may be a conferencing bridge. We're looking at better ways to do group calling. + ##### Can I log into other homeservers with my username and password? +Currently, no. We are looking at options for decentralising or migrating user accounts between multiple servers, and might add this feature at a later stage. + ##### Why Apache Licence? +The Apache Licence is a permissive licence. We want the Matrix protocol itself to be free and open, but people are free to create both free and commercial apps and services that uses the protocol. In our opinion, any Matrix-service only enhances the Matrix ecosystem. + ##### Can I write a Matrix homeserver? +Yes. Matrix is just a spec, so implementations of the spec are very welcome! It should be noted that as of October 2015 the server, changes are still being made to the spec, so if you want to write a Matrix homeserver, it is strongly recommended that you chat to the Matrix.org devs in #matrix:matrix.org first! You can also read about the [Federation API here]( https://github.com/matrix-org/matrix-doc/blob/master/specification/30_server_server_api.rst). + ##### How secure is this? -##### Why aren't you using an ORM layer like SqlAlchemy?   +Server-server traffic is mandatorily TLS from the outset. Server-client traffic mandates transport layer encryption other than for tinkering. Servers maintain a public/private key pair, and sign the integrity of all messages in the context of the historical conversation, preventing tampering. Server keys are distributed using a PERSPECTIVES-style system. -| +End-to-end encryption is coming shortly to clients for both 1:1 and group chats to protect user data stored on servers, using the [Olm](https://matrix.org/git/olm) cryptographic ratchet implementation. As of October 2015 this is blocked on implementing the necessary key distribution and fingerprint management. -### APIs +Privacy of metadata is not currently protected from server administrators - a malicious homeserver administrator can see who is talking to who and when, but not what is being said (once E2E encryption is enabled). See [this presentation from Jardin Entropique](http://matrix.org/~matthew/2015-06-26%20Matrix%20Jardin%20Entropique.pdf) for a more comprehensive discussion of privacy in Matrix. + +### Implementations + +##### What is Synapse? + +Synapse is a reference "homeserver" implementation of Matrix from the core development team at matrix.org, written in Python 2/Twisted. It is intended to showcase the concept of Matrix and let folks see the spec in the context of a codebase and let you run your own homeserver and generally help bootstrap the ecosystem. ##### How do I join the global Matrix federation? +You can download and run one of the available Matrix servers - please see [this guide](http://matrix.org/docs/guides/getting_involved.html#run) for details! + ##### What ports do I have to open up to join the global Matrix federation? -| +We recommend servers use port 8448 for server\<-\>server HTTPS traffic. Look at ["Setting up Federation"](https://github.com/matrix-org/synapse#setting-up-federation) in the Synapse readme file for details. -### Reference Implementations +Client\<-\>Server traffic can talk directly to Synapse via port 8448, but as by default Synapse creates a self-signed TLS certificate this can cause problems for clients which can't easily trust self-signed certificates (e.g. most web browsers). Instead, you can proxy access to Synapse's HTTP listener on port 8008 via an existing HTTPS proxy with a valid certificate (e.g. an nginx listening on port 443), or you can point Synapse at a valid X.509 signed TLS certificate. In future, Synapse will probably use letsencrypt to autogenerate valid certificates rather than self-signed ones during installation, simplifying this process enormously. -##### What is Matrix built on - and why? +You can also put Synapse entirely behind an existing TLS load balancer and not expose port 8448 at all. In this situation, Synapse will need to be configured to share the same *public* TLS certificate as the load balancer (as Synapse uses the public certificate for identity in other areas too, and it has to match the certificate that other servers see when they connect). -##### How do I run my own home server? +##### How do I run my own homeserver? + +Follow the instructions for the homeserver you want to run. If you want to run Synapse, the reference homeserver from Matrix.org, follow [these instructions](https://github.com/matrix-org/synapse#synapse-installation). ##### Can I run my own identity server? Yes - the reference implementation is [sydent](https://github.com/matrix-org/sydent) and you can run your own -ID server cluster that tracks 3rd party to Matrix ID mappings. If you -want your server to participate in the global replicated Matrix ID +ID server cluster that tracks 3rd party to Matrix ID mappings. This won't be very useful right now, though, and we don't recommend it. +If you want your server to participate in the global replicated Matrix ID service then please get in touch with us. Meanwhile, we are looking at ways of decentralising the 'official' Matrix identity service so that identity servers are 100% decentralised and can openly federate with @@ -394,20 +531,65 @@ each other. **N.B. that you can use Matrix without ever using the identity service - it exists only to map 3rd party IDs (e.g. email addresses) to matrix IDs to aid user discovery**. -##### What is Synapse? +##### What are Synapse's platform requirements? + +Synapse will use as much RAM as you give it in order to cache conversations in RAM to avoid hitting the database. For small deployments (<50 active users) around 512MB of RAM is probably okay. You can configure the amount of RAM used by synapse with the event_cache_size config parameter - the more events in the cache, the more RAM required. Synapse itself requires relatively little diskspace other than for logging (which as of October 2015 is quite verbose for debugging purposes), but as it caches the content of all the file attachments (images, videos etc) viewed by its users, you may need to size storage appropriately. Synapse is currently effectively single threaded, and will never use more than 1 core. + +| + +For better performance, one should back Synapse with a Postgres database rather than the default SQLite - see https://github.com/matrix-org/synapse/tree/master/README.rst#using-postgresql for details. ##### Why is Synapse in Python/Twisted? -##### What are Synapse's platform requirements? +This is because both provide a mature and well known event-driven async IO framework for writing serverside code. Whilst this has been okay for our initial experimentation and proof of concept, it's likely that future homeserver work will be written in a more strongly typed language (e.g. Go). + +##### Why aren't you using an ORM layer like SqlAlchemy in Synapse? -##### What are the Synapse webclient's requirements? +Synapse is *very* database dependent (as of Oct 2015; this is improving in the near future however), and we like having the flexibility to sculpt our own queries. -##### Where is the mobile app? +##### Where can I find a mobile app? + +The "Matrix Console" reference apps (ugly, geeky and powerful - intended for early adopter powerusers) can be downloaded from the [Google Play store](https://play.google.com/store/apps/details?id=org.matrix.androidsdk.alpha) + and [Apple store](https://itunes.apple.com/gb/app/matrix-console/id970074271). + +| + +For the Android app, you can also install the latest development version +built by [Jenkins](http://www.matrix.org/jenkins/job/AndroidConsoleDevelop/lastBuild/artifact/console/build/outputs/apk/console-alpha-debug.apk). + +##### Where can I find a web app? + +As of Oct 2015, the best web app options are to use https://vector.im - a glossy web client written on top of matrix-react-sdk, or the original AngularJS based client at https://matrix.org/beta, which has serious performance problems and is not currently being maintained. In future a "Matrix Console" reference web app built on matrix-react-sdk will be released by matrix.org to complement the mobile apps above. + +| -##### What decides the room member order on the webclient? +### QUESTIONS TO BE ANSWERED! + +This FAQ is a constant work in progress - patches and pull requests are *very* welcome to help us improve it. Some of the frequent questions where we need to write an answer include: + + * How do I rename servers? + * How do I change the TLS key of my server? + * How do I maintain my synapse's DB (e.g. prune old conversations)? + * How do I maintain my synapse's content repository (e.g. prune old content)? + * What are redactions? + * Why is the spec so big, especially relative to the XMPP baseline spec? + * How do I contribute to the spec? + * What is the privacy policy on Matrix.org? + * How precisely does E2E work? + * How does Matrix actually work architecturally? + * What IOT use cases are there for Matrix? + * Why is are the Matrix reference implementations written in so many different languages? + * How does push work? + * What's on the roadmap? + * How can I use Matrix to talk on Freenode or other IRC networks? + * Where can I learn more about Matrix? (link to PDFs of other presentations etc) + * Why HTTP? Doesn't HTTP suck? + * Why don't you use websockets? + * Why is synapse so resource intensive immediately after federating for the first time? + * \[your question goes here...\] | -Any other question? Please contact us on -[\#matrix:matrix.org](/alpha) or the [mailing +Any other questions? Please contact us in +[\#matrix:matrix.org](https://matrix.org/beta/#/room/%23matrix:matrix.org) or the [mailing lists](/mailman/listinfo/matrix-users)!