diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index b459e29f..42c0c0cf 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -214,7 +214,7 @@ paths: This API is called by the homeserver when it wants to present clients with specific information about the various third party networks that an application service supports. - operationId: queryMetadata + operationId: getProtocolMetadata parameters: - in: path name: protocol @@ -270,7 +270,7 @@ paths: required: true x-example: irc - in: query - name: field1, field2... + name: fields... type: string description: |- One or more custom fields that are passed to the application @@ -321,7 +321,7 @@ paths: required: true x-example: irc - in: query - name: field1, field2... + name: fields... type: string description: |- One or more custom fields that are passed to the application @@ -446,4 +446,4 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - $ref: ../client-server/definitions/errors/error.yaml \ No newline at end of file + $ref: ../client-server/definitions/errors/error.yaml diff --git a/api/application-service/definitions/protocol_metadata.yaml b/api/application-service/definitions/protocol_metadata.yaml index 72264060..2b2c8f4e 100644 --- a/api/application-service/definitions/protocol_metadata.yaml +++ b/api/application-service/definitions/protocol_metadata.yaml @@ -13,6 +13,8 @@ # limitations under the License. type: object description: Dictionary of supported third party protocols. +additionalProperties: + $ref: protocol.yaml example: { "irc": { "user_fields": ["network", "nickname"], diff --git a/api/application-service/definitions/user.yaml b/api/application-service/definitions/user.yaml index 5f8d0460..a7b2287e 100644 --- a/api/application-service/definitions/user.yaml +++ b/api/application-service/definitions/user.yaml @@ -27,5 +27,5 @@ properties: type: object example: "user": "jim" -title: Location +title: User type: object \ No newline at end of file diff --git a/api/check_examples.py b/api/check_examples.py index fb9430b6..0fb275b1 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -43,22 +43,23 @@ except ImportError as e: raise +def check_schema(filepath, example, schema): + example = resolve_references(filepath, example) + schema = resolve_references(filepath, schema) + resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file}) + jsonschema.validate(example, schema, resolver=resolver) + + def check_parameter(filepath, request, parameter): schema = parameter.get("schema") example = schema.get('example') - fileurl = "file://" + os.path.abspath(filepath) if example and schema: try: - print ("Checking request schema for: %r %r" % ( + print("Checking request schema for: %r %r" % ( filepath, request )) - # Setting the 'id' tells jsonschema where the file is so that it - # can correctly resolve relative $ref references in the schema - schema['id'] = fileurl - example = resolve_references(filepath, example) - resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file}) - jsonschema.validate(example, schema, resolver=resolver) + check_schema(filepath, example, schema) except Exception as e: raise ValueError("Error validating JSON schema for %r" % ( request @@ -68,18 +69,12 @@ def check_parameter(filepath, request, parameter): def check_response(filepath, request, code, response): example = response.get('examples', {}).get('application/json') schema = response.get('schema') - fileurl = "file://" + os.path.abspath(filepath) if example and schema: try: print ("Checking response schema for: %r %r %r" % ( filepath, request, code )) - # Setting the 'id' tells jsonschema where the file is so that it - # can correctly resolve relative $ref references in the schema - schema['id'] = fileurl - example = resolve_references(filepath, example) - resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file}) - jsonschema.validate(example, schema, resolver=resolver) + check_schema(filepath, example, schema) except Exception as e: raise ValueError("Error validating JSON schema for %r %r" % ( request, code @@ -127,30 +122,18 @@ def resolve_references(path, schema): return schema -def load_yaml(path): - if not path.startswith("file:///"): - raise Exception("Bad ref: %s" % (path,)) - path = path[len("file://"):] - with open(path, "r") as f: - return yaml.load(f) - - -def load_json(path): - if not path.startswith("file:///"): +def load_file(path): + print("Loading reference: %s" % path) + if not path.startswith("file://"): raise Exception("Bad ref: %s" % (path,)) path = path[len("file://"):] with open(path, "r") as f: - return json.load(f) - - -def load_file(path): - print("Loading reference: %s" % path) - if path.endswith(".json"): - return load_json(path) - else: - # We have to assume it's YAML because some of the YAML examples - # do not have file extensions. - return load_yaml(path) + if path.endswith(".json"): + return json.load(f) + else: + # We have to assume it's YAML because some of the YAML examples + # do not have file extensions. + return yaml.load(f) if __name__ == '__main__': diff --git a/api/client-server/tags.yaml b/api/client-server/tags.yaml index 9310b25f..b7bafab6 100644 --- a/api/client-server/tags.yaml +++ b/api/client-server/tags.yaml @@ -1,4 +1,5 @@ # Copyright 2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -64,8 +65,9 @@ paths: examples: application/json: { "tags": { - "work": {"order": "1"}, - "pinned": {} + "m.favourite": {}, + "u.Work": {"order": "1"}, + "u.Customers": {} } } tags: diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 55d4c70a..cba9ce22 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -34,7 +34,9 @@ paths: Fetches the overall metadata about protocols supported by the homeserver. Includes both the available protocols and all fields required for queries against each protocol. - operationId: queryMetadata + operationId: getProtocols + security: + - accessToken: [] responses: 200: description: The protocols supported by the homeserver. @@ -45,7 +47,9 @@ paths: summary: Retrieve metadata about a specific protocol that the homeserver supports. description: |- Fetches the metadata from the homeserver about a particular third party protocol. - operationId: queryMetadata + operationId: getProtocolMetadata + security: + - accessToken: [] parameters: - in: path name: protocol @@ -80,6 +84,8 @@ paths: identifier. It should attempt to canonicalise the identifier as much as reasonably possible given the network type. operationId: queryLocationByProtocol + security: + - accessToken: [] parameters: - in: path name: protocol @@ -113,6 +119,8 @@ paths: Retrieve a Matrix User ID linked to a user on the third party service, given a set of user parameters. operationId: queryUserByProtocol + security: + - accessToken: [] parameters: - in: path name: protocol @@ -122,7 +130,7 @@ paths: required: true x-example: irc - in: query - name: field1, field2... + name: fields... type: string description: |- One or more custom fields that are passed to the AS to help identify the user. @@ -146,12 +154,15 @@ paths: Retreive an array of third party network locations from a Matrix room alias. operationId: queryLocationByAlias + security: + - accessToken: [] parameters: - in: query name: alias type: string description: The Matrix room alias to look up. required: true + x-example: "#matrix:matrix.org" responses: 200: description: |- @@ -172,12 +183,15 @@ paths: description: |- Retreive an array of third party users from a Matrix User ID. operationId: queryUserByID + security: + - accessToken: [] parameters: - in: query name: userid type: string description: The Matrix User ID to look up. required: true + x-example: "@bob:matrix.org" responses: 200: description: |- @@ -191,4 +205,4 @@ paths: "errcode": "M_NOT_FOUND" } schema: - $ref: definitions/errors/error.yaml \ No newline at end of file + $ref: definitions/errors/error.yaml diff --git a/api/openapi_extensions.md b/api/openapi_extensions.md new file mode 100644 index 00000000..9f4745fd --- /dev/null +++ b/api/openapi_extensions.md @@ -0,0 +1,23 @@ +# OpenAPI Extensions + +For some functionality that is not directly provided by the OpenAPI v2 +specification, some extensions have been added that are to be consistent +across the specification. The defined extensions are listed below. Extensions +should not break parsers, however if extra functionality is required, aware +parsers should be able to take advantage of the added syntax. + +## Extensible Query Parameters + + + +If a unknown amount of query parameters can be added to a request, the `name` +must be `fields...`, with the trailing ellipses representing the possibility +of more fields. + +Example: + +``` + - in: query + name: fields... + type: string +``` diff --git a/api/server-server/definitions/keys.yaml b/api/server-server/definitions/keys.yaml index b08f2465..738e9e46 100644 --- a/api/server-server/definitions/keys.yaml +++ b/api/server-server/definitions/keys.yaml @@ -20,50 +20,62 @@ properties: server_name: type: string description: DNS name of the homeserver. - required: true # TODO: Verify + required: true example: "example.org" verify_keys: type: object - description: Public keys of the homeserver for verifying digital signatures. - required: true # TODO: Verify + description: |- + Public keys of the homeserver for verifying digital signatures. + + The object's key is the algorithm and version combined (``ed25519`` being the + algorithm and ``abc123`` being the version in the example below). Together, + this forms the Key ID. The version must have characters matching the regular + expression ``[a-zA-Z0-9_]``. + required: true additionalProperties: type: object title: Verify Key example: { - "ed25519:auto2": { - "key": "Base+64+Encoded+Signature+Verification+Key" + "ed25519:abc123": { + "key": "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA" } } properties: key: type: string - description: The key + description: The `Unpadded Base64`_ encoded key. required: true - example: "Base+64+Encoded+Signature+Verification+Key" + example: "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA" old_verify_keys: type: object - description: The public keys that the server used to use and when it stopped using them. + description: |- + The public keys that the server used to use and when it stopped using them. + + The object's key is the algorithm and version combined (``ed25519`` being the + algorithm and ``0ldK3y`` being the version in the example below). Together, + this forms the Key ID. The version must have characters matching the regular + expression ``[a-zA-Z0-9_]``. additionalProperties: type: object title: Old Verify Key example: { - "ed25519:auto1": { - "expired_ts": 922834800000, - "key": "Base+64+Encoded+Signature+Verification+Key" + "ed25519:0ldK3y": { + "expired_ts": 1532645052628, + "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg" } } properties: expired_ts: type: integer format: int64 - description: The expiration time. + description: POSIX timestamp in milliseconds for when this key expired. required: true - example: 922834800000 + example: 1532645052628 key: type: string - description: The key. + description: The `Unpadded Base64`_ encoded key. required: true - example: "Base+64+Encoded+Signature+Verification+Key" + example: "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg" signatures: type: object description: Digital signatures for this object signed using the ``verify_keys``. @@ -72,7 +84,7 @@ properties: title: Signed Server example: { "example.org": { - "ad25519:auto2": "Base+64+Encoded+Signature+Verification+Key" + "ad25519:abc123": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU" } } additionalProperties: @@ -80,17 +92,19 @@ properties: name: Encoded Signature Verification Key tls_fingerprints: type: array - description: Hashes of X.509 TLS certificates used by this server encoded as `Unpadded Base64`_. + description: Hashes of X.509 TLS certificates used by this server. items: type: object title: TLS Fingerprint properties: sha256: type: string - description: The encoded fingerprint. - example: Base+64+Encoded+SHA-256-Fingerprint + description: The `Unpadded Base64`_ encoded fingerprint. + example: "VGhpcyBpcyBoYXNoIHdoaWNoIHNob3VsZCBiZSBieXRlcw" valid_until_ts: type: integer format: int64 - description: POSIX timestamp when the list of valid keys should be refreshed. + description: |- + POSIX timestamp when the list of valid keys should be refreshed. Keys used beyond this + timestamp are no longer valid. example: 1052262000000 diff --git a/api/server-server/definitions/keys_query_response.yaml b/api/server-server/definitions/keys_query_response.yaml index bf2d238f..52ad506c 100644 --- a/api/server-server/definitions/keys_query_response.yaml +++ b/api/server-server/definitions/keys_query_response.yaml @@ -15,13 +15,13 @@ type: object description: Server keys example: { "server_keys": [{ - $ref: "../examples/server_key.json" + $ref: "../examples/server_key_notary_signed.json" }] } properties: server_keys: type: array title: Server Keys - description: The server keys. + description: The queried server's keys, signed by the notary server. items: $ref: "keys.yaml" diff --git a/api/server-server/examples/server_key.json b/api/server-server/examples/server_key.json index a3934bd3..bebd8445 100644 --- a/api/server-server/examples/server_key.json +++ b/api/server-server/examples/server_key.json @@ -1,23 +1,23 @@ { "server_name": "example.org", "verify_keys": { - "ed25519:auto2": { - "key": "Base+64+Encoded+Signature+Verification+Key" + "ed25519:abc123": { + "key": "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA" } }, "old_verify_keys": { - "ed25519:auto1": { - "expired_ts": 922834800000, - "key": "Base+64+Encoded+Old+Verify+Key" + "ed25519:0ldk3y": { + "expired_ts": 1532645052628, + "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg" } }, "signatures": { "example.org": { - "ed25519:auto2": "Base+64+Encoded+Signature" + "ed25519:auto2": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU" } }, "tls_fingerprints": [{ - "sha256": "Base+64+Encoded+SHA-256-Fingerprint" + "sha256": "VGhpcyBpcyBoYXNoIHdoaWNoIHNob3VsZCBiZSBieXRlcw" }], - "valid_until_ts": 1052262000000 + "valid_until_ts": 1652262000000 } \ No newline at end of file diff --git a/api/server-server/examples/server_key_notary_signed.json b/api/server-server/examples/server_key_notary_signed.json new file mode 100644 index 00000000..d3a461ba --- /dev/null +++ b/api/server-server/examples/server_key_notary_signed.json @@ -0,0 +1,11 @@ +{ + "$ref": "server_key.json", + "signatures": { + "example.org": { + "ed25519:abc123": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU" + }, + "notary.server.com": { + "ed25519:010203": "VGhpcyBpcyBhbm90aGVyIHNpZ25hdHVyZQ" + } + } +} \ No newline at end of file diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml index eaf14e71..759361b7 100644 --- a/api/server-server/joins.yaml +++ b/api/server-server/joins.yaml @@ -29,7 +29,6 @@ paths: description: |- Asks the receiving server to return information that the sending server will need to prepare a join event to get into the room. - This is part of the `Joining Rooms`_ handshake. operationId: makeJoin parameters: - in: path @@ -95,7 +94,9 @@ paths: type: array description: |- An event reference list containing the authorization events that would - allow the member to join the room. + allow the member to join the room. This should normally be the + ``m.room.create``, ``m.room.power_levels``, and ``m.room.join_rules`` + events. items: type: array maxItems: 2 @@ -128,7 +129,12 @@ paths: "state_key": "@someone:example.org", "content": { "membership": "join" - } + }, + "auth_events": [ + ["$room_cre4te_3vent:matrix.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}], + ["$room_j0in_rul3s_3vent:matrix.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}], + ["$room_p0wer_l3vels_3vent:matrix.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}] + ] } "/send_join/{roomId}/{eventId}": put: @@ -250,27 +256,30 @@ paths: title: Room State description: The state for the room. properties: + origin: + type: string + description: The resident server's DNS name. auth_chain: type: array description: The auth chain. items: type: object - properties: {} - # TODO: Verify schema + schema: + $ref: "definitions/pdu.yaml" state: type: array description: The room state. items: type: object - properties: {} - # TODO: Verify schema - required: ["auth_chain", "state"] + schema: + $ref: "definitions/pdu.yaml" + required: ["auth_chain", "state", "origin"] examples: application/json: [ 200, { - # TODO: Use the appropriate refs (see TODOs in schema) - "auth_chain": [], - "state": [] + "origin": "matrix.org", + "auth_chain": [{"$ref": "examples/pdu.json"}], + "state": [{"$ref": "examples/pdu.json"}] } ] diff --git a/api/server-server/keys_query.yaml b/api/server-server/keys_query.yaml index f41cb35b..8fbe00dc 100644 --- a/api/server-server/keys_query.yaml +++ b/api/server-server/keys_query.yaml @@ -25,49 +25,61 @@ produces: paths: "/query/{serverName}/{keyId}": get: - summary: Retrieve a server key. - description: Retrieve a server key. + summary: Query for another server's keys + description: |- + Query for another server's keys. The receiving (notary) server must + sign the keys returned by the queried server. operationId: perspectivesKeyQuery parameters: - in: path name: serverName type: string - description: Server name. + description: The server's DNS name to query required: true x-example: matrix.org - in: path name: keyId type: string - description: Key ID. - required: true - x-example: TODO # No examples in spec so far + description: |- + **Deprecated**. Servers should not use this parameter and instead + opt to return all keys, not just the requested one. The key ID to + look up. + required: false + x-example: "ed25519:abc123" - in: query name: minimum_valid_until_ts type: integer format: int64 - description: Minimum Valid Until Milliseconds. - required: true # TODO: Verify + description: |- + A millisecond POSIX timestamp in milliseconds indicating when the returned + certificates will need to be valid until to be useful to the requesting server. + + If not supplied, the current time as determined by the notary server is used. + required: false x-example: 1234567890 responses: 200: - description: The keys for the server + description: |- + The keys for the server, or an empty array if the server could not be reached + and no cached keys were available. schema: $ref: "definitions/keys_query_response.yaml" "/query": post: - summary: Retrieve a server key - description: Retrieve a server key. + summary: Query for several server's keys + description: |- + Query for keys from multiple servers in a batch format. The receiving (notary) + server must sign the keys returned by the queried servers. operationId: bulkPerspectivesKeyQuery parameters: - in: body name: body schema: type: object - # TODO: Improve example example: { "server_keys": { - "{server_name}": { - "{key_id}": { + "example.org": { + "ed25519:abc123": { "minimum_valid_until_ts": 1234567890 } } @@ -76,7 +88,16 @@ paths: properties: server_keys: type: object - description: The query criteria. + description: |- + The query criteria. The outer ``string`` key on the object is the + server name (eg: ``matrix.org``). The inner ``string`` key is the + Key ID to query for the particular server. If no key IDs are given + to be queried, the notary server should query for all keys. If no + servers are given, the notary server must return an empty ``server_keys`` + array in the response. + + The notary server may return multiple keys regardless of the Key IDs + given. additionalProperties: type: object name: ServerName @@ -84,16 +105,25 @@ paths: additionalProperties: type: object title: Query Criteria - description: The server keys to query. + description: The server key IDs to query. properties: minimum_valid_until_ts: type: integer format: int64 - description: Minimum Valid Until MS. + description: |- + A millisecond POSIX timestamp in milliseconds indicating when + the returned certificates will need to be valid until to be + useful to the requesting server. + + If not supplied, the current time as determined by the notary + server is used. example: 1234567890 required: ['server_keys'] responses: 200: - description: The keys for the server. + description: |- + The keys for the queried servers, signed by the notary server. Servers which + are offline and have no cached keys will not be included in the result. This + may result in an empty array. schema: $ref: "definitions/keys_query_response.yaml" diff --git a/api/server-server/keys_server.yaml b/api/server-server/keys_server.yaml index 46beeebb..8734f2ed 100644 --- a/api/server-server/keys_server.yaml +++ b/api/server-server/keys_server.yaml @@ -25,18 +25,37 @@ produces: paths: "/server/{keyId}": get: - summary: Get the server's key - description: Get the server's key. + summary: Get the homeserver's public key(s) + description: |- + Gets the homeserver's published TLS fingerprints and signing keys. + The homeserver may have any number of active keys and may have a + number of old keys. + + Intermediate notary servers should cache a response for half of its + lifetime to avoid serving a stale response. Originating servers should + avoid returning responses that expire in less than an hour to avoid + repeated reqests for a certificate that is about to expire. Requesting + servers should limit how frequently they query for certificates to + avoid flooding a server with requests. + + If the server fails to respond to this request, intermediate notary + servers should continue to return the last response they received + from the server so that the signatures of old events can still be + checked. operationId: getServerKey parameters: - in: path name: keyId type: string - description: Key ID + description: |- + **Deprecated**. Servers should not use this parameter and instead + opt to return all keys, not just the requested one. The key ID to + look up. required: false - x-example: TODO # No examples in the spec so far + x-example: "ed25519:abc123" + deprecated: true responses: 200: - description: The server's keys. + description: The homeserver's keys schema: $ref: "definitions/keys.yaml" diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 3797b9d7..2ef6fed9 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -123,8 +123,11 @@ func filter(e fsnotify.Event) bool { return false } - // Avoid some temp files that vim writes - if strings.HasSuffix(e.Name, "~") || strings.HasSuffix(e.Name, ".swp") || strings.HasPrefix(e.Name, ".") { + _, fname := filepath.Split(e.Name) + + // Avoid some temp files that vim or emacs writes + if strings.HasSuffix(e.Name, "~") || strings.HasSuffix(e.Name, ".swp") || strings.HasPrefix(fname, ".") || + (strings.HasPrefix(fname, "#") && strings.HasSuffix(fname, "#")) { return false } diff --git a/scripts/proposals.py b/scripts/proposals.py index eec6d652..5c670aff 100755 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -28,7 +28,7 @@ def getpage(url, page): def getbylabel(label): pagecount = 1 json = list() - urlbase = 'https://api.github.com/repos/matrix-org/matrix-doc/issues?state=all&labels=' + label + '&page=' + urlbase = 'https://api.github.com/repos/matrix-org/matrix-doc/issues?state=all&labels=proposal,' + label + '&page=' print(urlbase) json.extend(getpage(urlbase, 1)) for page in range(2, int(pagecount) + 1): @@ -68,7 +68,6 @@ for label in labels: for item in issues[label]: # set the created date, find local field, otherwise Github - print(item) body = str(item['body']) created = re.search('^Date: (.+?)\n', body, flags=re.MULTILINE) if created is not None: @@ -138,16 +137,27 @@ for label in labels: text_file.write(" - " + str(shepherd) + "\n") # PRs - pr_list = re.search('PRs: (.+?)$', str(item['body'])) - if pr_list is not None: - pr_list_formatted = set() - pr_list = pr_list.group(1) - for p in pr_list.split(","): - prs.add(p.strip()) - pr_list_formatted.add("`PR" + str(p.strip()) + "`_") - text_file.write(" - " + ', '.join(pr_list_formatted)) - text_file.write("\n") - else: + try: + pr_list = re.search('PRs: (.+?)$', str(item['body'])) + if pr_list is not None: + pr_list_formatted = set() + pr_list = pr_list.group(1) + for p in pr_list.split(","): + if re.match(r"#\d", p.strip()): + prs.add(p.strip()) + pr_list_formatted.add("`PR" + str(p.strip()) + "`_") + elif re.match(r"https://github.com/matrix-org/matrix-doc/pulls/\d", p.strip()): + pr = "#" + p.strip().replace('https://github.com/matrix-org/matrix-doc/pulls/', '') + prs.add(pr) + pr_list_formatted.add("`PR" + str(pr) + "`_") + else: + raise RuntimeWarning + text_file.write(" - " + ', '.join(pr_list_formatted)) + text_file.write("\n") + else: + text_file.write(" - \n") + except: + print("exception parsing PRs for MSC" + str(item['number'])) text_file.write(" - \n") text_file.write("\n\n\n") diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 25e48ab3..8c74c55f 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -1,4 +1,5 @@ .. Copyright 2016 OpenMarket Ltd +.. Copyright 2018 New Vector Ltd .. .. Licensed under the Apache License, Version 2.0 (the "License"); .. you may not use this file except in compliance with the License. @@ -17,22 +18,19 @@ Room Tagging .. _module:tagging: -Users can add tags to rooms. Tags are short strings used to label rooms, e.g. -"work", "family". A room may have multiple tags. Tags are only visible to the -user that set them but are shared across all their devices. +Users can add tags to rooms. Tags are namespaced strings used to label rooms. +A room may have multiple tags. Tags are only visible to the user that set them +but are shared across all their devices. Events ------ The tags on a room are received as single ``m.tag`` event in the -``account_data`` section of a room in a ``/sync``. +``account_data`` section of a room. The content of the ``m.tag`` event is a +``tags`` key whose value is an object mapping the name of each tag to another +object. -The ``m.tag`` can also be received in a ``/events`` response or in the -``account_data`` section of a room in ``/initialSync``. ``m.tag`` -events appearing in ``/events`` will have a ``room_id`` with the room -the tags are for. - -Each tag has an associated JSON object with information about the tag, e.g how +The JSON object associated with each tag gives information about the tag, e.g how to order the rooms with a given tag. Ordering information is given under the ``order`` key as a number between 0 and @@ -43,26 +41,28 @@ after the rooms with that tag that have an ``order`` key. The name of a tag MUST not exceed 255 bytes. -The name of a tag should be human readable. When displaying tags for a room a -client should display this human readable name. When adding a tag for a room -a client may offer a list to choose from that includes all the tags that the -user has previously set on any of their rooms. +Tags namespaces are defined in the following way, depending on how the client are expected to interpret them: + +* The namespace ``m.*`` is reserved for tags defined in the Matrix specification. Clients must ignore + any tags in this namespace they don't understand. +* The namespace ``u.*`` is reserved for user-defined tags. The portion of the string after the ``u.`` + is defined to be the display name of this tag. No other semantics should be inferred from tags in + this namespace. +* A client or app willing to use special tags for advanced functionnality should namespace them similarly to state keys: ``tld.name.*`` +* Any tag in the ``tld.name.*`` form but not matching the namespace of the current client should be ignored +* Any tag not matching the above rules should be interpreted as a user tag from the ``u.*`` namespace, as if + the name had already had ``u.`` stripped from the start (ie. the name of the tag is used as the + display name directly). These non-namespaced tags are supported for historical reasons. New tags should use + one of the defined namespaces above. Two special names are listed in the specification: +The following tags are defined in the ``m.*`` namespace: -* ``m.favourite`` -* ``m.lowpriority`` +* ``m.favourite``: The user's favourite rooms. These should be shown with higher precedence than other rooms. +* ``m.lowpriority``: These should be shown with lower precedence than others. {{m_tag_event}} -Tags namespaces are defined in the following way, depending on how the client are expected to interpret them: - -* The namespace ``m.*`` is reserved for tags defined in the current specification -* The namespace ``u.*`` is reserved for user-defined tags, and the client should not try to interpret as anything other than an utf8 string -* A client or app willing to use special tags for advanced functionnality should namespace them similarly to state keys: ``tld.name.*`` -* Any tag in the ``tld.name.*`` form but not matching the namespace of the current client should be ignored -* Any tag not matching the previous rules should be interpreted as an user tag from the ``u.*`` namespace - Client Behaviour ---------------- diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index b7d5a82f..bfc98fb8 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -106,15 +106,17 @@ Server implementation Retrieving Server Keys ~~~~~~~~~~~~~~~~~~~~~~ -Version 2 -+++++++++ +.. NOTE:: + There was once a "version 1" of the key exchange. It has been removed from the + specification due to lack of significance. It may be reviewed `here + `_. -Each homeserver publishes its public keys under ``/_matrix/key/v2/server/``. -Homeservers query for keys by either getting ``/_matrix/key/v2/server/`` +Each homeserver publishes its public keys under ``/_matrix/key/v2/server/{keyId}``. +Homeservers query for keys by either getting ``/_matrix/key/v2/server/{keyId}`` directly or by querying an intermediate notary server using a -``/_matrix/key/v2/query`` API. Intermediate notary servers query the -``/_matrix/key/v2/server/`` API on behalf of another server and sign the -response with their own key. A server may query multiple notary servers to +``/_matrix/key/v2/query/{serverName}/{keyId}`` API. Intermediate notary servers +query the ``/_matrix/key/v2/server/{keyId}`` API on behalf of another server and +sign the response with their own key. A server may query multiple notary servers to ensure that they all report the same public keys. This approach is borrowed from the `Perspectives Project`_, but modified to @@ -126,113 +128,33 @@ server by querying other servers. .. _Perspectives Project: https://web.archive.org/web/20170702024706/https://perspectives-project.org/ Publishing Keys -^^^^^^^^^^^^^^^ ++++++++++++++++ Homeservers publish the allowed TLS fingerprints and signing keys in a JSON object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of ``verify_keys`` that are valid for signing federation requests made by the -server and for signing events. It contains a list of ``old_verify_keys`` which +homeserver and for signing events. It contains a list of ``old_verify_keys`` which are only valid for signing events. Finally the response contains a list of TLS -certificate fingerprints to validate any connection made to the server. - -A server may have multiple keys active at a given time. A server may have any -number of old keys. It is recommended that servers return a single JSON -response listing all of its keys whenever any ``key_id`` is requested to reduce -the number of round trips needed to discover the relevant keys for a server. -However a server may return different responses for a different ``key_id``. - -The ``tls_certificates`` field contains a list of hashes of the X.509 TLS -certificates currently used by the server. The list must include SHA-256 hashes -for every certificate currently in use by the server. These fingerprints are -valid until the millisecond POSIX timestamp in ``valid_until_ts``. - -The ``verify_keys`` can be used to sign requests and events made by the server -until the millisecond POSIX timestamp in ``valid_until_ts``. If a homeserver -receives an event with a ``origin_server_ts`` after the ``valid_until_ts`` then -it should request that ``key_id`` for the originating server to check whether -the key has expired. - -The ``old_verify_keys`` can be used to sign events with an ``origin_server_ts`` -before the ``expired_ts``. The ``expired_ts`` is a millisecond POSIX timestamp -of when the originating server stopped using that key. - -Intermediate notary servers should cache a response for half of its remaining -lifetime to avoid serving a stale response. Originating servers should avoid -returning responses that expire in less than an hour to avoid repeated requests -for a certificate that is about to expire. Requesting servers should limit how -frequently they query for certificates to avoid flooding a server with -requests. - -If a server goes offline intermediate notary servers should continue to return -the last response they received from that server so that the signatures of old -events sent by that server can still be checked. +certificate fingerprints to validate any connection made to the homeserver. {{keys_server_ss_http_api}} Querying Keys Through Another Server -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Servers may offer a query API ``/_matrix/key/v2/query/`` for getting the keys -for another server. This API can be used to GET a list of JSON objects for a -given server or to POST a bulk query for a number of keys from a number of -servers. Either way the response is a list of JSON objects containing the -JSON published by the server under ``/_matrix/key/v2/server/`` signed by -both the originating server and by this server. - -The ``minimum_valid_until_ts`` is a millisecond POSIX timestamp indicating -when the returned certificate will need to be valid until to be useful to the -requesting server. This can be set using the maximum ``origin_server_ts`` of -a batch of events that a requesting server is trying to validate. This allows -an intermediate notary server to give a prompt cached response even if the -originating server is offline. - -This API can return keys for servers that are offline by using cached responses -taken from when the server was online. Keys can be queried from multiple -servers to mitigate against DNS spoofing. - -{{keys_query_ss_http_api}} +++++++++++++++++++++++++++++++++++++ -Version 1 -+++++++++ -.. WARNING:: - Version 1 of key distribution is obsolete. +Servers may query another server's keys through a notary server. The notary +server may be another homeserver. The notary server will retrieve keys from +the queried servers through use of the ``/_matrix/key/v2/server/{keyId}`` +API. The notary server will additionally sign the response from the queried +server before returning the results. +Notary servers can return keys for servers that are offline or having issues +serving their own keys by using cached responses. Keys can be queried from +multiple servers to mitigate against DNS spoofing. -Homeservers publish their TLS certificates and signing keys in a JSON object -at ``/_matrix/key/v1``. - -==================== =================== ====================================== - Key Type Description -==================== =================== ====================================== -``server_name`` String DNS name of the homeserver. -``verify_keys`` Object Public keys of the homeserver for - verifying digital signatures. -``signatures`` Object Digital signatures for this object - signed using the ``verify_keys``. -``tls_certificate`` String The X.509 TLS certificate used by this - this server encoded as `Unpadded Base64`_. -==================== =================== ====================================== - -.. code:: json - - { - "server_name": "example.org", - "signatures": { - "example.org": { - "ed25519:auto": "Base+64+Encoded+Signature" - } - }, - "tls_certificate": "Base+64+Encoded+DER+Encoded+X509+TLS+Certificate", - "verify_keys": { - "ed25519:auto": "Base+64+Encoded+Signature+Verification+Key" - } - } +{{keys_query_ss_http_api}} -When fetching the keys for a server the client should check that the TLS -certificate in the JSON matches the TLS server certificate for the connection -and should check that the JSON signatures are correct for the supplied -``verify_keys``. Transactions ------------ @@ -687,8 +609,6 @@ All these URLs are name-spaced within a prefix of:: {{events_ss_http_api}} -{{joins_ss_http_api}} - Joining Rooms ------------- @@ -740,94 +660,34 @@ homeservers, though most in practice will use just two. <---------- join response The first part of the handshake usually involves using the directory server to -request the room ID and join candidates. This is covered in more detail on the -directory server documentation, below. In the case of a new user joining a -room as a result of a received invite, the joining user's homeserver could -optimise this step away by picking the origin server of that invite message as -the join candidate. However, the joining server should be aware that the origin -server of the invite might since have left the room, so should be prepared to -fall back on the regular join flow if this optimisation fails. +request the room ID and join candidates through the |/query/directory|_ +API endpoint. In the case of a new user joining a room as a result of a received +invite, the joining user's homeserver could optimise this step away by picking +the origin server of that invite message as the join candidate. However, the +joining server should be aware that the origin server of the invite might since +have left the room, so should be prepared to fall back on the regular join flow +if this optimisation fails. Once the joining server has the room ID and the join candidates, it then needs to obtain enough information about the room to fill in the required fields of the ``m.room.member`` event. It obtains this by selecting a resident from the -candidate list, and requesting the ``make_join`` endpoint using a ``GET`` -request, specifying the room ID and the user ID of the new member who is -attempting to join. +candidate list, and using the ``GET /make_join`` endpoint. The resident server +will then reply with enough information for the joining server to fill in the +event. -The resident server replies to this request with a JSON-encoded object having a -single key called ``event``; within this is an object whose fields contain some -of the information that the joining server will need. Despite its name, this -object is not a full event; notably it does not need to be hashed or signed by -the resident homeserver. The required fields are: - -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``type`` String The value ``m.room.member``. -``auth_events`` List An event-reference list containing the - authorization events that would allow - this member to join. -``content`` Object The event content. -``depth`` Integer (this field must be present but is - ignored; it may be 0) -``origin`` String The name of the resident homeserver. -``origin_server_ts`` Integer A timestamp added by the resident - homeserver. -``prev_events`` List An event-reference list containing the - immediate predecessor events. -``room_id`` String The room ID of the room. -``sender`` String The user ID of the joining member. -``state_key`` String The user ID of the joining member. -======================== ============ ========================================= - -The ``content`` field itself must be an object, containing: - -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``membership`` String The value ``join``. -======================== ============ ========================================= - -The joining server now has sufficient information to construct the real join -event from these protoevent fields. It copies the values of most of them, -adding (or replacing) the following fields: - -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``event_id`` String A new event ID specified by the joining - homeserver. -``origin`` String The name of the joining homeserver. -``origin_server_ts`` Integer A timestamp added by the joining - homeserver. -======================== ============ ========================================= - -This will be a true event, so the joining server should apply the event-signing -algorithm to it, resulting in the addition of the ``hashes`` and ``signatures`` -fields. +The joining server is expected to add or replace the ``origin``, ``origin_server_ts``, +and ``event_id`` on the templated event received by the resident server. This +event is then signed by the joining server. To complete the join handshake, the joining server must now submit this new -event to an resident homeserver, by using the ``send_join`` endpoint. This is -invoked using the room ID and the event ID of the new member event. +event to a resident homeserver, by using the ``PUT /send_join`` endpoint. The resident homeserver then accepts this event into the room's event graph, and responds to the joining server with the full set of state for the -newly-joined room. This is returned as a two-element list, whose first element -is the integer 200, and whose second element is an object which contains the -following keys: +newly-joined room. The resident server must also send the event to other servers +participating in the room. -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``auth_chain`` List A list of events giving all of the events - in the auth chains for the join event and - the events in ``state``. -``state`` List A complete list of the prevailing state - events at the instant just before - accepting the new ``m.room.member`` - event. -======================== ============ ========================================= +{{joins_ss_http_api}} .. TODO-spec - (paul) I don't really understand why the full auth_chain events are given @@ -1286,6 +1146,9 @@ that are too long. [[TODO(markjh) We might want to allow the server to omit the output of well known hash functions like SHA-256 when none of the keys have been redacted]] +.. |/query/directory| replace:: ``/query/directory`` +.. _/query/directory: #get-matrix-federation-v1-query-directory + .. _`Invitation storage`: ../identity_service/unstable.html#invitation-storage .. _`Identity Service API`: ../identity_service/unstable.html .. _`Client-Server API`: ../client_server/unstable.html#m-room-member