diff --git a/README.md b/README.md index 8b40c0f5..17fa5614 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ place after an MSC has been accepted, not as part of a proposal itself. 1. Install the extended version (often the OS default) of Hugo: . Note that at least Hugo - v0.110.0 is required. + v0.113.0 is required. Alternatively, use the Docker image at https://hub.docker.com/r/klakegg/hugo/. (The "extended edition" is required diff --git a/changelogs/appendices/newsfragments/1791.clarification b/changelogs/appendices/newsfragments/1791.clarification new file mode 100644 index 00000000..22f59c5a --- /dev/null +++ b/changelogs/appendices/newsfragments/1791.clarification @@ -0,0 +1 @@ +Define 'Opaque Identifier Grammar'. diff --git a/changelogs/internal/newsfragments/1786.clarification b/changelogs/internal/newsfragments/1786.clarification new file mode 100644 index 00000000..1810c632 --- /dev/null +++ b/changelogs/internal/newsfragments/1786.clarification @@ -0,0 +1 @@ +Upgrade jsonschema and python-jsonpath CI scripts dependencies. diff --git a/changelogs/internal/newsfragments/1788.clarification b/changelogs/internal/newsfragments/1788.clarification new file mode 100644 index 00000000..649503a0 --- /dev/null +++ b/changelogs/internal/newsfragments/1788.clarification @@ -0,0 +1 @@ +Fix Hugo warnings. diff --git a/changelogs/internal/newsfragments/1789.clarification b/changelogs/internal/newsfragments/1789.clarification new file mode 100644 index 00000000..f90c8934 --- /dev/null +++ b/changelogs/internal/newsfragments/1789.clarification @@ -0,0 +1 @@ +Fix property type resolution in `render-object-table` partial. diff --git a/content/appendices.md b/content/appendices.md index a5ddb7bf..cdaf2629 100644 --- a/content/appendices.md +++ b/content/appendices.md @@ -921,6 +921,25 @@ unique servers based on the following criteria: specify the servers it can. For example, a room with only 2 users in it would result in maximum 2 `via` parameters. +### Opaque Identifiers + +The specification defines some identifiers to use the *Opaque Identifier +Grammar*. This is a common grammar intended for non-user-visible identifiers +which do not require parsing or interpretation (other than as a unique +identifier). + +The grammar is defined as: + +* Identifiers must be entirely composed of the characters `[0-9]`, `[A-Z]`, + `[a-z]`, `-`, `.`, `_`, and `~`. +* Unless otherwise specified, identifiers must be at least one character and at + most 255 characters in length. + +{{% boxes/note %}} +The acceptable character set matches the unreserved character set in [RFC +3986](https://datatracker.ietf.org/doc/html/rfc3986#section-2.3). +{{% /boxes/note %}} + ## 3PID Types Third-party Identifiers (3PIDs) represent identifiers on other diff --git a/content/client-server-api/_index.md b/content/client-server-api/_index.md index 5889ec0f..8c017e62 100644 --- a/content/client-server-api/_index.md +++ b/content/client-server-api/_index.md @@ -943,11 +943,12 @@ or completely closed registration (where the homeserver administrators create and distribute accounts). The token required for this authentication type is shared out of band from -Matrix and is an opaque string with maximum length of 64 characters in the -range `[A-Za-z0-9._~-]`. The server can keep any number of tokens for any -length of time/validity. Such cases might be a token limited to 100 uses or -for the next 2 hours - after the tokens expire, they can no longer be used -to create accounts. +Matrix and is an opaque string using the [Opaque Identifier +Grammar](/appendices#opaque-identifiers), with maximum length of 64 +characters. The server can keep any number of tokens for any length of +time/validity. Such cases might be a token limited to 100 uses or for the next +2 hours - after the tokens expire, they can no longer be used to create +accounts. To use this authentication type, clients should submit an auth dict with just the type, token, and session: @@ -1201,7 +1202,7 @@ is complete, the client will need to submit a `/login` request matching `m.login.token`. {{< added-in v="1.7" >}} Already-authenticated clients can additionally generate -a token for their user ID if supported by the homeserver using +a token for their user ID if supported by the homeserver using [`POST /login/get_token`](/client-server-api/#post_matrixclientv1loginget_token). {{% http-api spec="client-server" api="login" %}} diff --git a/content/client-server-api/modules/voip_events.md b/content/client-server-api/modules/voip_events.md index c45a4824..14b7d796 100644 --- a/content/client-server-api/modules/voip_events.md +++ b/content/client-server-api/modules/voip_events.md @@ -96,13 +96,8 @@ Matrix clients can send DTMF as specified by WebRTC. The WebRTC standard as of A in the RTP payload. #### Grammar for VoIP IDs -`call_id`s and `party_id` are explicitly defined to be between 1 and 255 characters long, consisting -of the characters `[0-9a-zA-Z._~-]`. -(Note that this matches the grammar of 'opaque IDs' from -[MSC1597](https://github.com/matrix-org/matrix-spec-proposals/blob/rav/proposals/id_grammar/proposals/1597-id-grammar.md#opaque-ids), -and that of the `id` property of the - [`m.login.sso` flow schema](#definition-mloginsso-flow-schema).) +`call_id`s and `party_id` must follow the [Opaque Identifier Grammar](/appendices#opaque-identifiers). #### Behaviour on Room Leave If the client sees the user it is in a call with leave the room, the client should treat this diff --git a/data/api/client-server/definitions/sso_login_flow.yaml b/data/api/client-server/definitions/sso_login_flow.yaml index e30b18f9..3b95e664 100644 --- a/data/api/client-server/definitions/sso_login_flow.yaml +++ b/data/api/client-server/definitions/sso_login_flow.yaml @@ -40,10 +40,7 @@ properties: description: |- Opaque string chosen by the homeserver, uniquely identifying the IdP from other IdPs the homeserver might support. Should - be between 1 and 255 characters in length, containing unreserved - characters under [RFC 3986](http://www.ietf.org/rfc/rfc3986.txt) - (`ALPHA DIGIT "-" / "." / "_" / "~"`). Clients are not intended - to parse or infer meaning from opaque strings. + use the [Opaque identifier Grammar](/appendices#opaque-identifiers). example: "com.example.idp.github" name: type: string diff --git a/layouts/partials/openapi/render-object-table.html b/layouts/partials/openapi/render-object-table.html index ce5a7986..030978b1 100644 --- a/layouts/partials/openapi/render-object-table.html +++ b/layouts/partials/openapi/render-object-table.html @@ -75,20 +75,20 @@ {{ end }} {{/* - Computes the type to display for a property, given: + Computes the type to display for a property's schema, given: - * `type`: string or array of strings for the type(s) of the property + * `type`: optional string or array of strings for the type(s) of the property * `title`: optional string for the title of the property * `oneOf`: optional array of dictionaries describing the different formats that the property can have - * `additionalProperties`: optional dictionary for properties with undefined - names + * `additionalProperties`: if the type is an object, optional dictionary for + properties with undefined names - * `patternProperties`: optional dictionary for properties with names - adhering to a regex pattern + * `patternProperties`: if the type is an object, optional dictionary for + properties with names adhering to a regex pattern * `items`: if the type is an array, array of dictionaries describing the format of the array's items @@ -97,34 +97,56 @@ */}} {{ define "partials/property-type" }} - {{ $type := .type }} - - {{ if or (eq .type "object") (and .oneOf (reflect.IsSlice .oneOf)) }} - {{ $type = partial "type-or-title" . }} - {{ end }} + {{ $type := "" }} - {{/* - If the property is an array, indicate this with square brackets, - like `[type]`. - */}} - {{ if eq .type "array"}} + {{ if eq .type "object" }} + {{/* Resolve the type or title of the object */}} + {{ $type = partial "object-type-or-title" . }} + {{ else if eq .type "array"}} + {{/* + If the property is an array, indicate this with square brackets, + like `[type]`. + */}} {{ $items := .items }} - {{ $inner_type := partial "type-or-title" $items }} + {{ $inner_type := partial "property-type" $items }} {{ $type = delimit (slice "[" $inner_type "]") "" }} + {{ else if or (reflect.IsSlice .type) .oneOf }} + {{/* + It's legal to specify an array of types. + + There are two ways to do that: + - Use an array of strings. + - Use oneOf, with items having a schema. + + Join them together in that case, like `type|other_type`. + */}} + {{ $types := slice }} + + {{ if .oneOf }} + {{ range .oneOf }} + {{ $types = $types | append (partial "property-type" .) }} + {{ end }} + {{ else }} + {{ range .type }} + {{ $types = $types | append . }} + {{ end }} + {{ end }} + + {{ $type = delimit $types "|" }} + {{ else }} + {{/* A simple type like string or boolean */}} + {{ $type = .type }} {{ end }} {{ return $type }} {{ end }} {{/* - Computes the type to display for a property's schema, given: + Computes the type to display for an object property's schema, given: - * `type`: string or array of strings for the type(s) of the property + * `type`: string equal to "object" - * `title`: optional string for the title of the property - - * `oneOf`: optional array of dictionaries describing the different formats - that the property can have + * `title`: optional string for the title of the object property * `additionalProperties`: optional dictionary for properties with undefined names @@ -136,8 +158,8 @@ The title has a higher priority than anything else. */}} -{{ define "partials/type-or-title" }} - {{ $type := "" }} +{{ define "partials/object-type-or-title" }} + {{ $type := "object" }} {{ if .title }} {{/* If the property has a `title`, use that rather than `type`. @@ -168,32 +190,8 @@ {{ end }} {{ $type = delimit (slice "{string: " (delimit $types "|") "}" ) "" }} - {{ else if reflect.IsSlice .type }} - {{/* It's legal to specify an array of types. Join them together in that case */}} - - {{ $types := slice }} - - {{ range .type }} - {{ $types = $types | append . }} - {{ end }} - - {{ $type = delimit $types "|" }} - {{ else if and .oneOf (reflect.IsSlice .oneOf) }} - {{/* - This is like an array of types except some of the types probably have a schema. - Join them together too. - */}} - - {{ $types := slice }} - - {{ range .oneOf }} - {{ $types = $types | append (partial "type-or-title" .) }} - {{ end }} - - {{ $type = delimit $types "|" }} - {{ else }} - {{ $type = .type }} {{ end }} + {{ return $type }} {{ end }} diff --git a/scripts/check-event-schema-examples.py b/scripts/check-event-schema-examples.py index 9058ff4e..fae129f6 100755 --- a/scripts/check-event-schema-examples.py +++ b/scripts/check-event-schema-examples.py @@ -42,6 +42,12 @@ except ImportError as e: import_error("jsonschema", "jsonschema", "jsonschema", e) raise +try: + import referencing +except ImportError as e: + import_error("referencing", "referencing", "referencing", e) + raise + try: import yaml except ImportError as e: @@ -56,13 +62,14 @@ def check_example_file(examplepath, schemapath): with open(schemapath) as f: schema = yaml.safe_load(f) + # $id as a URI with scheme is necessary to make registry resolver work. fileurl = "file://" + os.path.abspath(schemapath) - schema["id"] = fileurl - resolver = jsonschema.RefResolver(fileurl, schema, handlers={"file": helpers.load_file_from_uri}) + schema["$id"] = fileurl print ("Checking schema for: %r %r" % (examplepath, schemapath)) try: - validator = jsonschema.Draft202012Validator(schema, resolver) + registry = referencing.Registry(retrieve=helpers.load_resource_from_uri) + validator = jsonschema.validators.Draft202012Validator(schema, registry=registry) validator.validate(example) except Exception as e: raise ValueError("Error validating JSON schema for %r %r" % ( diff --git a/scripts/check-json-schemas.py b/scripts/check-json-schemas.py index 06b24106..5e0ceaa3 100755 --- a/scripts/check-json-schemas.py +++ b/scripts/check-json-schemas.py @@ -42,6 +42,12 @@ except ImportError as e: import_error("jsonschema", "jsonschema", "jsonschema", e) raise +try: + import referencing +except ImportError as e: + import_error("referencing", "referencing", "referencing", e) + raise + try: import yaml except ImportError as e: @@ -70,10 +76,12 @@ class SchemaDirReport: self.errors += other_report.errors def check_example(path, schema, example): - # URI with scheme is necessary to make RefResolver work. + # $id as a URI with scheme is necessary to make registry resolver work. fileurl = "file://" + os.path.abspath(path) - resolver = jsonschema.RefResolver(fileurl, schema, handlers={"file": helpers.load_file_from_uri}) - validator = jsonschema.Draft202012Validator(schema, resolver) + schema["$id"] = fileurl + + registry = referencing.Registry(retrieve=helpers.load_resource_from_uri) + validator = jsonschema.validators.Draft202012Validator(schema, registry=registry) validator.validate(example) @@ -128,7 +136,7 @@ def check_schema_file(schema_path): # Check schema is valid. try: - validator = jsonschema.Draft202012Validator + validator = jsonschema.validators.Draft202012Validator validator.check_schema(schema) except Exception as e: print(f"Failed to validate JSON schema: {e}") diff --git a/scripts/check-openapi-sources.py b/scripts/check-openapi-sources.py index 7f28d860..4ba0392e 100755 --- a/scripts/check-openapi-sources.py +++ b/scripts/check-openapi-sources.py @@ -42,6 +42,12 @@ except ImportError as e: import_error("jsonschema", "jsonschema", "jsonschema", e) raise +try: + import referencing +except ImportError as e: + import_error("referencing", "referencing", "referencing", e) + raise + try: import yaml except ImportError as e: @@ -50,8 +56,11 @@ except ImportError as e: def check_schema(filepath, example, schema): - resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": helpers.load_file_from_uri}) - validator = jsonschema.Draft202012Validator(schema, resolver) + # $id as a URI with scheme is necessary to make registry resolver work. + schema["$id"] = filepath + + registry = referencing.Registry(retrieve=helpers.load_resource_from_uri) + validator = jsonschema.validators.Draft202012Validator(schema, registry=registry) validator.validate(example) diff --git a/scripts/helpers.py b/scripts/helpers.py index c35e8e2a..a30b2831 100755 --- a/scripts/helpers.py +++ b/scripts/helpers.py @@ -19,6 +19,7 @@ import json import os import os.path +import referencing import urllib.parse import yaml @@ -84,4 +85,15 @@ def load_file_from_uri(path): else: # We have to assume it's YAML because some of the YAML examples # do not have file extensions. - return yaml.safe_load(f) \ No newline at end of file + return yaml.safe_load(f) + +def load_resource_from_uri(path): + """Load a JSON or YAML JSON Schema, as a `referencing.Resource` object, from + a file:// URI. + """ + contents = load_file_from_uri(path) + resource = referencing.Resource( + contents=contents, + specification=referencing.jsonschema.DRAFT202012 + ) + return resource diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 5878b9cc..4d508bf4 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1,10 +1,11 @@ # no doubt older versions would be fine for many of these but these were # current at the time of writing -# we need at least version 4.0.0 for support of JSON Schema Draft 2020-12. -jsonschema == 4.17.3 +# we need at least version 4.18.0 for support of referencing library. +jsonschema >= 4.18.0 +referencing >= 0.28.4 -python-jsonpath == 0.9.0 +python-jsonpath >= 1.0.0 attrs >= 23.1.0 PyYAML >= 3.12 requests >= 2.18.4