diff --git a/changelogs/internal/newsfragments/1751.clarification b/changelogs/internal/newsfragments/1751.clarification new file mode 100644 index 00000000..50c50693 --- /dev/null +++ b/changelogs/internal/newsfragments/1751.clarification @@ -0,0 +1 @@ +Add support for `$ref` URIs containing fragments in OpenAPI definitions and JSON schemas. diff --git a/layouts/partials/events/example.html b/layouts/partials/events/example.html index 181de88f..90752fbd 100644 --- a/layouts/partials/events/example.html +++ b/layouts/partials/events/example.html @@ -1,13 +1,18 @@ {{/* - Renders an event example. Resolves `$ref`s, serializes as JSON, and ensures + Renders an event example. Resolves `$ref`s, serializes as JSON, and ensures that it can be included in HTML. - This partial is called with the example event object as its context. + Parameters: + + * `schema`: the schema of the example + * `name`: the name of the example */}} -{{ $example_content := partial "json-schema/resolve-refs" (dict "schema" . "path" "event-schemas/examples") }} +{{ $path := delimit (slice "event-schemas/examples" .name) "/" }} + +{{ $example_content := partial "json-schema/resolve-refs" (dict "schema" .schema "path" $path) }} {{ $example_json := jsonify (dict "indent" " ") $example_content }} {{ $example_json = replace $example_json "\\u003c" "<" }} {{ $example_json = replace $example_json "\\u003e" ">" | safeHTML }} diff --git a/layouts/partials/events/render-event.html b/layouts/partials/events/render-event.html index 70752721..71ba60bd 100644 --- a/layouts/partials/events/render-event.html +++ b/layouts/partials/events/render-event.html @@ -77,7 +77,7 @@ */}} {{ if $desired_example_name }} {{ if eq $example_name $desired_example_name }} - {{ partial "events/example" $example }} + {{ partial "events/example" (dict "schema" $example "name" $example_name) }} {{ end }} {{/* If `$desired_example_name` is not given, we will include any @@ -86,7 +86,7 @@ the event name includes a "$". */}} {{ else if eq $event_name $example_name }} - {{ partial "events/example" $example }} + {{ partial "events/example" (dict "schema" $example "name" $example_name) }} {{/* If `$desired_example_name` is not given, we will include any examples whose first part (before "$") matches the event name @@ -96,7 +96,7 @@ {{ $pieces := split $example_name "$" }} {{ $example_base_name := index $pieces 0 }} {{ if eq $event_name $example_base_name }} - {{ partial "events/example" $example }} + {{ partial "events/example" (dict "schema" $example "name" $example_name) }} {{ end }} {{ end }} {{ end }} diff --git a/layouts/partials/json-schema/resolve-refs.html b/layouts/partials/json-schema/resolve-refs.html index 1d99201d..9a36e413 100644 --- a/layouts/partials/json-schema/resolve-refs.html +++ b/layouts/partials/json-schema/resolve-refs.html @@ -1,7 +1,10 @@ {{/* Resolves the `$ref` JSON schema keyword, by recursively replacing - it with the object it points to. + it with the object it points to, given: + + * `schema`: the schema where the references should be resolved + * `path`: the path of the file containing the schema This template uses [`Scratch`](https://gohugo.io/functions/scratch/) rather than a normal `dict` because with `dict` you can't replace key values: @@ -20,8 +23,12 @@ {{ $scratch.Set "result_map" dict }} {{ $ref_value := index $schema "$ref"}} - {{ if $ref_value}} - {{ $full_path := path.Join $path $ref_value }} + {{ if $ref_value }} + {{ $uri := urls.Parse $path }} + {{ $ref_uri := urls.Parse $ref_value }} + {{ $full_uri := $uri.ResolveReference $ref_uri }} + + {{ $full_path := strings.TrimPrefix "/" $full_uri.Path }} {{/* Apparently Hugo doesn't give us a nice way to split the extension off a filename. */}} @@ -30,11 +37,18 @@ {{ $ref := index site.Data $pieces }} + {{/* If there is a fragment, follow the JSON Pointer */}} + {{ if $full_uri.Fragment }} + {{ $fragment := strings.TrimPrefix "/" $full_uri.Fragment }} + {{ $pieces := split $fragment "/" }} + {{ $ref = index $ref $pieces }} + {{ end }} + {{ $new_path := (path.Split $full_path).Dir}} {{ $result_map := partial "json-schema/resolve-refs" (dict "schema" $ref "path" $new_path)}} {{ if $result_map}} {{ $scratch.Set "result_map" $result_map }} - {{end }} + {{ end }} {{ end }} diff --git a/layouts/partials/openapi/render-parameters.html b/layouts/partials/openapi/render-parameters.html index 925b0197..ecabfc05 100644 --- a/layouts/partials/openapi/render-parameters.html +++ b/layouts/partials/openapi/render-parameters.html @@ -5,6 +5,7 @@ * `parameters`: OpenAPI data specifying the parameters * `type`: the type of parameters to render: "header, ""path", "query" * `caption`: caption to use for the table + * `path`: the path where this definition was found, to enable us to resolve "$ref" This template renders a single table containing parameters of the given type. @@ -13,7 +14,9 @@ {{ $parameters := .parameters }} {{ $type := .type }} {{ $caption := .caption }} +{{ $path := .path }} +{{ $parameters = partial "json-schema/resolve-refs" (dict "schema" $parameters "path" $path) }} {{ $parameters_of_type := where $parameters "in" $type }} {{ with $parameters_of_type }} @@ -32,5 +35,4 @@ {{/* and render the parameters */}} {{ partial "openapi/render-object-table" (dict "title" $caption "properties" $param_dict) }} - {{ end }} diff --git a/layouts/partials/openapi/render-request.html b/layouts/partials/openapi/render-request.html index 3d4b0381..5ef55c64 100644 --- a/layouts/partials/openapi/render-request.html +++ b/layouts/partials/openapi/render-request.html @@ -26,9 +26,9 @@ {{ if $parameters }}

Request parameters

- {{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "header" "caption" "header parameters") }} - {{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "path" "caption" "path parameters") }} - {{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "query" "caption" "query parameters") }} + {{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "header" "caption" "header parameters" "path" .path) }} + {{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "path" "caption" "path parameters" "path" .path) }} + {{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "query" "caption" "query parameters" "path" .path) }} {{ end }} diff --git a/layouts/partials/openapi/render-responses.html b/layouts/partials/openapi/render-responses.html index 99662ad0..82c2f954 100644 --- a/layouts/partials/openapi/render-responses.html +++ b/layouts/partials/openapi/render-responses.html @@ -26,6 +26,8 @@ Description +{{ $responses = partial "json-schema/resolve-refs" (dict "schema" $responses "path" $path) }} + {{ range $code, $response := $responses }} @@ -49,8 +51,7 @@ Display the JSON schemas */}} - {{ $schema := partial "json-schema/resolve-refs" (dict "schema" $json_body.schema "path" $path) }} - {{ $schema := partial "json-schema/resolve-allof" $schema }} + {{ $schema := partial "json-schema/resolve-allof" $json_body.schema }} {{/* All this is to work out how to express the content of the response diff --git a/layouts/shortcodes/definition.html b/layouts/shortcodes/definition.html index 0699ff7b..23461878 100644 --- a/layouts/shortcodes/definition.html +++ b/layouts/shortcodes/definition.html @@ -22,10 +22,6 @@ {{ errorf "site data %s not found" $path }} {{ end }} -{{/* The base path, which we use to resolve $ref, omits the last component */}} -{{ $pieces = first (sub (len $pieces) 1) $pieces}} -{{ $path = delimit $pieces "/" }} - {{/* Resolve $ref and allOf */}} {{ $definition = partial "json-schema/resolve-refs" (dict "schema" $definition "path" $path) }} {{ $definition = partial "json-schema/resolve-allof" $definition }} diff --git a/layouts/shortcodes/event.html b/layouts/shortcodes/event.html index a9838542..c671318a 100644 --- a/layouts/shortcodes/event.html +++ b/layouts/shortcodes/event.html @@ -25,7 +25,7 @@ */}} {{ $event_data := index .Site.Data "event-schemas" "schema" .Params.event }} -{{ $path := "event-schemas/schema" }} +{{ $path := delimit (slice "event-schemas/schema" .Params.event) "/" }} {{ $event_data = partial "json-schema/resolve-refs" (dict "schema" $event_data "path" $path) }} {{ $event_data := partial "json-schema/resolve-allof" $event_data }} diff --git a/layouts/shortcodes/http-api.html b/layouts/shortcodes/http-api.html index a3b706db..43d08b9e 100644 --- a/layouts/shortcodes/http-api.html +++ b/layouts/shortcodes/http-api.html @@ -21,6 +21,6 @@ {{ $api_data := index .Site.Data.api .Params.spec .Params.api }} {{ $base_url := (index $api_data.servers 0).variables.basePath.default }} -{{ $path := delimit (slice "api" $spec) "/" }} +{{ $path := delimit (slice "api" $spec $api) "/" }} {{ partial "openapi/render-api" (dict "api_data" $api_data "base_url" $base_url "path" $path) }} diff --git a/layouts/shortcodes/msgtypes.html b/layouts/shortcodes/msgtypes.html index ba731111..1ab28aae 100644 --- a/layouts/shortcodes/msgtypes.html +++ b/layouts/shortcodes/msgtypes.html @@ -6,7 +6,6 @@ */}} -{{ $path := "event-schemas/schema" }} {{ $compact := false }} {{/* @@ -40,6 +39,7 @@ {{ range $msgtypes }} {{ $event_data := index $site_data "event-schemas" "schema" . }} + {{ $path := delimit (slice "event-schemas/schema" .) "/" }} {{ $event_data = partial "json-schema/resolve-refs" (dict "schema" $event_data "path" $path) }} {{ $event_data := partial "json-schema/resolve-allof" $event_data }} diff --git a/scripts/check-event-schema-examples.py b/scripts/check-event-schema-examples.py index b258ca2e..9058ff4e 100755 --- a/scripts/check-event-schema-examples.py +++ b/scripts/check-event-schema-examples.py @@ -18,6 +18,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import helpers import sys import json import os @@ -48,51 +49,16 @@ except ImportError as e: raise -def load_file(path): - print("Loading reference: %s" % path) - if not path.startswith("file://"): - raise Exception("Bad ref: %s" % (path,)) - path = path[len("file://"):] - with open(path, "r") as f: - if path.endswith(".json"): - return json.load(f) - else: - # We have to assume it's YAML because some of the YAML examples - # do not have file extensions. - return yaml.safe_load(f) - - -def resolve_references(path, schema): - if isinstance(schema, dict): - # do $ref first - if '$ref' in schema: - value = schema['$ref'] - path = os.path.abspath(os.path.join(os.path.dirname(path), value)) - ref = load_file("file://" + path) - result = resolve_references(path, ref) - del schema['$ref'] - else: - result = {} - - for key, value in schema.items(): - result[key] = resolve_references(path, value) - return result - elif isinstance(schema, list): - return [resolve_references(path, value) for value in schema] - else: - return schema - - def check_example_file(examplepath, schemapath): with open(examplepath) as f: - example = resolve_references(examplepath, json.load(f)) + example = helpers.resolve_references(examplepath, json.load(f)) with open(schemapath) as f: schema = yaml.safe_load(f) fileurl = "file://" + os.path.abspath(schemapath) schema["id"] = fileurl - resolver = jsonschema.RefResolver(fileurl, schema, handlers={"file": load_file}) + resolver = jsonschema.RefResolver(fileurl, schema, handlers={"file": helpers.load_file_from_uri}) print ("Checking schema for: %r %r" % (examplepath, schemapath)) try: diff --git a/scripts/check-json-schemas.py b/scripts/check-json-schemas.py index 3901300f..06b24106 100755 --- a/scripts/check-json-schemas.py +++ b/scripts/check-json-schemas.py @@ -18,6 +18,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import helpers import sys import json import os @@ -67,23 +68,11 @@ class SchemaDirReport: def add(self, other_report): self.files += other_report.files self.errors += other_report.errors - -def load_file(path): - if not path.startswith("file://"): - raise Exception(f"Bad ref: {path}") - path = path[len("file://"):] - with open(path, "r") as f: - if path.endswith(".json"): - return json.load(f) - else: - # We have to assume it's YAML because some of the YAML examples - # do not have file extensions. - return yaml.safe_load(f) def check_example(path, schema, example): # URI with scheme is necessary to make RefResolver work. fileurl = "file://" + os.path.abspath(path) - resolver = jsonschema.RefResolver(fileurl, schema, handlers={"file": load_file}) + resolver = jsonschema.RefResolver(fileurl, schema, handlers={"file": helpers.load_file_from_uri}) validator = jsonschema.Draft202012Validator(schema, resolver) validator.validate(example) diff --git a/scripts/check-openapi-sources.py b/scripts/check-openapi-sources.py index 467e8091..7f28d860 100755 --- a/scripts/check-openapi-sources.py +++ b/scripts/check-openapi-sources.py @@ -19,6 +19,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import helpers import sys import json import os @@ -49,9 +50,7 @@ except ImportError as e: def check_schema(filepath, example, schema): - example = resolve_references(filepath, example) - schema = resolve_references(filepath, schema) - resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file}) + resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": helpers.load_file_from_uri}) validator = jsonschema.Draft202012Validator(schema, resolver) validator.validate(example) @@ -120,6 +119,8 @@ def check_openapi_file(filepath): with open(filepath) as f: openapi = yaml.safe_load(f) + openapi = helpers.resolve_references(filepath, openapi) + openapi_version = openapi.get('openapi') if not openapi_version: # This is not an OpenAPI file, skip. @@ -149,64 +150,6 @@ def check_openapi_file(filepath): check_response(filepath, request, code, json_response) -def resolve_references(path, schema): - """Recurse through a given schema until we find a $ref key. Upon doing so, - check that the referenced file exists, then load it up and check all of the - references in that file. Continue on until we've hit all dead ends. - - $ref values are deleted from schemas as they are validated, to prevent - duplicate work. - """ - if isinstance(schema, dict): - # do $ref first - if '$ref' in schema: - # Pull the referenced filepath from the schema - referenced_file = schema['$ref'] - - # Referenced filepaths are relative, so take the current path's - # directory and append the relative, referenced path to it. - inner_path = os.path.join(os.path.dirname(path), referenced_file) - - # Then convert the path (which may contiain '../') into a - # normalised, absolute path - inner_path = os.path.abspath(inner_path) - - # Load the referenced file - ref = load_file("file://" + inner_path) - - # Check that the references in *this* file are valid - result = resolve_references(inner_path, ref) - - # They were valid, and so were the sub-references. Delete - # the reference here to ensure we don't pass over it again - # when checking other files - del schema['$ref'] - else: - result = {} - - for key, value in schema.items(): - result[key] = resolve_references(path, value) - return result - elif isinstance(schema, list): - return [resolve_references(path, value) for value in schema] - else: - return schema - - -def load_file(path): - print("Loading reference: %s" % path) - if not path.startswith("file://"): - raise Exception("Bad ref: %s" % (path,)) - path = path[len("file://"):] - with open(path, "r") as f: - if path.endswith(".json"): - return json.load(f) - else: - # We have to assume it's YAML because some of the YAML examples - # do not have file extensions. - return yaml.safe_load(f) - - if __name__ == '__main__': # Get the directory that this script is residing in script_directory = os.path.dirname(os.path.realpath(__file__)) diff --git a/scripts/dump-openapi.py b/scripts/dump-openapi.py index 1cc2279c..490ac9bf 100755 --- a/scripts/dump-openapi.py +++ b/scripts/dump-openapi.py @@ -20,6 +20,7 @@ import argparse import errno +import helpers import json import logging import os.path @@ -31,34 +32,6 @@ import yaml scripts_dir = os.path.dirname(os.path.abspath(__file__)) api_dir = os.path.join(os.path.dirname(scripts_dir), "data", "api") -def resolve_references(path, schema): - if isinstance(schema, dict): - # do $ref first - if '$ref' in schema: - value = schema['$ref'] - previous_path = path - path = os.path.join(os.path.dirname(path), value) - try: - with open(path, encoding="utf-8") as f: - ref = yaml.safe_load(f) - result = resolve_references(path, ref) - del schema['$ref'] - path = previous_path - except FileNotFoundError: - print("Resolving {}".format(schema)) - print("File not found: {}".format(path)) - result = {} - else: - result = {} - - for key, value in schema.items(): - result[key] = resolve_references(path, value) - return result - elif isinstance(schema, list): - return [resolve_references(path, value) for value in schema] - else: - return schema - def prefix_absolute_path_references(text, base_url): """Adds base_url to absolute-path references. @@ -176,7 +149,7 @@ for filename in os.listdir(selected_api_dir): print("Reading OpenAPI: %s" % filepath) with open(filepath, "r") as f: api = yaml.safe_load(f.read()) - api = resolve_references(filepath, api) + api = helpers.resolve_references(filepath, api) basePath = api['servers'][0]['variables']['basePath']['default'] for path, methods in api["paths"].items(): diff --git a/scripts/helpers.py b/scripts/helpers.py new file mode 100755 index 00000000..c35e8e2a --- /dev/null +++ b/scripts/helpers.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +# Helpers to resolve $ref recursively in OpenAPI and JSON schemas. + +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import os.path +import urllib.parse +import yaml + +def resolve_references(path, schema): + """Recurse through a given schema until we find a $ref key. Upon doing so, + check that the referenced file exists, then load it up and check all of the + references in that file. Continue on until we've hit all dead ends. + + $ref values are deleted from schemas as they are validated, to prevent + duplicate work. + """ + if isinstance(schema, dict): + # do $ref first + if '$ref' in schema: + # Pull the referenced URI from the schema + ref_uri = schema['$ref'] + + # Join the referenced URI with the URI of the file, to resolve + # relative URIs + full_ref_uri = urllib.parse.urljoin("file://" + path, ref_uri) + + # Separate the fragment. + (full_ref_uri, fragment) = urllib.parse.urldefrag(full_ref_uri) + + # Load the referenced file + ref = load_file_from_uri(full_ref_uri) + + if fragment: + # The fragment should be a JSON Pointer + keys = fragment.strip('/').split('/') + for key in keys: + ref = ref[key] + + # Check that the references in *this* file are valid + result = resolve_references(urllib.parse.urlsplit(full_ref_uri).path, ref) + + # They were valid, and so were the sub-references. Delete + # the reference here to ensure we don't pass over it again + # when checking other files + del schema['$ref'] + else: + result = {} + + for key, value in schema.items(): + result[key] = resolve_references(path, value) + return result + elif isinstance(schema, list): + return [resolve_references(path, value) for value in schema] + else: + return schema + + +def load_file_from_uri(path): + """Load a JSON or YAML file from a file:// URI. + """ + print("Loading reference: %s" % path) + if not path.startswith("file://"): + raise Exception("Bad ref: %s" % (path,)) + path = path[len("file://"):] + with open(path, "r") as f: + if path.endswith(".json"): + return json.load(f) + else: + # We have to assume it's YAML because some of the YAML examples + # do not have file extensions. + return yaml.safe_load(f) \ No newline at end of file