Add support for `$ref` URIs containing fragments in OpenAPI definitions and JSON schemas (#1751)

Signed-off-by: Kévin Commaille <zecakeh@tedomum.fr>
pull/1745/head
Kévin Commaille 8 months ago committed by GitHub
parent 0b43b5a343
commit 4d7e33ec26
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -0,0 +1 @@
Add support for `$ref` URIs containing fragments in OpenAPI definitions and JSON schemas.

@ -3,11 +3,16 @@
Renders an event example. Resolves `$ref`s, serializes as JSON, and ensures
that it can be included in HTML.
This partial is called with the example event object as its context.
Parameters:
* `schema`: the schema of the example
* `name`: the name of the example
*/}}
{{ $example_content := partial "json-schema/resolve-refs" (dict "schema" . "path" "event-schemas/examples") }}
{{ $path := delimit (slice "event-schemas/examples" .name) "/" }}
{{ $example_content := partial "json-schema/resolve-refs" (dict "schema" .schema "path" $path) }}
{{ $example_json := jsonify (dict "indent" " ") $example_content }}
{{ $example_json = replace $example_json "\\u003c" "<" }}
{{ $example_json = replace $example_json "\\u003e" ">" | safeHTML }}

@ -77,7 +77,7 @@
*/}}
{{ if $desired_example_name }}
{{ if eq $example_name $desired_example_name }}
{{ partial "events/example" $example }}
{{ partial "events/example" (dict "schema" $example "name" $example_name) }}
{{ end }}
{{/*
If `$desired_example_name` is not given, we will include any
@ -86,7 +86,7 @@
the event name includes a "$".
*/}}
{{ else if eq $event_name $example_name }}
{{ partial "events/example" $example }}
{{ partial "events/example" (dict "schema" $example "name" $example_name) }}
{{/*
If `$desired_example_name` is not given, we will include any
examples whose first part (before "$") matches the event name
@ -96,7 +96,7 @@
{{ $pieces := split $example_name "$" }}
{{ $example_base_name := index $pieces 0 }}
{{ if eq $event_name $example_base_name }}
{{ partial "events/example" $example }}
{{ partial "events/example" (dict "schema" $example "name" $example_name) }}
{{ end }}
{{ end }}
{{ end }}

@ -1,7 +1,10 @@
{{/*
Resolves the `$ref` JSON schema keyword, by recursively replacing
it with the object it points to.
it with the object it points to, given:
* `schema`: the schema where the references should be resolved
* `path`: the path of the file containing the schema
This template uses [`Scratch`](https://gohugo.io/functions/scratch/)
rather than a normal `dict` because with `dict` you can't replace key values:
@ -21,7 +24,11 @@
{{ $ref_value := index $schema "$ref"}}
{{ if $ref_value }}
{{ $full_path := path.Join $path $ref_value }}
{{ $uri := urls.Parse $path }}
{{ $ref_uri := urls.Parse $ref_value }}
{{ $full_uri := $uri.ResolveReference $ref_uri }}
{{ $full_path := strings.TrimPrefix "/" $full_uri.Path }}
{{/*
Apparently Hugo doesn't give us a nice way to split the extension off a filename.
*/}}
@ -30,6 +37,13 @@
{{ $ref := index site.Data $pieces }}
{{/* If there is a fragment, follow the JSON Pointer */}}
{{ if $full_uri.Fragment }}
{{ $fragment := strings.TrimPrefix "/" $full_uri.Fragment }}
{{ $pieces := split $fragment "/" }}
{{ $ref = index $ref $pieces }}
{{ end }}
{{ $new_path := (path.Split $full_path).Dir}}
{{ $result_map := partial "json-schema/resolve-refs" (dict "schema" $ref "path" $new_path)}}
{{ if $result_map}}

@ -5,6 +5,7 @@
* `parameters`: OpenAPI data specifying the parameters
* `type`: the type of parameters to render: "header, ""path", "query"
* `caption`: caption to use for the table
* `path`: the path where this definition was found, to enable us to resolve "$ref"
This template renders a single table containing parameters of the given type.
@ -13,7 +14,9 @@
{{ $parameters := .parameters }}
{{ $type := .type }}
{{ $caption := .caption }}
{{ $path := .path }}
{{ $parameters = partial "json-schema/resolve-refs" (dict "schema" $parameters "path" $path) }}
{{ $parameters_of_type := where $parameters "in" $type }}
{{ with $parameters_of_type }}
@ -32,5 +35,4 @@
{{/* and render the parameters */}}
{{ partial "openapi/render-object-table" (dict "title" $caption "properties" $param_dict) }}
{{ end }}

@ -26,9 +26,9 @@
{{ if $parameters }}
<h3>Request parameters</h3>
{{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "header" "caption" "header parameters") }}
{{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "path" "caption" "path parameters") }}
{{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "query" "caption" "query parameters") }}
{{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "header" "caption" "header parameters" "path" .path) }}
{{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "path" "caption" "path parameters" "path" .path) }}
{{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "query" "caption" "query parameters" "path" .path) }}
{{ end }}

@ -26,6 +26,8 @@
<th class="col-status-description">Description</th>
</thead>
{{ $responses = partial "json-schema/resolve-refs" (dict "schema" $responses "path" $path) }}
{{ range $code, $response := $responses }}
<tr>
@ -49,8 +51,7 @@
Display the JSON schemas
*/}}
{{ $schema := partial "json-schema/resolve-refs" (dict "schema" $json_body.schema "path" $path) }}
{{ $schema := partial "json-schema/resolve-allof" $schema }}
{{ $schema := partial "json-schema/resolve-allof" $json_body.schema }}
{{/*
All this is to work out how to express the content of the response

@ -22,10 +22,6 @@
{{ errorf "site data %s not found" $path }}
{{ end }}
{{/* The base path, which we use to resolve $ref, omits the last component */}}
{{ $pieces = first (sub (len $pieces) 1) $pieces}}
{{ $path = delimit $pieces "/" }}
{{/* Resolve $ref and allOf */}}
{{ $definition = partial "json-schema/resolve-refs" (dict "schema" $definition "path" $path) }}
{{ $definition = partial "json-schema/resolve-allof" $definition }}

@ -25,7 +25,7 @@
*/}}
{{ $event_data := index .Site.Data "event-schemas" "schema" .Params.event }}
{{ $path := "event-schemas/schema" }}
{{ $path := delimit (slice "event-schemas/schema" .Params.event) "/" }}
{{ $event_data = partial "json-schema/resolve-refs" (dict "schema" $event_data "path" $path) }}
{{ $event_data := partial "json-schema/resolve-allof" $event_data }}

@ -21,6 +21,6 @@
{{ $api_data := index .Site.Data.api .Params.spec .Params.api }}
{{ $base_url := (index $api_data.servers 0).variables.basePath.default }}
{{ $path := delimit (slice "api" $spec) "/" }}
{{ $path := delimit (slice "api" $spec $api) "/" }}
{{ partial "openapi/render-api" (dict "api_data" $api_data "base_url" $base_url "path" $path) }}

@ -6,7 +6,6 @@
*/}}
{{ $path := "event-schemas/schema" }}
{{ $compact := false }}
{{/*
@ -40,6 +39,7 @@
{{ range $msgtypes }}
{{ $event_data := index $site_data "event-schemas" "schema" . }}
{{ $path := delimit (slice "event-schemas/schema" .) "/" }}
{{ $event_data = partial "json-schema/resolve-refs" (dict "schema" $event_data "path" $path) }}
{{ $event_data := partial "json-schema/resolve-allof" $event_data }}

@ -18,6 +18,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import helpers
import sys
import json
import os
@ -48,51 +49,16 @@ except ImportError as e:
raise
def load_file(path):
print("Loading reference: %s" % path)
if not path.startswith("file://"):
raise Exception("Bad ref: %s" % (path,))
path = path[len("file://"):]
with open(path, "r") as f:
if path.endswith(".json"):
return json.load(f)
else:
# We have to assume it's YAML because some of the YAML examples
# do not have file extensions.
return yaml.safe_load(f)
def resolve_references(path, schema):
if isinstance(schema, dict):
# do $ref first
if '$ref' in schema:
value = schema['$ref']
path = os.path.abspath(os.path.join(os.path.dirname(path), value))
ref = load_file("file://" + path)
result = resolve_references(path, ref)
del schema['$ref']
else:
result = {}
for key, value in schema.items():
result[key] = resolve_references(path, value)
return result
elif isinstance(schema, list):
return [resolve_references(path, value) for value in schema]
else:
return schema
def check_example_file(examplepath, schemapath):
with open(examplepath) as f:
example = resolve_references(examplepath, json.load(f))
example = helpers.resolve_references(examplepath, json.load(f))
with open(schemapath) as f:
schema = yaml.safe_load(f)
fileurl = "file://" + os.path.abspath(schemapath)
schema["id"] = fileurl
resolver = jsonschema.RefResolver(fileurl, schema, handlers={"file": load_file})
resolver = jsonschema.RefResolver(fileurl, schema, handlers={"file": helpers.load_file_from_uri})
print ("Checking schema for: %r %r" % (examplepath, schemapath))
try:

@ -18,6 +18,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import helpers
import sys
import json
import os
@ -68,22 +69,10 @@ class SchemaDirReport:
self.files += other_report.files
self.errors += other_report.errors
def load_file(path):
if not path.startswith("file://"):
raise Exception(f"Bad ref: {path}")
path = path[len("file://"):]
with open(path, "r") as f:
if path.endswith(".json"):
return json.load(f)
else:
# We have to assume it's YAML because some of the YAML examples
# do not have file extensions.
return yaml.safe_load(f)
def check_example(path, schema, example):
# URI with scheme is necessary to make RefResolver work.
fileurl = "file://" + os.path.abspath(path)
resolver = jsonschema.RefResolver(fileurl, schema, handlers={"file": load_file})
resolver = jsonschema.RefResolver(fileurl, schema, handlers={"file": helpers.load_file_from_uri})
validator = jsonschema.Draft202012Validator(schema, resolver)
validator.validate(example)

@ -19,6 +19,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import helpers
import sys
import json
import os
@ -49,9 +50,7 @@ except ImportError as e:
def check_schema(filepath, example, schema):
example = resolve_references(filepath, example)
schema = resolve_references(filepath, schema)
resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file})
resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": helpers.load_file_from_uri})
validator = jsonschema.Draft202012Validator(schema, resolver)
validator.validate(example)
@ -120,6 +119,8 @@ def check_openapi_file(filepath):
with open(filepath) as f:
openapi = yaml.safe_load(f)
openapi = helpers.resolve_references(filepath, openapi)
openapi_version = openapi.get('openapi')
if not openapi_version:
# This is not an OpenAPI file, skip.
@ -149,64 +150,6 @@ def check_openapi_file(filepath):
check_response(filepath, request, code, json_response)
def resolve_references(path, schema):
"""Recurse through a given schema until we find a $ref key. Upon doing so,
check that the referenced file exists, then load it up and check all of the
references in that file. Continue on until we've hit all dead ends.
$ref values are deleted from schemas as they are validated, to prevent
duplicate work.
"""
if isinstance(schema, dict):
# do $ref first
if '$ref' in schema:
# Pull the referenced filepath from the schema
referenced_file = schema['$ref']
# Referenced filepaths are relative, so take the current path's
# directory and append the relative, referenced path to it.
inner_path = os.path.join(os.path.dirname(path), referenced_file)
# Then convert the path (which may contiain '../') into a
# normalised, absolute path
inner_path = os.path.abspath(inner_path)
# Load the referenced file
ref = load_file("file://" + inner_path)
# Check that the references in *this* file are valid
result = resolve_references(inner_path, ref)
# They were valid, and so were the sub-references. Delete
# the reference here to ensure we don't pass over it again
# when checking other files
del schema['$ref']
else:
result = {}
for key, value in schema.items():
result[key] = resolve_references(path, value)
return result
elif isinstance(schema, list):
return [resolve_references(path, value) for value in schema]
else:
return schema
def load_file(path):
print("Loading reference: %s" % path)
if not path.startswith("file://"):
raise Exception("Bad ref: %s" % (path,))
path = path[len("file://"):]
with open(path, "r") as f:
if path.endswith(".json"):
return json.load(f)
else:
# We have to assume it's YAML because some of the YAML examples
# do not have file extensions.
return yaml.safe_load(f)
if __name__ == '__main__':
# Get the directory that this script is residing in
script_directory = os.path.dirname(os.path.realpath(__file__))

@ -20,6 +20,7 @@
import argparse
import errno
import helpers
import json
import logging
import os.path
@ -31,34 +32,6 @@ import yaml
scripts_dir = os.path.dirname(os.path.abspath(__file__))
api_dir = os.path.join(os.path.dirname(scripts_dir), "data", "api")
def resolve_references(path, schema):
if isinstance(schema, dict):
# do $ref first
if '$ref' in schema:
value = schema['$ref']
previous_path = path
path = os.path.join(os.path.dirname(path), value)
try:
with open(path, encoding="utf-8") as f:
ref = yaml.safe_load(f)
result = resolve_references(path, ref)
del schema['$ref']
path = previous_path
except FileNotFoundError:
print("Resolving {}".format(schema))
print("File not found: {}".format(path))
result = {}
else:
result = {}
for key, value in schema.items():
result[key] = resolve_references(path, value)
return result
elif isinstance(schema, list):
return [resolve_references(path, value) for value in schema]
else:
return schema
def prefix_absolute_path_references(text, base_url):
"""Adds base_url to absolute-path references.
@ -176,7 +149,7 @@ for filename in os.listdir(selected_api_dir):
print("Reading OpenAPI: %s" % filepath)
with open(filepath, "r") as f:
api = yaml.safe_load(f.read())
api = resolve_references(filepath, api)
api = helpers.resolve_references(filepath, api)
basePath = api['servers'][0]['variables']['basePath']['default']
for path, methods in api["paths"].items():

@ -0,0 +1,87 @@
#!/usr/bin/env python3
# Helpers to resolve $ref recursively in OpenAPI and JSON schemas.
# Copyright 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import os.path
import urllib.parse
import yaml
def resolve_references(path, schema):
"""Recurse through a given schema until we find a $ref key. Upon doing so,
check that the referenced file exists, then load it up and check all of the
references in that file. Continue on until we've hit all dead ends.
$ref values are deleted from schemas as they are validated, to prevent
duplicate work.
"""
if isinstance(schema, dict):
# do $ref first
if '$ref' in schema:
# Pull the referenced URI from the schema
ref_uri = schema['$ref']
# Join the referenced URI with the URI of the file, to resolve
# relative URIs
full_ref_uri = urllib.parse.urljoin("file://" + path, ref_uri)
# Separate the fragment.
(full_ref_uri, fragment) = urllib.parse.urldefrag(full_ref_uri)
# Load the referenced file
ref = load_file_from_uri(full_ref_uri)
if fragment:
# The fragment should be a JSON Pointer
keys = fragment.strip('/').split('/')
for key in keys:
ref = ref[key]
# Check that the references in *this* file are valid
result = resolve_references(urllib.parse.urlsplit(full_ref_uri).path, ref)
# They were valid, and so were the sub-references. Delete
# the reference here to ensure we don't pass over it again
# when checking other files
del schema['$ref']
else:
result = {}
for key, value in schema.items():
result[key] = resolve_references(path, value)
return result
elif isinstance(schema, list):
return [resolve_references(path, value) for value in schema]
else:
return schema
def load_file_from_uri(path):
"""Load a JSON or YAML file from a file:// URI.
"""
print("Loading reference: %s" % path)
if not path.startswith("file://"):
raise Exception("Bad ref: %s" % (path,))
path = path[len("file://"):]
with open(path, "r") as f:
if path.endswith(".json"):
return json.load(f)
else:
# We have to assume it's YAML because some of the YAML examples
# do not have file extensions.
return yaml.safe_load(f)
Loading…
Cancel
Save