Add support for `$ref` URIs containing fragments in OpenAPI definitions and JSON schemas (#1751)

Signed-off-by: Kévin Commaille <zecakeh@tedomum.fr>

changelogs/internal/newsfragments/1751.clarification

@@ -0,0 +1 @@
+Add support for `$ref` URIs containing fragments in OpenAPI definitions and JSON schemas.

layouts/partials/events/example.html

@@ -3,11 +3,16 @@
   Renders an event example. Resolves `$ref`s, serializes as JSON, and ensures
   that it can be included in HTML.
 
-  This partial is called with the example event object as its context.
+  Parameters:
+
+    * `schema`: the schema of the example
+    * `name`: the name of the example
 
 */}}
 
-{{ $example_content := partial "json-schema/resolve-refs" (dict "schema" . "path" "event-schemas/examples") }}
+{{ $path := delimit (slice "event-schemas/examples" .name) "/" }}
+
+{{ $example_content := partial "json-schema/resolve-refs" (dict "schema" .schema "path" $path) }}
 {{ $example_json := jsonify (dict "indent" "  ") $example_content }}
 {{ $example_json = replace $example_json "\\u003c" "<" }}
 {{ $example_json = replace $example_json "\\u003e" ">" | safeHTML }}

layouts/partials/events/render-event.html

@@ -77,7 +77,7 @@
     */}}
     {{ if $desired_example_name }}
         {{ if eq $example_name $desired_example_name }}
-            {{ partial "events/example" $example }}
+            {{ partial "events/example" (dict "schema" $example "name" $example_name) }}
         {{ end }}
     {{/*
       If `$desired_example_name` is not given, we will include any
@@ -86,7 +86,7 @@
       the event name includes a "$".
     */}}
     {{ else if eq $event_name $example_name }}
-        {{ partial "events/example" $example }}
+        {{ partial "events/example" (dict "schema" $example "name" $example_name) }}
     {{/*
       If `$desired_example_name` is not given, we will include any
       examples whose first part (before "$") matches the event name
@@ -96,7 +96,7 @@
         {{ $pieces := split $example_name "$" }}
         {{ $example_base_name := index $pieces 0 }}
         {{ if eq $event_name $example_base_name }}
-             {{ partial "events/example" $example }}
+             {{ partial "events/example" (dict "schema" $example "name" $example_name) }}
         {{ end }}
     {{ end }}
 {{ end }}

layouts/partials/json-schema/resolve-refs.html

@@ -1,7 +1,10 @@
 {{/*
 
   Resolves the `$ref` JSON schema keyword, by recursively replacing
-  it with the object it points to.
+  it with the object it points to, given:
+
+  * `schema`: the schema where the references should be resolved
+  * `path`: the path of the file containing the schema
 
   This template uses [`Scratch`](https://gohugo.io/functions/scratch/)
   rather than a normal `dict` because with `dict` you can't replace key values:
@@ -21,7 +24,11 @@
 
     {{ $ref_value := index $schema "$ref"}}
     {{ if $ref_value }}
-        {{ $full_path := path.Join $path $ref_value }}
+        {{ $uri := urls.Parse $path }}
+        {{ $ref_uri := urls.Parse $ref_value }}
+        {{ $full_uri := $uri.ResolveReference $ref_uri }}
+
+        {{ $full_path := strings.TrimPrefix "/" $full_uri.Path }}
         {{/*
           Apparently Hugo doesn't give us a nice way to split the extension off a filename.
         */}}
@@ -30,6 +37,13 @@
 
         {{ $ref := index site.Data $pieces }}
 
+        {{/* If there is a fragment, follow the JSON Pointer */}}
+        {{ if $full_uri.Fragment }}
+            {{ $fragment := strings.TrimPrefix "/" $full_uri.Fragment }}
+            {{ $pieces := split $fragment "/" }}
+            {{ $ref = index $ref $pieces }}
+        {{ end }}
+
         {{ $new_path := (path.Split $full_path).Dir}}
         {{ $result_map := partial "json-schema/resolve-refs" (dict "schema" $ref "path" $new_path)}}
         {{ if $result_map}}

layouts/partials/openapi/render-parameters.html

@@ -5,6 +5,7 @@
   * `parameters`: OpenAPI data specifying the parameters
   * `type`: the type of parameters to render: "header, ""path", "query"
   * `caption`: caption to use for the table
+  * `path`: the path where this definition was found, to enable us to resolve "$ref"
 
   This template renders a single table containing parameters of the given type.
 
@@ -13,7 +14,9 @@
 {{ $parameters := .parameters }}
 {{ $type := .type }}
 {{ $caption := .caption }}
+{{ $path := .path }}
 
+{{ $parameters = partial "json-schema/resolve-refs" (dict "schema" $parameters "path" $path) }}
 {{ $parameters_of_type := where $parameters "in" $type }}
 
 {{ with $parameters_of_type }}
@@ -32,5 +35,4 @@
 
     {{/* and render the parameters */}}
     {{ partial "openapi/render-object-table" (dict "title" $caption "properties" $param_dict) }}
-
 {{ end }}

layouts/partials/openapi/render-request.html

@@ -26,9 +26,9 @@
     {{ if $parameters }}
 <h3>Request parameters</h3>
 
-        {{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "header" "caption" "header parameters") }}
+        {{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "header" "caption" "header parameters" "path" .path) }}
-        {{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "path" "caption" "path parameters") }}
+        {{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "path" "caption" "path parameters" "path" .path) }}
-        {{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "query" "caption" "query parameters") }}
+        {{ partial "openapi/render-parameters" (dict "parameters" $parameters "type" "query" "caption" "query parameters" "path" .path) }}
 
     {{ end }}
 

layouts/partials/openapi/render-responses.html

@@ -26,6 +26,8 @@
   <th class="col-status-description">Description</th>
  </thead>
 
+{{ $responses = partial "json-schema/resolve-refs" (dict "schema" $responses "path" $path) }}
+
 {{ range $code, $response := $responses }}
 
  <tr>
@@ -49,8 +51,7 @@
                 Display the JSON schemas
             */}}
 
-            {{ $schema := partial "json-schema/resolve-refs" (dict "schema" $json_body.schema "path" $path) }}
+            {{ $schema := partial "json-schema/resolve-allof" $json_body.schema }}
-            {{ $schema := partial "json-schema/resolve-allof" $schema }}
 
             {{/*
             All this is to work out how to express the content of the response

layouts/shortcodes/definition.html

@@ -22,10 +22,6 @@
   {{ errorf "site data %s not found" $path }}
 {{ end }}
 
-{{/* The base path, which we use to resolve $ref, omits the last component */}}
-{{ $pieces = first (sub (len $pieces) 1) $pieces}}
-{{ $path = delimit $pieces "/" }}
-
 {{/* Resolve $ref and allOf */}}
 {{ $definition = partial "json-schema/resolve-refs" (dict "schema" $definition "path" $path) }}
 {{ $definition = partial "json-schema/resolve-allof" $definition }}

layouts/shortcodes/event.html

@@ -25,7 +25,7 @@
 */}}
 
 {{ $event_data := index .Site.Data "event-schemas" "schema" .Params.event }}
-{{ $path := "event-schemas/schema" }}
+{{ $path := delimit (slice "event-schemas/schema" .Params.event) "/" }}
 
 {{ $event_data = partial "json-schema/resolve-refs" (dict "schema" $event_data "path" $path) }}
 {{ $event_data := partial "json-schema/resolve-allof" $event_data }}

layouts/shortcodes/http-api.html

@@ -21,6 +21,6 @@
 
 {{ $api_data := index .Site.Data.api .Params.spec .Params.api }}
 {{ $base_url := (index $api_data.servers 0).variables.basePath.default }}
-{{ $path := delimit (slice "api" $spec) "/" }}
+{{ $path := delimit (slice "api" $spec $api) "/" }}
 
 {{ partial "openapi/render-api" (dict "api_data" $api_data "base_url" $base_url "path" $path) }}

layouts/shortcodes/msgtypes.html

@@ -6,7 +6,6 @@
 
 */}}
 
-{{ $path := "event-schemas/schema" }}
 {{ $compact := false }}
 
 {{/*
@@ -40,6 +39,7 @@
 {{ range $msgtypes }}
 
     {{ $event_data := index $site_data "event-schemas" "schema" . }}
+    {{ $path := delimit (slice "event-schemas/schema" .) "/" }}
     {{ $event_data = partial "json-schema/resolve-refs" (dict "schema" $event_data "path" $path) }}
     {{ $event_data := partial "json-schema/resolve-allof" $event_data }}
 

scripts/check-event-schema-examples.py

@@ -18,6 +18,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import helpers
 import sys
 import json
 import os
@@ -48,51 +49,16 @@ except ImportError as e:
     raise
 
 
-def load_file(path):
-    print("Loading reference: %s" % path)
-    if not path.startswith("file://"):
-        raise Exception("Bad ref: %s" % (path,))
-    path = path[len("file://"):]
-    with open(path, "r") as f:
-        if path.endswith(".json"):
-            return json.load(f)
-        else:
-            # We have to assume it's YAML because some of the YAML examples
-            # do not have file extensions.
-            return yaml.safe_load(f)
-
-
-def resolve_references(path, schema):
-    if isinstance(schema, dict):
-        # do $ref first
-        if '$ref' in schema:
-            value = schema['$ref']
-            path = os.path.abspath(os.path.join(os.path.dirname(path), value))
-            ref = load_file("file://" + path)
-            result = resolve_references(path, ref)
-            del schema['$ref']
-        else:
-            result = {}
-
-        for key, value in schema.items():
-            result[key] = resolve_references(path, value)
-        return result
-    elif isinstance(schema, list):
-        return [resolve_references(path, value) for value in schema]
-    else:
-        return schema
-
-
 def check_example_file(examplepath, schemapath):
     with open(examplepath) as f:
-        example = resolve_references(examplepath, json.load(f))
+        example = helpers.resolve_references(examplepath, json.load(f))
 
     with open(schemapath) as f:
         schema = yaml.safe_load(f)
 
     fileurl = "file://" + os.path.abspath(schemapath)
     schema["id"] = fileurl
-    resolver = jsonschema.RefResolver(fileurl, schema, handlers={"file": load_file})
+    resolver = jsonschema.RefResolver(fileurl, schema, handlers={"file": helpers.load_file_from_uri})
 
     print ("Checking schema for: %r %r" % (examplepath, schemapath))
     try:

scripts/check-json-schemas.py

@@ -18,6 +18,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import helpers
 import sys
 import json
 import os
@@ -68,22 +69,10 @@ class SchemaDirReport:
         self.files += other_report.files
         self.errors += other_report.errors
         
-def load_file(path):
-    if not path.startswith("file://"):
-        raise Exception(f"Bad ref: {path}")
-    path = path[len("file://"):]
-    with open(path, "r") as f:
-        if path.endswith(".json"):
-            return json.load(f)
-        else:
-            # We have to assume it's YAML because some of the YAML examples
-            # do not have file extensions.
-            return yaml.safe_load(f)
-        
 def check_example(path, schema, example):
     # URI with scheme is necessary to make RefResolver work.
     fileurl = "file://" + os.path.abspath(path)
-    resolver = jsonschema.RefResolver(fileurl, schema, handlers={"file": load_file})
+    resolver = jsonschema.RefResolver(fileurl, schema, handlers={"file": helpers.load_file_from_uri})
     validator = jsonschema.Draft202012Validator(schema, resolver)
 
     validator.validate(example)

scripts/check-openapi-sources.py

@@ -19,6 +19,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import helpers
 import sys
 import json
 import os
@@ -49,9 +50,7 @@ except ImportError as e:
 
 
 def check_schema(filepath, example, schema):
-    example = resolve_references(filepath, example)
+    resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": helpers.load_file_from_uri})
-    schema = resolve_references(filepath, schema)
-    resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file})
     validator = jsonschema.Draft202012Validator(schema, resolver)
     validator.validate(example)
 
@@ -120,6 +119,8 @@ def check_openapi_file(filepath):
     with open(filepath) as f:
         openapi = yaml.safe_load(f)
 
+    openapi = helpers.resolve_references(filepath, openapi)
+
     openapi_version = openapi.get('openapi')
     if not openapi_version:
         # This is not an OpenAPI file, skip.
@@ -149,64 +150,6 @@ def check_openapi_file(filepath):
                     check_response(filepath, request, code, json_response)
 
 
-def resolve_references(path, schema):
-    """Recurse through a given schema until we find a $ref key. Upon doing so,
-    check that the referenced file exists, then load it up and check all of the
-    references in that file. Continue on until we've hit all dead ends.
-
-    $ref values are deleted from schemas as they are validated, to prevent
-    duplicate work.
-    """
-    if isinstance(schema, dict):
-        # do $ref first
-        if '$ref' in schema:
-            # Pull the referenced filepath from the schema
-            referenced_file = schema['$ref']
-
-            # Referenced filepaths are relative, so take the current path's
-            # directory and append the relative, referenced path to it.
-            inner_path = os.path.join(os.path.dirname(path), referenced_file)
-
-            # Then convert the path (which may contiain '../') into a
-            # normalised, absolute path
-            inner_path = os.path.abspath(inner_path)
-
-            # Load the referenced file
-            ref = load_file("file://" + inner_path)
-
-            # Check that the references in *this* file are valid
-            result = resolve_references(inner_path, ref)
-
-            # They were valid, and so were the sub-references. Delete
-            # the reference here to ensure we don't pass over it again
-            # when checking other files
-            del schema['$ref']
-        else:
-            result = {}
-
-        for key, value in schema.items():
-            result[key] = resolve_references(path, value)
-        return result
-    elif isinstance(schema, list):
-        return [resolve_references(path, value) for value in schema]
-    else:
-        return schema
-
-
-def load_file(path):
-    print("Loading reference: %s" % path)
-    if not path.startswith("file://"):
-        raise Exception("Bad ref: %s" % (path,))
-    path = path[len("file://"):]
-    with open(path, "r") as f:
-        if path.endswith(".json"):
-            return json.load(f)
-        else:
-            # We have to assume it's YAML because some of the YAML examples
-            # do not have file extensions.
-            return yaml.safe_load(f)
-
-
 if __name__ == '__main__':
     # Get the directory that this script is residing in
     script_directory = os.path.dirname(os.path.realpath(__file__))

scripts/dump-openapi.py

@@ -20,6 +20,7 @@
 
 import argparse
 import errno
+import helpers
 import json
 import logging
 import os.path
@@ -31,34 +32,6 @@ import yaml
 scripts_dir = os.path.dirname(os.path.abspath(__file__))
 api_dir = os.path.join(os.path.dirname(scripts_dir), "data", "api")
 
-def resolve_references(path, schema):
-    if isinstance(schema, dict):
-        # do $ref first
-        if '$ref' in schema:
-            value = schema['$ref']
-            previous_path = path
-            path = os.path.join(os.path.dirname(path), value)
-            try:
-                with open(path, encoding="utf-8") as f:
-                    ref = yaml.safe_load(f)
-                result = resolve_references(path, ref)
-                del schema['$ref']
-                path = previous_path
-            except FileNotFoundError:
-                print("Resolving {}".format(schema))
-                print("File not found: {}".format(path))
-                result = {}
-        else:
-            result = {}
-
-        for key, value in schema.items():
-            result[key] = resolve_references(path, value)
-        return result
-    elif isinstance(schema, list):
-        return [resolve_references(path, value) for value in schema]
-    else:
-        return schema
-
 def prefix_absolute_path_references(text, base_url):
     """Adds base_url to absolute-path references.
 
@@ -176,7 +149,7 @@ for filename in os.listdir(selected_api_dir):
     print("Reading OpenAPI: %s" % filepath)
     with open(filepath, "r") as f:
         api = yaml.safe_load(f.read())
-        api = resolve_references(filepath, api)
+        api = helpers.resolve_references(filepath, api)
 
         basePath = api['servers'][0]['variables']['basePath']['default']
         for path, methods in api["paths"].items():

scripts/helpers.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+
+# Helpers to resolve $ref recursively in OpenAPI and JSON schemas.
+
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import os.path
+import urllib.parse
+import yaml
+
+def resolve_references(path, schema):
+    """Recurse through a given schema until we find a $ref key. Upon doing so,
+    check that the referenced file exists, then load it up and check all of the
+    references in that file. Continue on until we've hit all dead ends.
+
+    $ref values are deleted from schemas as they are validated, to prevent
+    duplicate work.
+    """
+    if isinstance(schema, dict):
+        # do $ref first
+        if '$ref' in schema:
+            # Pull the referenced URI from the schema
+            ref_uri = schema['$ref']
+
+            # Join the referenced URI with the URI of the file, to resolve
+            # relative URIs
+            full_ref_uri = urllib.parse.urljoin("file://" + path, ref_uri)
+
+            # Separate the fragment.
+            (full_ref_uri, fragment) = urllib.parse.urldefrag(full_ref_uri)
+
+            # Load the referenced file
+            ref = load_file_from_uri(full_ref_uri)
+
+            if fragment:
+                # The fragment should be a JSON Pointer
+                keys = fragment.strip('/').split('/')
+                for key in keys:
+                    ref = ref[key]
+
+            # Check that the references in *this* file are valid
+            result = resolve_references(urllib.parse.urlsplit(full_ref_uri).path, ref)
+
+            # They were valid, and so were the sub-references. Delete
+            # the reference here to ensure we don't pass over it again
+            # when checking other files
+            del schema['$ref']
+        else:
+            result = {}
+
+        for key, value in schema.items():
+            result[key] = resolve_references(path, value)
+        return result
+    elif isinstance(schema, list):
+        return [resolve_references(path, value) for value in schema]
+    else:
+        return schema
+
+
+def load_file_from_uri(path):
+    """Load a JSON or YAML file from a file:// URI.
+    """
+    print("Loading reference: %s" % path)
+    if not path.startswith("file://"):
+        raise Exception("Bad ref: %s" % (path,))
+    path = path[len("file://"):]
+    with open(path, "r") as f:
+        if path.endswith(".json"):
+            return json.load(f)
+        else:
+            # We have to assume it's YAML because some of the YAML examples
+            # do not have file extensions.
+            return yaml.safe_load(f)