From 560d98ba9b25f51f2da4dc2f3892b2be274ac76e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= <76261501+zecakeh@users.noreply.github.com> Date: Wed, 11 Oct 2023 12:36:39 +0200 Subject: [PATCH] Add more CI checks for OpenAPI definitions and JSON Schemas (#1656) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: KΓ©vin Commaille Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- .github/workflows/main.yml | 42 +++- .../internal/newsfragments/1656.feature | 1 + .../msgtype_infos/image_info.yaml | 1 - .../msgtype_infos/thumbnail_info.yaml | 1 - .../schema/m.room.third_party_invite.yaml | 1 - scripts/check-event-schema-examples.py | 9 +- scripts/check-json-schemas.py | 196 ++++++++++++++++++ scripts/check-openapi-sources.py | 88 ++++++-- scripts/requirements.txt | 2 + 9 files changed, 311 insertions(+), 30 deletions(-) create mode 100644 changelogs/internal/newsfragments/1656.feature create mode 100755 scripts/check-json-schemas.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c28a3329..5fa57a4f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,7 +27,7 @@ jobs: run: | npx @redocly/cli@latest lint data/api/*/*.yaml - check-examples: + check-event-examples: name: "πŸ”Ž Check Event schema examples" runs-on: ubuntu-latest steps: @@ -45,7 +45,45 @@ jobs: - name: "πŸ”Ž Run validator" run: | python scripts/check-event-schema-examples.py - + + check-openapi-examples: + name: "πŸ”Ž Check OpenAPI definitions examples" + runs-on: ubuntu-latest + steps: + - name: "πŸ“₯ Source checkout" + uses: actions/checkout@v2 + - name: "βž• Setup Python" + uses: actions/setup-python@v4 + with: + python-version: '3.9' + cache: 'pip' + cache-dependency-path: scripts/requirements.txt + - name: "βž• Install dependencies" + run: | + pip install -r scripts/requirements.txt + - name: "πŸ”Ž Run validator" + run: | + python scripts/check-openapi-sources.py + + check-schemas-examples: + name: "πŸ”Ž Check JSON Schemas inline examples" + runs-on: ubuntu-latest + steps: + - name: "πŸ“₯ Source checkout" + uses: actions/checkout@v2 + - name: "βž• Setup Python" + uses: actions/setup-python@v4 + with: + python-version: '3.9' + cache: 'pip' + cache-dependency-path: scripts/requirements.txt + - name: "βž• Install dependencies" + run: | + pip install -r scripts/requirements.txt + - name: "πŸ”Ž Run validator" + run: | + python scripts/check-json-schemas.py + calculate-baseurl: name: "βš™οΈ Calculate baseURL for later jobs" runs-on: ubuntu-latest diff --git a/changelogs/internal/newsfragments/1656.feature b/changelogs/internal/newsfragments/1656.feature new file mode 100644 index 00000000..3d6fda19 --- /dev/null +++ b/changelogs/internal/newsfragments/1656.feature @@ -0,0 +1 @@ +Add more CI checks for OpenAPI definitions and JSON Schemas. diff --git a/data/event-schemas/schema/core-event-schema/msgtype_infos/image_info.yaml b/data/event-schemas/schema/core-event-schema/msgtype_infos/image_info.yaml index 9607d6fd..7cbfcc87 100644 --- a/data/event-schemas/schema/core-event-schema/msgtype_infos/image_info.yaml +++ b/data/event-schemas/schema/core-event-schema/msgtype_infos/image_info.yaml @@ -1,4 +1,3 @@ -$schema: http://json-schema.org/draft-04/schema# description: Metadata about an image. properties: h: diff --git a/data/event-schemas/schema/core-event-schema/msgtype_infos/thumbnail_info.yaml b/data/event-schemas/schema/core-event-schema/msgtype_infos/thumbnail_info.yaml index 79f7c253..31a3b1b2 100644 --- a/data/event-schemas/schema/core-event-schema/msgtype_infos/thumbnail_info.yaml +++ b/data/event-schemas/schema/core-event-schema/msgtype_infos/thumbnail_info.yaml @@ -1,4 +1,3 @@ -$schema: http://json-schema.org/draft-04/schema# description: Metadata about a thumbnail image. properties: h: diff --git a/data/event-schemas/schema/m.room.third_party_invite.yaml b/data/event-schemas/schema/m.room.third_party_invite.yaml index 7a00616b..bb4883f5 100644 --- a/data/event-schemas/schema/m.room.third_party_invite.yaml +++ b/data/event-schemas/schema/m.room.third_party_invite.yaml @@ -1,5 +1,4 @@ --- -$schema: http://json-schema.org/draft-04/schema# allOf: - $ref: core-event-schema/state_event.yaml description: "Acts as an `m.room.member` invite event, where there isn't a target user_id to invite. This event contains a token and a public key whose private key must be used to sign the token. Any user who can present that signature may use this invitation to join the target room." diff --git a/scripts/check-event-schema-examples.py b/scripts/check-event-schema-examples.py index c6191321..b258ca2e 100755 --- a/scripts/check-event-schema-examples.py +++ b/scripts/check-event-schema-examples.py @@ -1,5 +1,9 @@ #!/usr/bin/env python -# + +# Validates the examples under `../data/event_schemas` against their JSON +# schemas. In the process, the JSON schemas are validated against the JSON +# Schema 2020-12 specification. + # Copyright 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -92,7 +96,8 @@ def check_example_file(examplepath, schemapath): print ("Checking schema for: %r %r" % (examplepath, schemapath)) try: - jsonschema.validate(example, schema, resolver=resolver) + validator = jsonschema.Draft202012Validator(schema, resolver) + validator.validate(example) except Exception as e: raise ValueError("Error validating JSON schema for %r %r" % ( examplepath, schemapath diff --git a/scripts/check-json-schemas.py b/scripts/check-json-schemas.py new file mode 100755 index 00000000..3901300f --- /dev/null +++ b/scripts/check-json-schemas.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 + +# Validates the JSON schemas under `../data`. The schemas are validated against +# the JSON Schema 2020-12 specification, and their inline examples and default +# values are validated against the schema. + +# Copyright 2023 KΓ©vin Commaille +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import json +import os +import traceback + + +def import_error(module, package, debian, error): + sys.stderr.write(( + "Error importing %(module)s: %(error)r\n" + "To install %(module)s run:\n" + " pip install %(package)s\n" + "or on Debian run:\n" + " sudo apt-get install python-%(debian)s\n" + ) % locals()) + if __name__ == '__main__': + sys.exit(1) + +try: + import jsonschema +except ImportError as e: + import_error("jsonschema", "jsonschema", "jsonschema", e) + raise + +try: + import yaml +except ImportError as e: + import_error("yaml", "PyYAML", "yaml", e) + raise + +try: + import jsonpath +except ImportError as e: + import_error("jsonpath", "python-jsonpath", "jsonpath", e) + raise + +try: + import attrs +except ImportError as e: + import_error("attrs", "attrs", "attrs", e) + raise + +@attrs.define +class SchemaDirReport: + files: int = 0 + errors: int = 0 + + def add(self, other_report): + self.files += other_report.files + self.errors += other_report.errors + +def load_file(path): + if not path.startswith("file://"): + raise Exception(f"Bad ref: {path}") + path = path[len("file://"):] + with open(path, "r") as f: + if path.endswith(".json"): + return json.load(f) + else: + # We have to assume it's YAML because some of the YAML examples + # do not have file extensions. + return yaml.safe_load(f) + +def check_example(path, schema, example): + # URI with scheme is necessary to make RefResolver work. + fileurl = "file://" + os.path.abspath(path) + resolver = jsonschema.RefResolver(fileurl, schema, handlers={"file": load_file}) + validator = jsonschema.Draft202012Validator(schema, resolver) + + validator.validate(example) + +def check_schema_examples(path, full_schema): + """Search objects with inline examples in the schema and check they validate + against the object's definition. + """ + errors = [] + matches = jsonpath.finditer( + # Recurse through all objects and filter out those that don't have an + # `example`, `examples` or `default` field. + "$..[?(@.example != undefined || @.examples != undefined || @.default != undefined)]", + full_schema + ) + + for match in matches: + schema = match.obj + if "example" in schema: + try: + check_example(path, schema, schema["example"]) + except Exception as e: + example_path = f"{match.path}['example']" + print(f"Failed to validate example at {example_path}: {e}") + errors.append(e) + + if "examples" in schema: + for index, example in enumerate(schema["examples"]): + try: + check_example(path, schema, example) + except Exception as e: + example_path = f"{match.path}['examples'][{index}]" + print(f"Failed to validate example at {example_path}: {e}") + errors.append(e) + + if "default" in schema: + try: + check_example(path, schema, schema["default"]) + except Exception as e: + example_path = f"{match.path}['default']" + print(f"Failed to validate example at {example_path}: {e}") + errors.append(e) + + if len(errors) > 0: + raise Exception(errors) + + +def check_schema_file(schema_path): + with open(schema_path) as f: + schema = yaml.safe_load(f) + + print(f"Checking schema: {schema_path}") + + # Check schema is valid. + try: + validator = jsonschema.Draft202012Validator + validator.check_schema(schema) + except Exception as e: + print(f"Failed to validate JSON schema: {e}") + raise + + # Check schema examples are valid. + check_schema_examples(schema_path, schema) + +def check_schema_dir(schemadir: str) -> SchemaDirReport: + report = SchemaDirReport() + for root, dirs, files in os.walk(schemadir): + for schemadir in dirs: + dir_report = check_schema_dir(os.path.join(root, schemadir)) + report.add(dir_report) + for filename in files: + if filename.startswith("."): + # Skip over any vim .swp files. + continue + if filename.endswith(".json"): + # Skip over any explicit examples (partial event definitions) + continue + try: + report.files += 1 + check_schema_file(os.path.join(root, filename)) + except Exception as e: + report.errors += 1 + return report + +# The directory that this script is residing in. +script_dir = os.path.dirname(os.path.realpath(__file__)) +# The directory of the project. +project_dir = os.path.abspath(os.path.join(script_dir, "../")) +print(f"Project dir: {project_dir}") + +# Directories to check, relative to the data folder. +schema_dirs = [ + "api/application-service/definitions", + "api/client-server/definitions", + "api/identity/definitions", + "api/server-server/definitions", + "event-schemas/schema", + "schemas", +] + +report = SchemaDirReport() +for schema_dir in schema_dirs: + dir_report = check_schema_dir(os.path.join(project_dir, "data", schema_dir)) + report.add(dir_report) + +print(f"Found {report.errors} errors in {report.files} files") + +if report.errors: + sys.exit(1) + diff --git a/scripts/check-openapi-sources.py b/scripts/check-openapi-sources.py index 2fb8ad93..467e8091 100755 --- a/scripts/check-openapi-sources.py +++ b/scripts/check-openapi-sources.py @@ -1,5 +1,10 @@ #! /usr/bin/env python -# + +# Validates the OpenAPI definitions under `../data/api`. Checks the request +# parameters and body, and response body. The schemas are validated against the +# JSON Schema 2020-12 specification and the examples are validated against those +# schemas. + # Copyright 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -47,17 +52,21 @@ def check_schema(filepath, example, schema): example = resolve_references(filepath, example) schema = resolve_references(filepath, schema) resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file}) - jsonschema.validate(example, schema, resolver=resolver) + validator = jsonschema.Draft202012Validator(schema, resolver) + validator.validate(example) def check_parameter(filepath, request, parameter): - schema = parameter.get("schema") - example = schema.get('example') + schema = parameter.get('schema') + example = parameter.get('example') + + if not example: + example = schema.get('example') if example and schema: try: - print("Checking request schema for: %r %r" % ( - filepath, request + print("Checking schema for request parameter: %r %r %r" % ( + filepath, request, parameter.get("name") )) check_schema(filepath, example, schema) except Exception as e: @@ -65,46 +74,79 @@ def check_parameter(filepath, request, parameter): request ), e) +def check_request_body(filepath, request, body): + schema = body.get('schema') + example = body.get('example') + + if not example: + example = schema.get('example') -def check_response(filepath, request, code, response): - example = response.get('examples', {}).get('application/json') - schema = response.get('schema') if example and schema: try: - print ("Checking response schema for: %r %r %r" % ( - filepath, request, code + print("Checking schema for request body: %r %r" % ( + filepath, request, )) check_schema(filepath, example, schema) - except jsonschema.SchemaError as error: - for suberror in sorted(error.context, key=lambda e: e.schema_path): - print(list(suberror.schema_path), suberror.message, sep=", ") - raise ValueError("Error validating JSON schema for %r %r" % ( - request, code - ), e) except Exception as e: - raise ValueError("Error validating JSON schema for %r %r" % ( - request, code + raise ValueError("Error validating JSON schema for %r" % ( + request ), e) +def check_response(filepath, request, code, response): + schema = response.get('schema') + if schema: + for name, example in response.get('examples', {}).items(): + value = example.get('value') + if value: + try: + print ("Checking response schema for: %r %r %r %r" % ( + filepath, request, code, name + )) + check_schema(filepath, value, schema) + except jsonschema.SchemaError as error: + for suberror in sorted(error.context, key=lambda e: e.schema_path): + print(list(suberror.schema_path), suberror.message, sep=", ") + raise ValueError("Error validating JSON schema for %r %r" % ( + request, code + ), e) + except Exception as e: + raise ValueError("Error validating JSON schema for %r %r" % ( + request, code + ), e) + + def check_openapi_file(filepath): with open(filepath) as f: openapi = yaml.safe_load(f) + openapi_version = openapi.get('openapi') + if not openapi_version: + # This is not an OpenAPI file, skip. + return + elif openapi_version != '3.1.0': + raise ValueError("File %r is not using the proper OpenAPI version: expected '3.1.0', got %r" % (filepath, openapi_version)) + for path, path_api in openapi.get('paths', {}).items(): for method, request_api in path_api.items(): request = "%s %s" % (method.upper(), path) for parameter in request_api.get('parameters', ()): - if parameter['in'] == 'body': - check_parameter(filepath, request, parameter) + check_parameter(filepath, request, parameter) + + json_body = request_api.get('requestBody', {}).get('content', {}).get('application/json') + if json_body: + check_request_body(filepath, request, json_body) try: responses = request_api['responses'] except KeyError: raise ValueError("No responses for %r" % (request,)) for code, response in responses.items(): - check_response(filepath, request, code, response) + json_response = response.get('content', {}).get('application/json') + + if json_response: + check_response(filepath, request, code, json_response) def resolve_references(path, schema): @@ -171,7 +213,7 @@ if __name__ == '__main__': # Resolve the directory containing the OpenAPI sources, # relative to the script path - source_files_directory = os.path.realpath(os.path.join(script_directory, "../data")) + source_files_directory = os.path.realpath(os.path.join(script_directory, "../data/api")) # Walk the source path directory, looking for YAML files to check for (root, dirs, files) in os.walk(source_files_directory): diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 0349d87f..5878b9cc 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -4,6 +4,8 @@ # we need at least version 4.0.0 for support of JSON Schema Draft 2020-12. jsonschema == 4.17.3 +python-jsonpath == 0.9.0 +attrs >= 23.1.0 PyYAML >= 3.12 requests >= 2.18.4 towncrier == 23.6.0