|
|
@ -73,7 +73,8 @@ def inherit_parents(obj):
|
|
|
|
return result
|
|
|
|
return result
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_json_schema_object_fields(obj, enforce_title=False):
|
|
|
|
def get_json_schema_object_fields(obj, enforce_title=False,
|
|
|
|
|
|
|
|
mark_required=True):
|
|
|
|
# Algorithm:
|
|
|
|
# Algorithm:
|
|
|
|
# f.e. property => add field info (if field is object then recurse)
|
|
|
|
# f.e. property => add field info (if field is object then recurse)
|
|
|
|
if obj.get("type") != "object":
|
|
|
|
if obj.get("type") != "object":
|
|
|
@ -168,6 +169,7 @@ def get_json_schema_object_fields(obj, enforce_title=False):
|
|
|
|
nested_objects = get_json_schema_object_fields(
|
|
|
|
nested_objects = get_json_schema_object_fields(
|
|
|
|
prop,
|
|
|
|
prop,
|
|
|
|
enforce_title=True,
|
|
|
|
enforce_title=True,
|
|
|
|
|
|
|
|
mark_required=mark_required,
|
|
|
|
)
|
|
|
|
)
|
|
|
|
value_type = nested_objects[0]["title"]
|
|
|
|
value_type = nested_objects[0]["title"]
|
|
|
|
value_id = value_type
|
|
|
|
value_id = value_type
|
|
|
@ -180,6 +182,7 @@ def get_json_schema_object_fields(obj, enforce_title=False):
|
|
|
|
nested_objects = get_json_schema_object_fields(
|
|
|
|
nested_objects = get_json_schema_object_fields(
|
|
|
|
items,
|
|
|
|
items,
|
|
|
|
enforce_title=True,
|
|
|
|
enforce_title=True,
|
|
|
|
|
|
|
|
mark_required=mark_required,
|
|
|
|
)
|
|
|
|
)
|
|
|
|
value_id = nested_objects[0]["title"]
|
|
|
|
value_id = nested_objects[0]["title"]
|
|
|
|
value_type = "[%s]" % value_id
|
|
|
|
value_type = "[%s]" % value_id
|
|
|
@ -221,22 +224,24 @@ def get_json_schema_object_fields(obj, enforce_title=False):
|
|
|
|
if isinstance(value_type, list):
|
|
|
|
if isinstance(value_type, list):
|
|
|
|
value_type = " or ".join(value_type)
|
|
|
|
value_type = " or ".join(value_type)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if required and mark_required:
|
|
|
|
|
|
|
|
desc = "**Required.** " + desc
|
|
|
|
fields["rows"].append({
|
|
|
|
fields["rows"].append({
|
|
|
|
"key": key_name,
|
|
|
|
"key": key_name,
|
|
|
|
"type": value_type,
|
|
|
|
"type": value_type,
|
|
|
|
"id": value_id,
|
|
|
|
"id": value_id,
|
|
|
|
"required": required,
|
|
|
|
"required": required,
|
|
|
|
"desc": desc,
|
|
|
|
"desc": desc,
|
|
|
|
"req_str": "**Required.** " if required else ""
|
|
|
|
|
|
|
|
})
|
|
|
|
})
|
|
|
|
logger.debug("Done property %s" % key_name)
|
|
|
|
logger.debug("Done property %s" % key_name)
|
|
|
|
|
|
|
|
|
|
|
|
return tables
|
|
|
|
return tables
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_tables_for_schema(schema):
|
|
|
|
def get_tables_for_schema(schema, mark_required=True):
|
|
|
|
schema = inherit_parents(schema)
|
|
|
|
schema = inherit_parents(schema)
|
|
|
|
tables = get_json_schema_object_fields(schema)
|
|
|
|
tables = get_json_schema_object_fields(schema,
|
|
|
|
|
|
|
|
mark_required=mark_required)
|
|
|
|
|
|
|
|
|
|
|
|
# the result may contain duplicates, if objects are referred to more than
|
|
|
|
# the result may contain duplicates, if objects are referred to more than
|
|
|
|
# once. Filter them out.
|
|
|
|
# once. Filter them out.
|
|
|
@ -276,7 +281,8 @@ class MatrixUnits(Units):
|
|
|
|
"path": full_path.strip(),
|
|
|
|
"path": full_path.strip(),
|
|
|
|
"requires_auth": "security" in single_api,
|
|
|
|
"requires_auth": "security" in single_api,
|
|
|
|
"rate_limited": 429 in single_api.get("responses", {}),
|
|
|
|
"rate_limited": 429 in single_api.get("responses", {}),
|
|
|
|
"req_params": [],
|
|
|
|
"req_param_by_loc": {},
|
|
|
|
|
|
|
|
"req_body_tables": [],
|
|
|
|
"res_tables": [],
|
|
|
|
"res_tables": [],
|
|
|
|
"example": {
|
|
|
|
"example": {
|
|
|
|
"req": "",
|
|
|
|
"req": "",
|
|
|
@ -286,6 +292,13 @@ class MatrixUnits(Units):
|
|
|
|
}
|
|
|
|
}
|
|
|
|
self.log(" ------- Endpoint: %s %s ------- " % (method, path))
|
|
|
|
self.log(" ------- Endpoint: %s %s ------- " % (method, path))
|
|
|
|
for param in single_api.get("parameters", []):
|
|
|
|
for param in single_api.get("parameters", []):
|
|
|
|
|
|
|
|
param_loc = param["in"]
|
|
|
|
|
|
|
|
if param_loc == "body":
|
|
|
|
|
|
|
|
self._handle_body_param(param, endpoint)
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
param_name = param["name"]
|
|
|
|
|
|
|
|
|
|
|
|
# description
|
|
|
|
# description
|
|
|
|
desc = param.get("description", "")
|
|
|
|
desc = param.get("description", "")
|
|
|
|
if param.get("required"):
|
|
|
|
if param.get("required"):
|
|
|
@ -300,114 +313,12 @@ class MatrixUnits(Units):
|
|
|
|
" One of: %s" % json.dumps(param.get("enum"))
|
|
|
|
" One of: %s" % json.dumps(param.get("enum"))
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
refType = Units.prop(param, "schema/$ref/") # Error,Event
|
|
|
|
endpoint["req_param_by_loc"].setdefault(param_loc, []).append({
|
|
|
|
schemaFmt = Units.prop(param, "schema/format") # bytes e.g. uploads
|
|
|
|
"key": param_name,
|
|
|
|
if not val_type and refType:
|
|
|
|
|
|
|
|
val_type = refType # TODO: Resolve to human-readable.
|
|
|
|
|
|
|
|
if not val_type and schemaFmt:
|
|
|
|
|
|
|
|
val_type = schemaFmt
|
|
|
|
|
|
|
|
# handle top-level strings/bools
|
|
|
|
|
|
|
|
if not val_type and Units.prop(param, "schema/type") == "string":
|
|
|
|
|
|
|
|
val_type = "string"
|
|
|
|
|
|
|
|
if not val_type and Units.prop(param, "schema/type") == "boolean":
|
|
|
|
|
|
|
|
val_type = "boolean"
|
|
|
|
|
|
|
|
if val_type:
|
|
|
|
|
|
|
|
endpoint["req_params"].append({
|
|
|
|
|
|
|
|
"key": param["name"],
|
|
|
|
|
|
|
|
"loc": param["in"],
|
|
|
|
|
|
|
|
"type": val_type,
|
|
|
|
"type": val_type,
|
|
|
|
"desc": desc
|
|
|
|
"desc": desc
|
|
|
|
})
|
|
|
|
})
|
|
|
|
continue
|
|
|
|
|
|
|
|
# If we're here, either the param has no value or it is an
|
|
|
|
|
|
|
|
# object which we haven't $reffed (so probably just a json
|
|
|
|
|
|
|
|
# object with some keys; we'll add entries f.e one)
|
|
|
|
|
|
|
|
if "schema" not in param:
|
|
|
|
|
|
|
|
raise Exception(
|
|
|
|
|
|
|
|
("API endpoint group=%s path=%s method=%s param=%s"+
|
|
|
|
|
|
|
|
" has no valid parameter value.") % (
|
|
|
|
|
|
|
|
group_name, path, method, param
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
if Units.prop(param, "schema/type") != "object":
|
|
|
|
|
|
|
|
raise Exception(
|
|
|
|
|
|
|
|
("API endpoint group=%s path=%s method=%s defines a"+
|
|
|
|
|
|
|
|
" param with a schema which isn't an object. Array?")
|
|
|
|
|
|
|
|
% (group_name, path, method)
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
# loop top-level json keys
|
|
|
|
|
|
|
|
json_body = Units.prop(param, "schema/properties")
|
|
|
|
|
|
|
|
required_params = []
|
|
|
|
|
|
|
|
if Units.prop(param, "schema/required"):
|
|
|
|
|
|
|
|
required_params = Units.prop(param, "schema/required")
|
|
|
|
|
|
|
|
for key in json_body:
|
|
|
|
|
|
|
|
req_obj = json_body[key]
|
|
|
|
|
|
|
|
pdesc = req_obj["description"]
|
|
|
|
|
|
|
|
if key in required_params:
|
|
|
|
|
|
|
|
pdesc = "**Required.** " + pdesc
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
is_array = req_obj["type"] == "array"
|
|
|
|
|
|
|
|
is_array_of_objects = (
|
|
|
|
|
|
|
|
is_array and req_obj["items"]["type"] == "object"
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
endpoint["req_params"].append({
|
|
|
|
|
|
|
|
"key": key,
|
|
|
|
|
|
|
|
"loc": "JSON body",
|
|
|
|
|
|
|
|
"type": (
|
|
|
|
|
|
|
|
req_obj["type"] if not is_array else
|
|
|
|
|
|
|
|
"array[%s]" % req_obj["items"]["type"]
|
|
|
|
|
|
|
|
),
|
|
|
|
|
|
|
|
"desc": pdesc
|
|
|
|
|
|
|
|
})
|
|
|
|
|
|
|
|
if not is_array_of_objects and req_obj["type"] == "array":
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
# Put in request.dot.notation for nested keys
|
|
|
|
|
|
|
|
if req_obj["type"] in ["object", "array"]:
|
|
|
|
|
|
|
|
if is_array_of_objects:
|
|
|
|
|
|
|
|
req_obj = req_obj["items"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
req_tables = get_tables_for_schema(req_obj)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if req_tables > 1:
|
|
|
|
|
|
|
|
for table in req_tables[1:]:
|
|
|
|
|
|
|
|
nested_key_name = {
|
|
|
|
|
|
|
|
"key": s["key"]
|
|
|
|
|
|
|
|
for rtable in req_tables
|
|
|
|
|
|
|
|
for s in rtable["rows"]
|
|
|
|
|
|
|
|
if s["id"] == table["title"]
|
|
|
|
|
|
|
|
}.get("key", None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if nested_key_name is None:
|
|
|
|
|
|
|
|
raise Exception("Failed to find table for %r" % (table["title"],))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for row in table["rows"]:
|
|
|
|
|
|
|
|
row["key"] = "%s.%s" % (nested_key_name, row["key"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
key_sep = "[0]." if is_array else "."
|
|
|
|
|
|
|
|
for table in req_tables:
|
|
|
|
|
|
|
|
if table.get("no-table"):
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
for row in table["rows"]:
|
|
|
|
|
|
|
|
nested_key = key + key_sep + row["key"]
|
|
|
|
|
|
|
|
endpoint["req_params"].append({
|
|
|
|
|
|
|
|
"key": nested_key,
|
|
|
|
|
|
|
|
"loc": "JSON body",
|
|
|
|
|
|
|
|
"type": row["type"],
|
|
|
|
|
|
|
|
"desc": row["req_str"] + row["desc"]
|
|
|
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# endfor[param]
|
|
|
|
# endfor[param]
|
|
|
|
for row in endpoint["req_params"]:
|
|
|
|
|
|
|
|
self.log("Request parameter: %s" % row)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# group params by location to ease templating
|
|
|
|
|
|
|
|
endpoint["req_param_by_loc"] = {
|
|
|
|
|
|
|
|
# path: [...], query: [...], body: [...]
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
for p in endpoint["req_params"]:
|
|
|
|
|
|
|
|
if p["loc"] not in endpoint["req_param_by_loc"]:
|
|
|
|
|
|
|
|
endpoint["req_param_by_loc"][p["loc"]] = []
|
|
|
|
|
|
|
|
endpoint["req_param_by_loc"][p["loc"]].append(p)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
good_response = None
|
|
|
|
good_response = None
|
|
|
|
for code, res in single_api.get("responses", {}).items():
|
|
|
|
for code, res in single_api.get("responses", {}).items():
|
|
|
@ -494,7 +405,9 @@ class MatrixUnits(Units):
|
|
|
|
elif res_type and Units.prop(good_response, "schema/properties"):
|
|
|
|
elif res_type and Units.prop(good_response, "schema/properties"):
|
|
|
|
# response is an object:
|
|
|
|
# response is an object:
|
|
|
|
schema = good_response["schema"]
|
|
|
|
schema = good_response["schema"]
|
|
|
|
res_tables = get_tables_for_schema(schema)
|
|
|
|
res_tables = get_tables_for_schema(schema,
|
|
|
|
|
|
|
|
mark_required=False,
|
|
|
|
|
|
|
|
)
|
|
|
|
endpoint["res_tables"].extend(res_tables)
|
|
|
|
endpoint["res_tables"].extend(res_tables)
|
|
|
|
elif res_type and Units.prop(good_response, "schema/items"):
|
|
|
|
elif res_type and Units.prop(good_response, "schema/items"):
|
|
|
|
# response is an array:
|
|
|
|
# response is an array:
|
|
|
@ -544,6 +457,32 @@ class MatrixUnits(Units):
|
|
|
|
"endpoints": endpoints,
|
|
|
|
"endpoints": endpoints,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _handle_body_param(self, param, endpoint_data):
|
|
|
|
|
|
|
|
"""Update endpoint_data object with the details of the body param
|
|
|
|
|
|
|
|
:param string filepath path to the yaml
|
|
|
|
|
|
|
|
:param dict param the parameter data from the yaml
|
|
|
|
|
|
|
|
:param dict endpoint_data dictionary of endpoint data to be updated
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
req_body_tables = get_tables_for_schema(param["schema"])
|
|
|
|
|
|
|
|
except Exception, e:
|
|
|
|
|
|
|
|
logger.warning("Error decoding body of API endpoint %s %s: %s",
|
|
|
|
|
|
|
|
endpoint_data["method"], endpoint_data["path"],
|
|
|
|
|
|
|
|
e.args[0])
|
|
|
|
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# put the top-level parameters into 'req_param_by_loc', and the others
|
|
|
|
|
|
|
|
# into 'req_body_tables'
|
|
|
|
|
|
|
|
body_params = endpoint_data['req_param_by_loc'].setdefault("JSON body",[])
|
|
|
|
|
|
|
|
body_params.extend(req_body_tables[0]["rows"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
body_tables = req_body_tables[1:]
|
|
|
|
|
|
|
|
# TODO: remove this when PR #255 has landed
|
|
|
|
|
|
|
|
body_tables = (t for t in body_tables if not t.get('no-table'))
|
|
|
|
|
|
|
|
endpoint_data['req_body_tables'].extend(body_tables)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def load_swagger_apis(self):
|
|
|
|
def load_swagger_apis(self):
|
|
|
|
apis = {}
|
|
|
|
apis = {}
|
|
|
|
for path in HTTP_APIS:
|
|
|
|
for path in HTTP_APIS:
|
|
|
|