Refactor a bunch of the logic in matrix_templates/units.py

Move 'inherit_parents' calls up to before get_json_schema_object_fields so that
things can inherit their types via allOf declarations. Also make it possible to
inherit descriptions.

Remove redundant 'include_parents' parameter, which was never used.

Move 'resolve_references' call out of get_tables_for_schema (it was redundant
sometimes and not others, and now it is clearer who has responsibility for it)

templating/matrix_templates/units.py

@@ -62,7 +62,7 @@ def inherit_parents(obj):
     # iterate through the parents first, and then overwrite with the settings
     # from the child.
     for p in map(inherit_parents, parents) + [obj]:
-        for key in ('title', 'type', 'required'):
+        for key in ('title', 'type', 'required', 'description'):
             if p.get(key):
                 result[key] = p[key]
 
@@ -73,7 +73,7 @@ def inherit_parents(obj):
     return result
 
 
-def get_json_schema_object_fields(obj, enforce_title=False, include_parents=False):
+def get_json_schema_object_fields(obj, enforce_title=False):
     # Algorithm:
     # f.e. property => add field info (if field is object then recurse)
     if obj.get("type") != "object":
@@ -81,8 +81,6 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals
             "get_json_schema_object_fields: Object %s isn't an object." % obj
         )
 
-    obj = inherit_parents(obj)
-
     logger.debug("Processing object with title '%s'", obj.get("title"))
 
     if enforce_title and not obj.get("title"):
@@ -92,6 +90,8 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals
 
     additionalProps = obj.get("additionalProperties")
     if additionalProps:
+        additionalProps = inherit_parents(additionalProps)
+
         # not "really" an object, just a KV store
         logger.debug("%s is a pseudo-object", obj.get("title"))
 
@@ -102,7 +102,6 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals
             nested_objects = get_json_schema_object_fields(
                 additionalProps,
                 enforce_title=True,
-                include_parents=include_parents,
             )
             value_type = nested_objects[0]["title"]
             tables = [x for x in nested_objects if not x.get("no-table")]
@@ -153,10 +152,12 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals
 
     for key_name in sorted(props):
         logger.debug("Processing property %s.%s", obj.get('title'), key_name)
+        prop = inherit_parents(props[key_name])
+
         value_type = None
         required = key_name in required_keys
-        desc = props[key_name].get("description", "")
+        desc = prop.get("description", "")
-        prop_type = props[key_name].get('type')
+        prop_type = prop.get('type')
 
         if prop_type is None:
             raise KeyError("Property '%s' of object '%s' missing 'type' field"
@@ -165,32 +166,31 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals
 
         if prop_type == "object":
             nested_objects = get_json_schema_object_fields(
-                props[key_name],
+                prop,
                 enforce_title=True,
-                include_parents=include_parents,
             )
             value_type = nested_objects[0]["title"]
             value_id = value_type
 
             tables += [x for x in nested_objects if not x.get("no-table")]
         elif prop_type == "array":
+            items = inherit_parents(prop["items"])
             # if the items of the array are objects then recurse
-            if props[key_name]["items"]["type"] == "object":
+            if items["type"] == "object":
                 nested_objects = get_json_schema_object_fields(
-                    props[key_name]["items"],
+                    items,
                     enforce_title=True,
-                    include_parents=include_parents,
                 )
                 value_id = nested_objects[0]["title"]
                 value_type = "[%s]" % value_id
                 tables += nested_objects
             else:
-                value_type = props[key_name]["items"]["type"]
+                value_type = items["type"]
                 if isinstance(value_type, list):
                     value_type = " or ".join(value_type)
                 value_id = value_type
                 value_type = "[%s]" % value_type
-                array_enums = props[key_name]["items"].get("enum")
+                array_enums = items.get("enum")
                 if array_enums:
                     if len(array_enums) > 1:
                         value_type = "[enum]"
@@ -204,19 +204,19 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals
         else:
             value_type = prop_type
             value_id = prop_type
-            if props[key_name].get("enum"):
+            if prop.get("enum"):
-                if len(props[key_name].get("enum")) > 1:
+                if len(prop["enum"]) > 1:
                     value_type = "enum"
                     if desc:
                         desc += " "
                     desc += (
-                        "One of: %s" % json.dumps(props[key_name]["enum"])
+                        "One of: %s" % json.dumps(prop["enum"])
                     )
                 else:
                     if desc:
                         desc += " "
                     desc += (
-                        "Must be '%s'." % props[key_name]["enum"][0]
+                        "Must be '%s'." % prop["enum"][0]
                     )
             if isinstance(value_type, list):
                 value_type = " or ".join(value_type)
@@ -234,11 +234,9 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals
     return tables
 
 
-def get_tables_for_schema(path, schema, include_parents=False):
+def get_tables_for_schema(schema):
-    resolved_schema = resolve_references(path, schema)
+    schema = inherit_parents(schema)
-    tables = get_json_schema_object_fields(resolved_schema,
+    tables = get_json_schema_object_fields(schema)
-        include_parents=include_parents,
-    )
 
     # the result may contain duplicates, if objects are referred to more than
     # once. Filter them out.
@@ -249,6 +247,9 @@ def get_tables_for_schema(path, schema, include_parents=False):
     titles = set()
     filtered = []
     for table in reversed(tables):
+        if table.get("no-table"):
+            continue
+
         if table.get("title") in titles:
             continue
 
@@ -261,7 +262,7 @@ def get_tables_for_schema(path, schema, include_parents=False):
 
 class MatrixUnits(Units):
 
-    def _load_swagger_meta(self, filepath, api, group_name):
+    def _load_swagger_meta(self, api, group_name):
         endpoints = []
         for path in api["paths"]:
             for method in api["paths"][path]:
@@ -494,12 +495,8 @@ class MatrixUnits(Units):
                     elif res_type and Units.prop(good_response, "schema/properties"):
                         # response is an object:
                         schema = good_response["schema"]
-                        res_tables = get_tables_for_schema(filepath, schema,
+                        res_tables = get_tables_for_schema(schema)
-                            include_parents=True,
+                        endpoint["res_tables"].extend(res_tables)
-                        )
-                        for table in res_tables:
-                            if "no-table" not in table:
-                                endpoint["res_tables"].append(table)
                     elif res_type and Units.prop(good_response, "schema/items"):
                         # response is an array:
                         # FIXME: Doesn't recurse at all.
@@ -562,7 +559,7 @@ class MatrixUnits(Units):
                     api = yaml.load(f.read())
                     api = resolve_references(filepath, api)
                     api["__meta"] = self._load_swagger_meta(
-                        filepath, api, group_name
+                        api, group_name
                     )
                     apis[group_name] = api
         return apis
@@ -657,11 +654,10 @@ class MatrixUnits(Units):
                     ROOM_EVENT: "Message Event",
                     STATE_EVENT: "State Event"
                 }
-                if type(json_schema.get("allOf")) == list:
+
-                    schema["typeof"] = base_defs.get(
+                json_schema = resolve_references(filepath, json_schema)
-                        json_schema["allOf"][0].get("$ref")
+
-                    )
+                if json_schema.get("title"):
-                elif json_schema.get("title"):
                     schema["typeof"] = json_schema["title"]
 
                 # add type
@@ -674,14 +670,14 @@ class MatrixUnits(Units):
                 schema["desc"] = json_schema.get("description", "")
 
                 # walk the object for field info
-                schema["content_fields"] = get_tables_for_schema(filepath,
+                schema["content_fields"] = get_tables_for_schema(
                     Units.prop(json_schema, "properties/content")
                 )
 
                 # This is horrible because we're special casing a key on m.room.member.
                 # We need to do this because we want to document a non-content object.
                 if schema["type"] == "m.room.member":
-                    invite_room_state = get_tables_for_schema(filepath,
+                    invite_room_state = get_tables_for_schema(
                         json_schema["properties"]["invite_room_state"]["items"],
                     )
                     schema["content_fields"].extend(invite_room_state)