Merge branch 'master' into rav/body_params_in_tables

Conflicts:
	templating/matrix_templates/units.py
pull/977/head
Richard van der Hoff 9 years ago
commit 7a244ff977

@ -1,3 +1,4 @@
title: Filter
properties: properties:
limit: limit:
description: The maximum number of events to return. description: The maximum number of events to return.

@ -1,5 +1,6 @@
allOf: allOf:
- $ref: event_filter.yaml - $ref: event_filter.yaml
title: RoomEventFilter
properties: properties:
not_rooms: not_rooms:
description: A list of room IDs to exclude. If this list is absent then no rooms description: A list of room IDs to exclude. If this list is absent then no rooms

@ -25,6 +25,8 @@ properties:
- $ref: event_filter.yaml - $ref: event_filter.yaml
description: The user account data that isn't associated with rooms to include. description: The user account data that isn't associated with rooms to include.
room: room:
title: RoomFilter
description: Filters to be applied to room data.
properties: properties:
not_rooms: not_rooms:
description: A list of room IDs to exclude. If this list is absent then no rooms description: A list of room IDs to exclude. If this list is absent then no rooms

@ -6,7 +6,7 @@ host: localhost:8008
schemes: schemes:
- https - https
- http - http
basePath: /_matrix/client/api/%CLIENT_MAJOR_VERSION% basePath: /_matrix/client/%CLIENT_MAJOR_VERSION%
consumes: consumes:
- application/json - application/json
produces: produces:

@ -28,6 +28,10 @@ paths:
This API is the same as the non-guest /events endpoint, but can be This API is the same as the non-guest /events endpoint, but can be
called by guest users. called by guest users.
Note that the non-guest ``/events`` endpoint has been deprecated. This
API will also be deprecated at some point, but its replacement is not
yet known.
security: security:
- accessToken: [] - accessToken: []
parameters: parameters:

@ -1,5 +1,29 @@
r0 r0.0.1
=== ======
This release includes the following changes since r0.0.0:
- API changes:
- Added new ``/versions`` API
- ``/createRoom`` takes an optional ``invite_3pid`` parameter
- ``/publicRooms`` returns an ``avatar_url`` result
- The following APIs are now deprecated:
- ``/initialSync``
- ``/events``
- ``/events/:eventId``
- ``/rooms/:roomId/initialSync``
- Spec clarifications
- Document inter-version compatibility
- Document the ``next_batch`` parameter on ``/search``
- Document the membership states on ``m.room.member`` events
- Minor clarifications/corrections to:
- Guest access module
- Search module
- ``/login`` API
- ``/rooms/:roomId/send/:eventType/:txnId`` API
r0.0.0
======
This is the first release of the client-server specification. It is largely a dump of what has currently been implemented, and there are several inconsistencies. This is the first release of the client-server specification. It is largely a dump of what has currently been implemented, and there are several inconsistencies.

@ -151,6 +151,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
def build_spec(target, out_filename): def build_spec(target, out_filename):
log("Building templated file %s" % out_filename)
with open(out_filename, "wb") as outfile: with open(out_filename, "wb") as outfile:
for file_info in target["files"]: for file_info in target["files"]:
section = get_rst( section = get_rst(
@ -174,6 +175,7 @@ This function replaces these relative titles with actual title styles from the
array in targets.yaml. array in targets.yaml.
""" """
def fix_relative_titles(target, filename, out_filename): def fix_relative_titles(target, filename, out_filename):
log("Fix relative titles, %s -> %s" % (filename, out_filename))
title_styles = target["title_styles"] title_styles = target["title_styles"]
relative_title_chars = [ relative_title_chars = [
target["relative_title_styles"]["subtitle"], target["relative_title_styles"]["subtitle"],
@ -226,6 +228,7 @@ def fix_relative_titles(target, filename, out_filename):
def rst2html(i, o): def rst2html(i, o):
log("rst2html %s -> %s" % (i, o))
with open(i, "r") as in_file: with open(i, "r") as in_file:
with open(o, "w") as out_file: with open(o, "w") as out_file:
publish_file( publish_file(
@ -239,6 +242,8 @@ def rst2html(i, o):
def addAnchors(path): def addAnchors(path):
log("add anchors %s" % path)
with open(path, "r") as f: with open(path, "r") as f:
lines = f.readlines() lines = f.readlines()
@ -250,34 +255,27 @@ def addAnchors(path):
f.write(line + "\n") f.write(line + "\n")
def run_through_template(input, set_verbose, substitutions): def run_through_template(input_files, set_verbose, substitutions):
tmpfile = './tmp/output' args = [
try: 'python', 'build.py',
with open(tmpfile, 'w') as out: "-o", "../scripts/tmp",
args = [ "-i", "matrix_templates",
'python', 'build.py', ]
"-i", "matrix_templates",
"-o", "../scripts/tmp",
"../scripts/"+input
]
for k, v in substitutions.items():
args.append("--substitution=%s=%s" % (k, v))
if set_verbose:
args.insert(2, "-v")
log("EXEC: %s" % " ".join(args))
log(" ==== build.py output ==== ")
print subprocess.check_output(
args,
stderr=out,
cwd="../templating"
)
except subprocess.CalledProcessError as e:
print e.output
with open(tmpfile, 'r') as f:
sys.stderr.write(f.read() + "\n")
raise
for k, v in substitutions.items():
args.append("--substitution=%s=%s" % (k, v))
if set_verbose:
args.insert(2, "-v")
args.extend("../scripts/"+f for f in input_files)
log("EXEC: %s" % " ".join(args))
log(" ==== build.py output ==== ")
subprocess.check_call(
args,
cwd="../templating"
)
def get_build_targets(targets_listing): def get_build_targets(targets_listing):
with open(targets_listing, "r") as targ_file: with open(targets_listing, "r") as targ_file:
@ -401,16 +399,27 @@ def main(requested_target_name, keep_intermediates, substitutions):
targets = [requested_target_name] targets = [requested_target_name]
if requested_target_name == "all": if requested_target_name == "all":
targets = get_build_targets("../specification/targets.yaml") targets = get_build_targets("../specification/targets.yaml") + ["howtos"]
templated_files = []
for target_name in targets:
templated_file = "tmp/templated_%s.rst" % (target_name,)
if target_name == "howtos":
shutil.copy("../supporting-docs/howtos/client-server.rst", templated_file)
else:
target = get_build_target("../specification/targets.yaml", target_name)
build_spec(target=target, out_filename=templated_file)
templated_files.append(templated_file)
# we do all the templating at once, because it's slow
run_through_template(templated_files, VERBOSE, substitutions)
for target_name in targets: for target_name in targets:
templated_file = "tmp/templated_%s.rst" % (target_name,) templated_file = "tmp/templated_%s.rst" % (target_name,)
rst_file = "tmp/spec_%s.rst" % (target_name,) rst_file = "tmp/spec_%s.rst" % (target_name,)
html_file = "gen/%s.html" % (target_name,) html_file = "gen/%s.html" % (target_name,)
target = get_build_target("../specification/targets.yaml", target_name)
build_spec(target=target, out_filename=templated_file)
run_through_template(templated_file, VERBOSE, substitutions)
fix_relative_titles( fix_relative_titles(
target=target, filename=templated_file, target=target, filename=templated_file,
out_filename=rst_file, out_filename=rst_file,
@ -418,11 +427,6 @@ def main(requested_target_name, keep_intermediates, substitutions):
rst2html(rst_file, html_file) rst2html(rst_file, html_file)
addAnchors(html_file) addAnchors(html_file)
if requested_target_name == "all":
shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst")
run_through_template("tmp/howto.rst", False, substitutions) # too spammy to mark -v on this
rst2html("tmp/howto.rst", "gen/howtos.html")
if not keep_intermediates: if not keep_intermediates:
cleanup_env() cleanup_env()

@ -233,7 +233,7 @@ including the AS token on a ``/register`` request, along with a login type of
Content: Content:
{ {
type: "m.login.application_service", type: "m.login.application_service",
username: "<desired user localpart in AS namespace>" user: "<desired user localpart in AS namespace>"
} }
Application services which attempt to create users or aliases *outside* of Application services which attempt to create users or aliases *outside* of

@ -65,6 +65,11 @@ They will only return events which happened while the room state had the
value ``world_readable``. Guest clients do not need to join rooms in order to value ``world_readable``. Guest clients do not need to join rooms in order to
receive events for them. receive events for them.
The intention is that guest users will call ``/events`` once per room in
parallel for rooms they wish to view without joining. For rooms they wish to
join, they will call ``/join`` and receive events by calling ``/sync`` as
non-guest users do.
Server behaviour Server behaviour
---------------- ----------------
Servers are required to only return events to guest accounts for rooms where Servers are required to only return events to guest accounts for rooms where

@ -57,7 +57,7 @@ def check_unaccessed(name, store):
log("Found %s unused %s keys." % (len(unaccessed_keys), name)) log("Found %s unused %s keys." % (len(unaccessed_keys), name))
log(unaccessed_keys) log(unaccessed_keys)
def main(input_module, file_stream=None, out_dir=None, verbose=False, substitutions={}): def main(input_module, files=None, out_dir=None, verbose=False, substitutions={}):
if out_dir and not os.path.exists(out_dir): if out_dir and not os.path.exists(out_dir):
os.makedirs(out_dir) os.makedirs(out_dir)
@ -138,7 +138,7 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti
sections = in_mod.exports["sections"](env, units, debug=verbose).get_sections() sections = in_mod.exports["sections"](env, units, debug=verbose).get_sections()
# print out valid section keys if no file supplied # print out valid section keys if no file supplied
if not file_stream: if not files:
print "\nValid template variables:" print "\nValid template variables:"
for key in sections.keys(): for key in sections.keys():
sec_text = "" if (len(sections[key]) > 75) else ( sec_text = "" if (len(sections[key]) > 75) else (
@ -152,8 +152,19 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti
return return
# check the input files and substitute in sections where required # check the input files and substitute in sections where required
log("Parsing input template: %s" % file_stream.name) for input_filename in files:
temp_str = file_stream.read().decode("utf-8") output_filename = os.path.join(out_dir,
os.path.basename(input_filename))
process_file(env, sections, input_filename, output_filename)
check_unaccessed("units", units)
def process_file(env, sections, filename, output_filename):
log("Parsing input template: %s" % filename)
with open(filename, "r") as file_stream:
temp_str = file_stream.read().decode("utf-8")
# do sanity checking on the template to make sure they aren't reffing things # do sanity checking on the template to make sure they aren't reffing things
# which will never be replaced with a section. # which will never be replaced with a section.
ast = env.parse(temp_str) ast = env.parse(temp_str)
@ -166,7 +177,6 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti
) )
# process the template # process the template
temp = Template(temp_str) temp = Template(temp_str)
log("Creating output for: %s" % file_stream.name)
output = create_from_template(temp, sections) output = create_from_template(temp, sections)
# Do these substitutions outside of the ordinary templating system because # Do these substitutions outside of the ordinary templating system because
@ -174,12 +184,11 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti
# generate the templates, not just the top-level sections. # generate the templates, not just the top-level sections.
for old, new in substitutions.items(): for old, new in substitutions.items():
output = output.replace(old, new) output = output.replace(old, new)
with open(
os.path.join(out_dir, os.path.basename(file_stream.name)), "w" with open(output_filename, "w") as f:
) as f:
f.write(output.encode("utf-8")) f.write(output.encode("utf-8"))
log("Output file for: %s" % file_stream.name) log("Output file for: %s" % output_filename)
check_unaccessed("units", units)
def log(line): def log(line):
print "batesian: %s" % line print "batesian: %s" % line
@ -191,8 +200,8 @@ if __name__ == '__main__':
"list of possible template variables, add --show-template-vars." "list of possible template variables, add --show-template-vars."
) )
parser.add_argument( parser.add_argument(
"file", nargs="?", type=FileType('r'), "files", nargs="+",
help="The input file to process. This will be passed through Jinja "+ help="The input files to process. These will be passed through Jinja "+
"then output under the same name to the output directory." "then output under the same name to the output directory."
) )
parser.add_argument( parser.add_argument(
@ -234,11 +243,6 @@ if __name__ == '__main__':
main(args.input, verbose=args.verbose) main(args.input, verbose=args.verbose)
sys.exit(0) sys.exit(0)
if not args.file:
log("No file supplied.")
parser.print_help()
sys.exit(1)
substitutions = {} substitutions = {}
for substitution in args.substitution: for substitution in args.substitution:
parts = substitution.split("=", 1) parts = substitution.split("=", 1)
@ -247,6 +251,6 @@ if __name__ == '__main__':
substitutions[parts[0]] = parts[1] substitutions[parts[0]] = parts[1]
main( main(
args.input, file_stream=args.file, out_dir=args.out_directory, args.input, files=args.files, out_dir=args.out_directory,
substitutions=substitutions, verbose=args.verbose substitutions=substitutions, verbose=args.verbose
) )

@ -62,7 +62,7 @@ def inherit_parents(obj):
# iterate through the parents first, and then overwrite with the settings # iterate through the parents first, and then overwrite with the settings
# from the child. # from the child.
for p in map(inherit_parents, parents) + [obj]: for p in map(inherit_parents, parents) + [obj]:
for key in ('title', 'type', 'required'): for key in ('title', 'type', 'required', 'description'):
if p.get(key): if p.get(key):
result[key] = p[key] result[key] = p[key]
@ -73,7 +73,7 @@ def inherit_parents(obj):
return result return result
def get_json_schema_object_fields(obj, enforce_title=False, include_parents=False, def get_json_schema_object_fields(obj, enforce_title=False,
mark_required=True): mark_required=True):
# Algorithm: # Algorithm:
# f.e. property => add field info (if field is object then recurse) # f.e. property => add field info (if field is object then recurse)
@ -82,8 +82,6 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals
"get_json_schema_object_fields: Object %s isn't an object." % obj "get_json_schema_object_fields: Object %s isn't an object." % obj
) )
obj = inherit_parents(obj)
logger.debug("Processing object with title '%s'", obj.get("title")) logger.debug("Processing object with title '%s'", obj.get("title"))
if enforce_title and not obj.get("title"): if enforce_title and not obj.get("title"):
@ -93,6 +91,8 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals
additionalProps = obj.get("additionalProperties") additionalProps = obj.get("additionalProperties")
if additionalProps: if additionalProps:
additionalProps = inherit_parents(additionalProps)
# not "really" an object, just a KV store # not "really" an object, just a KV store
logger.debug("%s is a pseudo-object", obj.get("title")) logger.debug("%s is a pseudo-object", obj.get("title"))
@ -103,7 +103,6 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals
nested_objects = get_json_schema_object_fields( nested_objects = get_json_schema_object_fields(
additionalProps, additionalProps,
enforce_title=True, enforce_title=True,
include_parents=include_parents,
) )
value_type = nested_objects[0]["title"] value_type = nested_objects[0]["title"]
tables = [x for x in nested_objects if not x.get("no-table")] tables = [x for x in nested_objects if not x.get("no-table")]
@ -154,10 +153,12 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals
for key_name in props: for key_name in props:
logger.debug("Processing property %s.%s", obj.get('title'), key_name) logger.debug("Processing property %s.%s", obj.get('title'), key_name)
prop = inherit_parents(props[key_name])
value_type = None value_type = None
required = key_name in required_keys required = key_name in required_keys
desc = props[key_name].get("description", "") desc = prop.get("description", "")
prop_type = props[key_name].get('type') prop_type = prop.get('type')
if prop_type is None: if prop_type is None:
raise KeyError("Property '%s' of object '%s' missing 'type' field" raise KeyError("Property '%s' of object '%s' missing 'type' field"
@ -166,9 +167,8 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals
if prop_type == "object": if prop_type == "object":
nested_objects = get_json_schema_object_fields( nested_objects = get_json_schema_object_fields(
props[key_name], prop,
enforce_title=True, enforce_title=True,
include_parents=include_parents,
mark_required=mark_required, mark_required=mark_required,
) )
value_type = nested_objects[0]["title"] value_type = nested_objects[0]["title"]
@ -176,24 +176,24 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals
tables += [x for x in nested_objects if not x.get("no-table")] tables += [x for x in nested_objects if not x.get("no-table")]
elif prop_type == "array": elif prop_type == "array":
items = inherit_parents(prop["items"])
# if the items of the array are objects then recurse # if the items of the array are objects then recurse
if props[key_name]["items"]["type"] == "object": if items["type"] == "object":
nested_objects = get_json_schema_object_fields( nested_objects = get_json_schema_object_fields(
props[key_name]["items"], items,
enforce_title=True, enforce_title=True,
include_parents=include_parents,
mark_required=mark_required, mark_required=mark_required,
) )
value_id = nested_objects[0]["title"] value_id = nested_objects[0]["title"]
value_type = "[%s]" % value_id value_type = "[%s]" % value_id
tables += nested_objects tables += nested_objects
else: else:
value_type = props[key_name]["items"]["type"] value_type = items["type"]
if isinstance(value_type, list): if isinstance(value_type, list):
value_type = " or ".join(value_type) value_type = " or ".join(value_type)
value_id = value_type value_id = value_type
value_type = "[%s]" % value_type value_type = "[%s]" % value_type
array_enums = props[key_name]["items"].get("enum") array_enums = items.get("enum")
if array_enums: if array_enums:
if len(array_enums) > 1: if len(array_enums) > 1:
value_type = "[enum]" value_type = "[enum]"
@ -207,19 +207,19 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals
else: else:
value_type = prop_type value_type = prop_type
value_id = prop_type value_id = prop_type
if props[key_name].get("enum"): if prop.get("enum"):
if len(props[key_name].get("enum")) > 1: if len(prop["enum"]) > 1:
value_type = "enum" value_type = "enum"
if desc: if desc:
desc += " " desc += " "
desc += ( desc += (
"One of: %s" % json.dumps(props[key_name]["enum"]) "One of: %s" % json.dumps(prop["enum"])
) )
else: else:
if desc: if desc:
desc += " " desc += " "
desc += ( desc += (
"Must be '%s'." % props[key_name]["enum"][0] "Must be '%s'." % prop["enum"][0]
) )
if isinstance(value_type, list): if isinstance(value_type, list):
value_type = " or ".join(value_type) value_type = " or ".join(value_type)
@ -238,12 +238,9 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals
return tables return tables
def get_tables_for_schema(path, schema, include_parents=False, def get_tables_for_schema(schema, mark_required=True):
mark_required=True): schema = inherit_parents(schema)
resolved_schema = resolve_references(path, schema) tables = get_json_schema_object_fields(schema,
tables = get_json_schema_object_fields(
resolved_schema,
include_parents=include_parents,
mark_required=mark_required) mark_required=mark_required)
# the result may contain duplicates, if objects are referred to more than # the result may contain duplicates, if objects are referred to more than
@ -255,6 +252,9 @@ def get_tables_for_schema(path, schema, include_parents=False,
titles = set() titles = set()
filtered = [] filtered = []
for table in reversed(tables): for table in reversed(tables):
if table.get("no-table"):
continue
if table.get("title") in titles: if table.get("title") in titles:
continue continue
@ -267,7 +267,7 @@ def get_tables_for_schema(path, schema, include_parents=False,
class MatrixUnits(Units): class MatrixUnits(Units):
def _load_swagger_meta(self, filepath, api, group_name): def _load_swagger_meta(self, api, group_name):
endpoints = [] endpoints = []
for path in api["paths"]: for path in api["paths"]:
for method in api["paths"][path]: for method in api["paths"][path]:
@ -294,7 +294,7 @@ class MatrixUnits(Units):
for param in single_api.get("parameters", []): for param in single_api.get("parameters", []):
param_loc = param["in"] param_loc = param["in"]
if param_loc == "body": if param_loc == "body":
self._handle_body_param(filepath, param, endpoint) self._handle_body_param(param, endpoint)
continue continue
param_name = param["name"] param_name = param["name"]
@ -405,13 +405,10 @@ class MatrixUnits(Units):
elif res_type and Units.prop(good_response, "schema/properties"): elif res_type and Units.prop(good_response, "schema/properties"):
# response is an object: # response is an object:
schema = good_response["schema"] schema = good_response["schema"]
res_tables = get_tables_for_schema(filepath, schema, res_tables = get_tables_for_schema(schema,
include_parents=True,
mark_required=False, mark_required=False,
) )
for table in res_tables: endpoint["res_tables"].extend(res_tables)
if "no-table" not in table:
endpoint["res_tables"].append(table)
elif res_type and Units.prop(good_response, "schema/items"): elif res_type and Units.prop(good_response, "schema/items"):
# response is an array: # response is an array:
# FIXME: Doesn't recurse at all. # FIXME: Doesn't recurse at all.
@ -461,14 +458,14 @@ class MatrixUnits(Units):
} }
def _handle_body_param(self, filepath, param, endpoint_data): def _handle_body_param(self, param, endpoint_data):
"""Update endpoint_data object with the details of the body param """Update endpoint_data object with the details of the body param
:param string filepath path to the yaml :param string filepath path to the yaml
:param dict param the parameter data from the yaml :param dict param the parameter data from the yaml
:param dict endpoint_data dictionary of endpoint data to be updated :param dict endpoint_data dictionary of endpoint data to be updated
""" """
try: try:
req_body_tables = get_tables_for_schema(filepath, param["schema"]) req_body_tables = get_tables_for_schema(param["schema"])
except Exception, e: except Exception, e:
logger.warning("Error decoding body of API endpoint %s %s: %s", logger.warning("Error decoding body of API endpoint %s %s: %s",
endpoint_data["method"], endpoint_data["path"], endpoint_data["method"], endpoint_data["path"],
@ -500,7 +497,7 @@ class MatrixUnits(Units):
api = yaml.load(f.read()) api = yaml.load(f.read())
api = resolve_references(filepath, api) api = resolve_references(filepath, api)
api["__meta"] = self._load_swagger_meta( api["__meta"] = self._load_swagger_meta(
filepath, api, group_name api, group_name
) )
apis[group_name] = api apis[group_name] = api
return apis return apis
@ -602,6 +599,8 @@ class MatrixUnits(Units):
elif json_schema.get("title"): elif json_schema.get("title"):
schema["typeof"] = json_schema["title"] schema["typeof"] = json_schema["title"]
json_schema = resolve_references(filepath, json_schema)
# add type # add type
schema["type"] = Units.prop( schema["type"] = Units.prop(
json_schema, "properties/type/enum" json_schema, "properties/type/enum"
@ -612,14 +611,14 @@ class MatrixUnits(Units):
schema["desc"] = json_schema.get("description", "") schema["desc"] = json_schema.get("description", "")
# walk the object for field info # walk the object for field info
schema["content_fields"] = get_tables_for_schema(filepath, schema["content_fields"] = get_tables_for_schema(
Units.prop(json_schema, "properties/content") Units.prop(json_schema, "properties/content")
) )
# This is horrible because we're special casing a key on m.room.member. # This is horrible because we're special casing a key on m.room.member.
# We need to do this because we want to document a non-content object. # We need to do this because we want to document a non-content object.
if schema["type"] == "m.room.member": if schema["type"] == "m.room.member":
invite_room_state = get_tables_for_schema(filepath, invite_room_state = get_tables_for_schema(
json_schema["properties"]["invite_room_state"]["items"], json_schema["properties"]["invite_room_state"]["items"],
) )
schema["content_fields"].extend(invite_room_state) schema["content_fields"].extend(invite_room_state)

Loading…
Cancel
Save