diff --git a/api/client-server/definitions/event_filter.yaml b/api/client-server/definitions/event_filter.yaml index 8bedacb2..df872af4 100644 --- a/api/client-server/definitions/event_filter.yaml +++ b/api/client-server/definitions/event_filter.yaml @@ -1,3 +1,4 @@ +title: Filter properties: limit: description: The maximum number of events to return. diff --git a/api/client-server/definitions/room_event_filter.yaml b/api/client-server/definitions/room_event_filter.yaml index 0ad1ae33..afd407de 100644 --- a/api/client-server/definitions/room_event_filter.yaml +++ b/api/client-server/definitions/room_event_filter.yaml @@ -1,5 +1,6 @@ allOf: - $ref: event_filter.yaml +title: RoomEventFilter properties: not_rooms: description: A list of room IDs to exclude. If this list is absent then no rooms diff --git a/api/client-server/definitions/sync_filter.yaml b/api/client-server/definitions/sync_filter.yaml index 09307c99..b3d1cd84 100644 --- a/api/client-server/definitions/sync_filter.yaml +++ b/api/client-server/definitions/sync_filter.yaml @@ -25,6 +25,8 @@ properties: - $ref: event_filter.yaml description: The user account data that isn't associated with rooms to include. room: + title: RoomFilter + description: Filters to be applied to room data. properties: not_rooms: description: A list of room IDs to exclude. If this list is absent then no rooms diff --git a/api/client-server/event_context.yaml b/api/client-server/event_context.yaml index da2be90f..25e5ecf6 100644 --- a/api/client-server/event_context.yaml +++ b/api/client-server/event_context.yaml @@ -6,7 +6,7 @@ host: localhost:8008 schemes: - https - http -basePath: /_matrix/client/api/%CLIENT_MAJOR_VERSION% +basePath: /_matrix/client/%CLIENT_MAJOR_VERSION% consumes: - application/json produces: diff --git a/api/client-server/guest_events.yaml b/api/client-server/guest_events.yaml index 3dce4140..146a5335 100644 --- a/api/client-server/guest_events.yaml +++ b/api/client-server/guest_events.yaml @@ -28,6 +28,10 @@ paths: This API is the same as the non-guest /events endpoint, but can be called by guest users. + + Note that the non-guest ``/events`` endpoint has been deprecated. This + API will also be deprecated at some point, but its replacement is not + yet known. security: - accessToken: [] parameters: diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index f1c8f10c..51832dc8 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -1,5 +1,29 @@ -r0 -=== +r0.0.1 +====== + +This release includes the following changes since r0.0.0: + +- API changes: + - Added new ``/versions`` API + - ``/createRoom`` takes an optional ``invite_3pid`` parameter + - ``/publicRooms`` returns an ``avatar_url`` result +- The following APIs are now deprecated: + - ``/initialSync`` + - ``/events`` + - ``/events/:eventId`` + - ``/rooms/:roomId/initialSync`` +- Spec clarifications + - Document inter-version compatibility + - Document the ``next_batch`` parameter on ``/search`` + - Document the membership states on ``m.room.member`` events + - Minor clarifications/corrections to: + - Guest access module + - Search module + - ``/login`` API + - ``/rooms/:roomId/send/:eventType/:txnId`` API + +r0.0.0 +====== This is the first release of the client-server specification. It is largely a dump of what has currently been implemented, and there are several inconsistencies. diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 88ca6bdb..dbf84fc8 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -151,6 +151,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): def build_spec(target, out_filename): + log("Building templated file %s" % out_filename) with open(out_filename, "wb") as outfile: for file_info in target["files"]: section = get_rst( @@ -174,6 +175,7 @@ This function replaces these relative titles with actual title styles from the array in targets.yaml. """ def fix_relative_titles(target, filename, out_filename): + log("Fix relative titles, %s -> %s" % (filename, out_filename)) title_styles = target["title_styles"] relative_title_chars = [ target["relative_title_styles"]["subtitle"], @@ -226,6 +228,7 @@ def fix_relative_titles(target, filename, out_filename): def rst2html(i, o): + log("rst2html %s -> %s" % (i, o)) with open(i, "r") as in_file: with open(o, "w") as out_file: publish_file( @@ -239,6 +242,8 @@ def rst2html(i, o): def addAnchors(path): + log("add anchors %s" % path) + with open(path, "r") as f: lines = f.readlines() @@ -250,34 +255,27 @@ def addAnchors(path): f.write(line + "\n") -def run_through_template(input, set_verbose, substitutions): - tmpfile = './tmp/output' - try: - with open(tmpfile, 'w') as out: - args = [ - 'python', 'build.py', - "-i", "matrix_templates", - "-o", "../scripts/tmp", - "../scripts/"+input - ] - for k, v in substitutions.items(): - args.append("--substitution=%s=%s" % (k, v)) - - if set_verbose: - args.insert(2, "-v") - log("EXEC: %s" % " ".join(args)) - log(" ==== build.py output ==== ") - print subprocess.check_output( - args, - stderr=out, - cwd="../templating" - ) - except subprocess.CalledProcessError as e: - print e.output - with open(tmpfile, 'r') as f: - sys.stderr.write(f.read() + "\n") - raise +def run_through_template(input_files, set_verbose, substitutions): + args = [ + 'python', 'build.py', + "-o", "../scripts/tmp", + "-i", "matrix_templates", + ] + for k, v in substitutions.items(): + args.append("--substitution=%s=%s" % (k, v)) + + if set_verbose: + args.insert(2, "-v") + + args.extend("../scripts/"+f for f in input_files) + + log("EXEC: %s" % " ".join(args)) + log(" ==== build.py output ==== ") + subprocess.check_call( + args, + cwd="../templating" + ) def get_build_targets(targets_listing): with open(targets_listing, "r") as targ_file: @@ -401,16 +399,27 @@ def main(requested_target_name, keep_intermediates, substitutions): targets = [requested_target_name] if requested_target_name == "all": - targets = get_build_targets("../specification/targets.yaml") + targets = get_build_targets("../specification/targets.yaml") + ["howtos"] + + templated_files = [] + for target_name in targets: + templated_file = "tmp/templated_%s.rst" % (target_name,) + + if target_name == "howtos": + shutil.copy("../supporting-docs/howtos/client-server.rst", templated_file) + else: + target = get_build_target("../specification/targets.yaml", target_name) + build_spec(target=target, out_filename=templated_file) + templated_files.append(templated_file) + + # we do all the templating at once, because it's slow + run_through_template(templated_files, VERBOSE, substitutions) for target_name in targets: templated_file = "tmp/templated_%s.rst" % (target_name,) rst_file = "tmp/spec_%s.rst" % (target_name,) html_file = "gen/%s.html" % (target_name,) - target = get_build_target("../specification/targets.yaml", target_name) - build_spec(target=target, out_filename=templated_file) - run_through_template(templated_file, VERBOSE, substitutions) fix_relative_titles( target=target, filename=templated_file, out_filename=rst_file, @@ -418,11 +427,6 @@ def main(requested_target_name, keep_intermediates, substitutions): rst2html(rst_file, html_file) addAnchors(html_file) - if requested_target_name == "all": - shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") - run_through_template("tmp/howto.rst", False, substitutions) # too spammy to mark -v on this - rst2html("tmp/howto.rst", "gen/howtos.html") - if not keep_intermediates: cleanup_env() diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 0429ec69..1f442748 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -233,7 +233,7 @@ including the AS token on a ``/register`` request, along with a login type of Content: { type: "m.login.application_service", - username: "" + user: "" } Application services which attempt to create users or aliases *outside* of diff --git a/specification/modules/guest_access.rst b/specification/modules/guest_access.rst index 60f1eec1..1325abe8 100644 --- a/specification/modules/guest_access.rst +++ b/specification/modules/guest_access.rst @@ -65,6 +65,11 @@ They will only return events which happened while the room state had the value ``world_readable``. Guest clients do not need to join rooms in order to receive events for them. +The intention is that guest users will call ``/events`` once per room in +parallel for rooms they wish to view without joining. For rooms they wish to +join, they will call ``/join`` and receive events by calling ``/sync`` as +non-guest users do. + Server behaviour ---------------- Servers are required to only return events to guest accounts for rooms where diff --git a/templating/build.py b/templating/build.py index d677d550..4787b7b1 100755 --- a/templating/build.py +++ b/templating/build.py @@ -57,7 +57,7 @@ def check_unaccessed(name, store): log("Found %s unused %s keys." % (len(unaccessed_keys), name)) log(unaccessed_keys) -def main(input_module, file_stream=None, out_dir=None, verbose=False, substitutions={}): +def main(input_module, files=None, out_dir=None, verbose=False, substitutions={}): if out_dir and not os.path.exists(out_dir): os.makedirs(out_dir) @@ -138,7 +138,7 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti sections = in_mod.exports["sections"](env, units, debug=verbose).get_sections() # print out valid section keys if no file supplied - if not file_stream: + if not files: print "\nValid template variables:" for key in sections.keys(): sec_text = "" if (len(sections[key]) > 75) else ( @@ -152,8 +152,19 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti return # check the input files and substitute in sections where required - log("Parsing input template: %s" % file_stream.name) - temp_str = file_stream.read().decode("utf-8") + for input_filename in files: + output_filename = os.path.join(out_dir, + os.path.basename(input_filename)) + process_file(env, sections, input_filename, output_filename) + + check_unaccessed("units", units) + +def process_file(env, sections, filename, output_filename): + log("Parsing input template: %s" % filename) + + with open(filename, "r") as file_stream: + temp_str = file_stream.read().decode("utf-8") + # do sanity checking on the template to make sure they aren't reffing things # which will never be replaced with a section. ast = env.parse(temp_str) @@ -166,7 +177,6 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti ) # process the template temp = Template(temp_str) - log("Creating output for: %s" % file_stream.name) output = create_from_template(temp, sections) # Do these substitutions outside of the ordinary templating system because @@ -174,12 +184,11 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False, substituti # generate the templates, not just the top-level sections. for old, new in substitutions.items(): output = output.replace(old, new) - with open( - os.path.join(out_dir, os.path.basename(file_stream.name)), "w" - ) as f: + + with open(output_filename, "w") as f: f.write(output.encode("utf-8")) - log("Output file for: %s" % file_stream.name) - check_unaccessed("units", units) + log("Output file for: %s" % output_filename) + def log(line): print "batesian: %s" % line @@ -191,8 +200,8 @@ if __name__ == '__main__': "list of possible template variables, add --show-template-vars." ) parser.add_argument( - "file", nargs="?", type=FileType('r'), - help="The input file to process. This will be passed through Jinja "+ + "files", nargs="+", + help="The input files to process. These will be passed through Jinja "+ "then output under the same name to the output directory." ) parser.add_argument( @@ -234,11 +243,6 @@ if __name__ == '__main__': main(args.input, verbose=args.verbose) sys.exit(0) - if not args.file: - log("No file supplied.") - parser.print_help() - sys.exit(1) - substitutions = {} for substitution in args.substitution: parts = substitution.split("=", 1) @@ -247,6 +251,6 @@ if __name__ == '__main__': substitutions[parts[0]] = parts[1] main( - args.input, file_stream=args.file, out_dir=args.out_directory, + args.input, files=args.files, out_dir=args.out_directory, substitutions=substitutions, verbose=args.verbose ) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 8bf6ff74..2d225adb 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -62,7 +62,7 @@ def inherit_parents(obj): # iterate through the parents first, and then overwrite with the settings # from the child. for p in map(inherit_parents, parents) + [obj]: - for key in ('title', 'type', 'required'): + for key in ('title', 'type', 'required', 'description'): if p.get(key): result[key] = p[key] @@ -73,7 +73,7 @@ def inherit_parents(obj): return result -def get_json_schema_object_fields(obj, enforce_title=False, include_parents=False, +def get_json_schema_object_fields(obj, enforce_title=False, mark_required=True): # Algorithm: # f.e. property => add field info (if field is object then recurse) @@ -82,8 +82,6 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals "get_json_schema_object_fields: Object %s isn't an object." % obj ) - obj = inherit_parents(obj) - logger.debug("Processing object with title '%s'", obj.get("title")) if enforce_title and not obj.get("title"): @@ -93,6 +91,8 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals additionalProps = obj.get("additionalProperties") if additionalProps: + additionalProps = inherit_parents(additionalProps) + # not "really" an object, just a KV store logger.debug("%s is a pseudo-object", obj.get("title")) @@ -103,7 +103,6 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals nested_objects = get_json_schema_object_fields( additionalProps, enforce_title=True, - include_parents=include_parents, ) value_type = nested_objects[0]["title"] tables = [x for x in nested_objects if not x.get("no-table")] @@ -154,10 +153,12 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals for key_name in props: logger.debug("Processing property %s.%s", obj.get('title'), key_name) + prop = inherit_parents(props[key_name]) + value_type = None required = key_name in required_keys - desc = props[key_name].get("description", "") - prop_type = props[key_name].get('type') + desc = prop.get("description", "") + prop_type = prop.get('type') if prop_type is None: raise KeyError("Property '%s' of object '%s' missing 'type' field" @@ -166,9 +167,8 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals if prop_type == "object": nested_objects = get_json_schema_object_fields( - props[key_name], + prop, enforce_title=True, - include_parents=include_parents, mark_required=mark_required, ) value_type = nested_objects[0]["title"] @@ -176,24 +176,24 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals tables += [x for x in nested_objects if not x.get("no-table")] elif prop_type == "array": + items = inherit_parents(prop["items"]) # if the items of the array are objects then recurse - if props[key_name]["items"]["type"] == "object": + if items["type"] == "object": nested_objects = get_json_schema_object_fields( - props[key_name]["items"], + items, enforce_title=True, - include_parents=include_parents, mark_required=mark_required, ) value_id = nested_objects[0]["title"] value_type = "[%s]" % value_id tables += nested_objects else: - value_type = props[key_name]["items"]["type"] + value_type = items["type"] if isinstance(value_type, list): value_type = " or ".join(value_type) value_id = value_type value_type = "[%s]" % value_type - array_enums = props[key_name]["items"].get("enum") + array_enums = items.get("enum") if array_enums: if len(array_enums) > 1: value_type = "[enum]" @@ -207,19 +207,19 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals else: value_type = prop_type value_id = prop_type - if props[key_name].get("enum"): - if len(props[key_name].get("enum")) > 1: + if prop.get("enum"): + if len(prop["enum"]) > 1: value_type = "enum" if desc: desc += " " desc += ( - "One of: %s" % json.dumps(props[key_name]["enum"]) + "One of: %s" % json.dumps(prop["enum"]) ) else: if desc: desc += " " desc += ( - "Must be '%s'." % props[key_name]["enum"][0] + "Must be '%s'." % prop["enum"][0] ) if isinstance(value_type, list): value_type = " or ".join(value_type) @@ -238,12 +238,9 @@ def get_json_schema_object_fields(obj, enforce_title=False, include_parents=Fals return tables -def get_tables_for_schema(path, schema, include_parents=False, - mark_required=True): - resolved_schema = resolve_references(path, schema) - tables = get_json_schema_object_fields( - resolved_schema, - include_parents=include_parents, +def get_tables_for_schema(schema, mark_required=True): + schema = inherit_parents(schema) + tables = get_json_schema_object_fields(schema, mark_required=mark_required) # the result may contain duplicates, if objects are referred to more than @@ -255,6 +252,9 @@ def get_tables_for_schema(path, schema, include_parents=False, titles = set() filtered = [] for table in reversed(tables): + if table.get("no-table"): + continue + if table.get("title") in titles: continue @@ -267,7 +267,7 @@ def get_tables_for_schema(path, schema, include_parents=False, class MatrixUnits(Units): - def _load_swagger_meta(self, filepath, api, group_name): + def _load_swagger_meta(self, api, group_name): endpoints = [] for path in api["paths"]: for method in api["paths"][path]: @@ -294,7 +294,7 @@ class MatrixUnits(Units): for param in single_api.get("parameters", []): param_loc = param["in"] if param_loc == "body": - self._handle_body_param(filepath, param, endpoint) + self._handle_body_param(param, endpoint) continue param_name = param["name"] @@ -405,13 +405,10 @@ class MatrixUnits(Units): elif res_type and Units.prop(good_response, "schema/properties"): # response is an object: schema = good_response["schema"] - res_tables = get_tables_for_schema(filepath, schema, - include_parents=True, + res_tables = get_tables_for_schema(schema, mark_required=False, ) - for table in res_tables: - if "no-table" not in table: - endpoint["res_tables"].append(table) + endpoint["res_tables"].extend(res_tables) elif res_type and Units.prop(good_response, "schema/items"): # response is an array: # FIXME: Doesn't recurse at all. @@ -461,14 +458,14 @@ class MatrixUnits(Units): } - def _handle_body_param(self, filepath, param, endpoint_data): + def _handle_body_param(self, param, endpoint_data): """Update endpoint_data object with the details of the body param :param string filepath path to the yaml :param dict param the parameter data from the yaml :param dict endpoint_data dictionary of endpoint data to be updated """ try: - req_body_tables = get_tables_for_schema(filepath, param["schema"]) + req_body_tables = get_tables_for_schema(param["schema"]) except Exception, e: logger.warning("Error decoding body of API endpoint %s %s: %s", endpoint_data["method"], endpoint_data["path"], @@ -500,7 +497,7 @@ class MatrixUnits(Units): api = yaml.load(f.read()) api = resolve_references(filepath, api) api["__meta"] = self._load_swagger_meta( - filepath, api, group_name + api, group_name ) apis[group_name] = api return apis @@ -602,6 +599,8 @@ class MatrixUnits(Units): elif json_schema.get("title"): schema["typeof"] = json_schema["title"] + json_schema = resolve_references(filepath, json_schema) + # add type schema["type"] = Units.prop( json_schema, "properties/type/enum" @@ -612,14 +611,14 @@ class MatrixUnits(Units): schema["desc"] = json_schema.get("description", "") # walk the object for field info - schema["content_fields"] = get_tables_for_schema(filepath, + schema["content_fields"] = get_tables_for_schema( Units.prop(json_schema, "properties/content") ) # This is horrible because we're special casing a key on m.room.member. # We need to do this because we want to document a non-content object. if schema["type"] == "m.room.member": - invite_room_state = get_tables_for_schema(filepath, + invite_room_state = get_tables_for_schema( json_schema["properties"]["invite_room_state"]["items"], ) schema["content_fields"].extend(invite_room_state)