Merge branch 'master' of github.com:matrix-org/matrix-doc

api/client-server/v1/definitions/push_rule.json

@@ -1,4 +1,5 @@
 {
+  "title": "PushRule",
   "type": "object", 
   "properties": {
     "default": {
@@ -17,4 +18,4 @@
       "type": "array"
     }
   }
-}
+}

api/client-server/v1/definitions/push_ruleset.json

@@ -1,60 +1,65 @@
 {
-  "type": "object", 
+  "type": "object",
   "properties": {
     "content": {
       "items": {
-        "type": "object", 
+        "type": "object",
+        "title": "PushRule",
         "allOf": [
           {
             "$ref": "push_rule.json"
           }
         ]
-      }, 
+      },
       "type": "array"
-    }, 
+    },
     "override": {
       "items": {
-        "type": "object", 
+        "type": "object",
+        "title": "PushRule",
         "allOf": [
           {
             "$ref": "push_rule.json"
           }
         ]
-      }, 
+      },
       "type": "array"
-    }, 
+    },
     "sender": {
       "items": {
-        "type": "object", 
+        "type": "object",
+        "title": "PushRule",
         "allOf": [
           {
             "$ref": "push_rule.json"
           }
         ]
-      }, 
+      },
       "type": "array"
-    }, 
+    },
     "underride": {
       "items": {
-        "type": "object", 
+        "type": "object",
+        "title": "PushRule",
         "allOf": [
           {
             "$ref": "push_rule.json"
           }
         ]
-      }, 
+      },
       "type": "array"
-    }, 
+    },
     "room": {
       "items": {
-        "type": "object", 
+        "type": "object",
+        "title": "PushRule",
         "allOf": [
           {
             "$ref": "push_rule.json"
           }
         ]
-      }, 
+      },
       "type": "array"
     }
   }
-}
+}

api/client-server/v2_alpha/definitions/event_batch.json

@@ -5,6 +5,7 @@
       "type": "array",
       "description": "List of events",
       "items": {
+        "title": "Event",
         "type": "object"
       }
     }

api/client-server/v2_alpha/sync.yaml

@@ -241,6 +241,9 @@ paths:
                           "type": "m.room.member",
                           "state_key": "@bob:example.com",
                           "content": {"membership": "join"},
+                          "unsigned": {
+                            "prev_content": {"membership": "invite"}
+                          },
                           "origin_server_ts": 1417731086795
                         },
                         "$74686972643033:example.com": {

event-schemas/schema/v1/core-event-schema/event.json

@@ -5,6 +5,7 @@
   "properties": {
     "content": {
       "type": "object",
+      "title": "EventContent",
       "description": "The fields in this object will vary depending on the type of event. When interacting with the REST API, this is the HTTP body."
     },
     "type": {

event-schemas/schema/v1/core-event-schema/state_event.json

@@ -11,6 +11,7 @@
       "description": "A unique key which defines the overwriting semantics for this piece of room state. This value is often a zero-length string. The presence of this key makes this event a State Event. The key MUST NOT start with '_'."
     },
     "prev_content": {
+      "title": "EventContent",
       "type": "object",
       "description": "Optional. The previous ``content`` for this event. If there is no previous content, this key will be missing."
     }

event-schemas/schema/v1/m.room.member

@@ -8,6 +8,7 @@
     "properties": {
         "content": {
             "type": "object",
+            "title": "EventContent",
             "properties": {
                 "membership": {
                     "type": "string",

specification/events.rst

@@ -12,6 +12,42 @@ server-server and application-service APIs, and are described below.
 {{common_state_event_fields}}
 
 
+Differences between /v1 and /v2 events
+--------------------------------------
+
+There are a few differences between how events are formatted for sending
+between servers over federation and how they are formatted for sending between
+a server and its clients.
+
+Additionally there are a few differences between the format of events in the
+responses to client APIs with a /v1 prefix and responses APIs with a /v2
+prefix.
+
+Events in responses for APIs with the /v2 prefix are generated from an event
+formatted for federation by:
+
+* Removing the following keys:
+  ``auth_events``, ``prev_events``, ``hashes``, ``signatures``, ``depth``,
+  ``origin``, ``prev_state``.
+* Adding an ``age`` to the ``unsigned`` object which gives the time in
+  milliseconds that has ellapsed since the event was sent.
+* Adding a ``prev_content`` to the ``unsigned`` object if the event is
+  a ``state event`` which gives previous content of that state key.
+* Adding a ``redacted_because`` to the ``unsigned`` object if the event was
+  redacted which gives the event that redacted it.
+* Adding a ``transaction_id`` if the event was sent by the client requesting it.
+
+Events in responses for APIs with the /v1 prefix are generated from an event
+formatted for the /v2 prefix by:
+
+* Moving the folling keys from the ``unsigned`` object to the top level event
+  object: ``age``, ``redacted_because``, ``replaces_state``, ``prev_content``.
+* Removing the ``unsigned`` object.
+* Rename the ``sender`` key to ``user_id``.
+* If the event was an ``m.room.member`` with ``membership`` set to ``invite``
+  then adding a ``invite_room_state`` key to the top level event object.
+
+
 Size limits
 -----------
 

specification/modules/instant_messaging.rst

@@ -116,12 +116,63 @@ of the client-server API will resolve this by attaching the transaction ID of th
 sending request to the event itself.
 
 
+Calculating the display name for a user
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Clients may wish to show the human-readable display name of a room member as
+part of a membership list, or when they send a message. However, different
+members may have conflicting display names. Display names MUST be disambiguated
+before showing them to the user, in order to prevent spoofing of other users.
+
+To ensure this is done consistently across clients, clients SHOULD use the
+following algorithm to calculate a disambiguated display name for a given user:
+
+1. Inspect the ``m.room.member`` state event for the relevant user id.
+2. If the ``m.room.member`` state event has no ``displayname`` field, or if
+   that field has a ``null`` value, use the raw user id as the display
+   name. Otherwise:
+3. If the ``m.room.member`` event has a ``displayname`` which is unique among
+   members of the room with ``membership: join`` or ``membership: invite``, use
+   the given ``displayname`` as the user-visible display name. Otherwise:
+4. The ``m.room.member`` event has a non-unique ``displayname``. This should be
+   disambiguated using the user id, for example "display name
+   (@id:homeserver.org)".
+
+   .. TODO-spec
+     what does it mean for a ``displayname`` to be 'unique'? Are we
+     case-sensitive?  Do we care about homograph attacks? See
+     https://matrix.org/jira/browse/SPEC-221.
+
+Developers should take note of the following when implementing the above
+algorithm:
+
+* The user-visible display name of one member can be affected by changes in the
+  state of another member. For example, if ``@user1:matrix.org`` is present in
+  a room, with ``displayname: Alice``, then when ``@user2:example.com`` joins
+  the room, also with ``displayname: Alice``, *both* users must be given
+  disambiguated display names. Similarly, when one of the users then changes
+  their display name, there is no longer a clash, and *both* users can be given
+  their chosen display name. Clients should be alert to this possibility and
+  ensure that all affected users are correctly renamed.
+
+* The display name of a room may also be affected by changes in the membership
+  list. This is due to the room name sometimes being based on user display
+  names (see `Calculating the display name for a room`_).
+
+* If the entire membership list is searched for clashing display names, this
+  leads to an O(N^2) implementation for building the list of room members. This
+  will be very inefficient for rooms with large numbers of members. It is
+  recommended that client implementations maintain a hash table mapping from
+  ``displayname`` to a list of room members using that name. Such a table can
+  then be used for efficient calculation of whether disambiguation is needed.
+
+
 Displaying membership information with messages
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Clients may wish to show the display name and avatar URL of the room member who
 sent a message. This can be achieved by inspecting the ``m.room.member`` state
-event for that user ID.
+event for that user ID (see `Calculating the display name for a user`_).
 
 When a user paginates the message history, clients may wish to show the
 **historical** display name and avatar URL for a room member. This is possible
@@ -133,6 +184,85 @@ events update the old state. When paginated events are processed sequentially,
 the old state represents the state of the room *at the time the event was sent*.
 This can then be used to set the historical display name and avatar URL.
 
+
+Calculating the display name for a room
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Clients may wish to show a human-readable name for a room. There are a number
+of possibilities for choosing a useful name. To ensure that rooms are named
+consistently across clients, clients SHOULD use the following algorithm to
+choose a name:
+
+1. If the room has an `m.room.name`_ state event, use the name given by that
+   event.
+
+#. If the room has an `m.room.canonical_alias`_ state event, use the alias
+   given by that event.
+
+#. If neither of the above events are present, a name should be composed based
+   on the members of the room. Clients should consider `m.room.member`_ events
+   for users other than the logged-in user, with ``membership: join`` or
+   ``membership: invite``.
+
+   .. _active_members:
+
+   i. If there is only one such event, the display name for the room should be
+      the `disambiguated display name`_ of the corresponding user.
+
+   #. If there are two such events, they should be lexicographically sorted by
+      their ``state_key`` (i.e. the corresponding user IDs), and the display
+      name for the room should be the  `disambiguated display name`_ of both
+      users: "<user1> and <user2>", or a localised variant thereof.
+
+   #. If there are three or more such events, the display name for the room
+      should be based on the disambiguated display name of the user
+      corresponding to the first such event, under a lexicographical sorting
+      according to their ``state_key``. The display name should be in the
+      format "<user1> and <N> others" (or a localised variant thereof), where N
+      is the number of `m.room.member`_ events with ``membership: join`` or
+      ``membership: invite``, excluding the logged-in user and "user1".
+
+      For example, if Alice joins a room, where Bob (whose user id is
+      ``@superuser:example.com``), Carol (user id ``@carol:example.com``) and
+      Dan (user id ``@dan:matrix.org``) are in conversation, Alice's
+      client should show the room name as "Carol and 2 others".
+
+   .. TODO-spec
+     Sorting by user_id certainly isn't ideal, as IDs at the start of the
+     alphabet will end up dominating room names: they will all be called
+     "Arathorn and 15 others". Furthermore - user_ids are not necessarily
+     ASCII, which means we need to either specify a collation order, or specify
+     how to choose one.
+
+     Ideally we might sort by the time when the user was first invited to, or
+     first joined the room. But we don't have this information.
+
+     See https://matrix.org/jira/browse/SPEC-267 for further discussion.
+
+#. If the room has no ``m.room.name`` or ``m.room.canonical_alias`` events, and
+   no active members other than the current user, clients should consider
+   ``m.room.member`` events with ``membership: leave``. If such events exist, a
+   display name such as "Empty room (was <user1> and <N> others)" (or a
+   localised variant thereof) should be used, following similar rules as for
+   active members (see `above <active_members_>`_).
+
+#. A complete absence of ``m.room.name``, ``m.room.canonical_alias``, and
+   ``m.room.member`` events is likely to indicate a problem with creating the
+   room or synchronising the state table; however clients should still handle
+   this situation. A display name such as "Empty room" (or a localised variant
+   thereof) should be used in this situation.
+
+.. _`disambiguated display name`: `Calculating the display name for a user`_
+
+Clients SHOULD NOT use `m.room.aliases`_ events as a source for room names, as
+it is difficult for clients to agree on the best alias to use, and aliases can
+change unexpectedly.
+
+.. TODO-spec
+  How can we make this less painful for clients to implement, without forcing
+  an English-language implementation on them all?
+
+
 Server behaviour
 ----------------
 

templating/matrix_templates/templates/msgtypes.tmpl

@@ -18,6 +18,8 @@
 ================== ================= ===========================================
 
 {% endfor %}
-Example::
+Example:
+
+.. code:: json
 
     {{example | jsonify(4, 4)}}

templating/matrix_templates/units.py

@@ -28,7 +28,25 @@ ROOM_EVENT = "core-event-schema/room_event.json"
 STATE_EVENT = "core-event-schema/state_event.json"
 
 
-def get_json_schema_object_fields(obj, enforce_title=False):
+def resolve_references(path, schema):
+    if isinstance(schema, dict):
+        result = {}
+        for key, value in schema.items():
+            if key == "$ref":
+                path = os.path.join(os.path.dirname(path), value)
+                with open(path) as f:
+                    schema = json.load(f)
+                return resolve_references(path, schema)
+            else:
+                result[key] = resolve_references(path, value)
+        return result
+    elif isinstance(schema, list):
+        return [resolve_references(path, value) for value in schema]
+    else:
+        return schema
+
+
+def get_json_schema_object_fields(obj, enforce_title=False, include_parents=False):
     # Algorithm:
     # f.e. property => add field info (if field is object then recurse)
     if obj.get("type") != "object":
@@ -36,9 +54,9 @@ def get_json_schema_object_fields(obj, enforce_title=False):
             "get_json_schema_object_fields: Object %s isn't an object." % obj
         )
     if enforce_title and not obj.get("title"):
-        raise Exception(
+        # Force a default titile of "NO_TITLE" to make it obvious in the
-            "get_json_schema_object_fields: Nested object %s doesn't have a title." % obj
+        # specification output which parts of the schema are missing a title
-        )
+        obj["title"] = 'NO_TITLE'
 
     required_keys = obj.get("required")
     if not required_keys:
@@ -73,9 +91,15 @@ def get_json_schema_object_fields(obj, enforce_title=False):
             "Object %s has no properties or parents." % obj
         )
     if not props:  # parents only
+        if include_parents:
+            if obj["title"] == "NO_TITLE" and parents[0].get("title"):
+                obj["title"] = parents[0].get("title")
+            props = parents[0].get("properties")
+
+    if not props:
         return [{
             "title": obj["title"],
-            "parent": parents[0]["$ref"],
+            "parent": parents[0].get("$ref"),
             "no-table": True
         }]
 
@@ -91,7 +115,8 @@ def get_json_schema_object_fields(obj, enforce_title=False):
                 if prop_val == "object":
                     nested_object = get_json_schema_object_fields(
                         props[key_name]["additionalProperties"],
-                        enforce_title=True
+                        enforce_title=True,
+                        include_parents=include_parents,
                     )
                     key = props[key_name]["additionalProperties"].get(
                         "x-pattern", "string"
@@ -103,8 +128,9 @@ def get_json_schema_object_fields(obj, enforce_title=False):
                     value_type = "{string: %s}" % prop_val
             else:
                 nested_object = get_json_schema_object_fields(
-                    props[key_name], 
+                    props[key_name],
-                    enforce_title=True
+                    enforce_title=True,
+                    include_parents=include_parents,
                 )
                 value_type = "{%s}" % nested_object[0]["title"]
 
@@ -114,13 +140,17 @@ def get_json_schema_object_fields(obj, enforce_title=False):
             # if the items of the array are objects then recurse
             if props[key_name]["items"]["type"] == "object":
                 nested_object = get_json_schema_object_fields(
-                    props[key_name]["items"], 
+                    props[key_name]["items"],
-                    enforce_title=True
+                    enforce_title=True,
+                    include_parents=include_parents,
                 )
                 value_type = "[%s]" % nested_object[0]["title"]
                 tables += nested_object
             else:
-                value_type = "[%s]" % props[key_name]["items"]["type"]
+                value_type = props[key_name]["items"]["type"]
+                if isinstance(value_type, list):
+                    value_type = " or ".join(value_type)
+                value_type = "[%s]" % value_type
                 array_enums = props[key_name]["items"].get("enum")
                 if array_enums:
                     if len(array_enums) > 1:
@@ -137,12 +167,16 @@ def get_json_schema_object_fields(obj, enforce_title=False):
             if props[key_name].get("enum"):
                 if len(props[key_name].get("enum")) > 1:
                     value_type = "enum"
+                    if desc:
+                        desc += " "
                     desc += (
-                        " One of: %s" % json.dumps(props[key_name]["enum"])
+                        "One of: %s" % json.dumps(props[key_name]["enum"])
                     )
                 else:
+                    if desc:
+                        desc += " "
                     desc += (
-                        " Must be '%s'." % props[key_name]["enum"][0]
+                        "Must be '%s'." % props[key_name]["enum"][0]
                     )
             if isinstance(value_type, list):
                 value_type = " or ".join(value_type)
@@ -154,12 +188,22 @@ def get_json_schema_object_fields(obj, enforce_title=False):
             "desc": desc,
             "req_str": "**Required.** " if required else ""
         })
-    return tables
+
+    titles = set()
+    filtered = []
+    for table in tables:
+        if table.get("title") in titles:
+            continue
+
+        titles.add(table.get("title"))
+        filtered.append(table)
+
+    return filtered
 
 
 class MatrixUnits(Units):
 
-    def _load_swagger_meta(self, api, group_name):
+    def _load_swagger_meta(self, filepath, api, group_name):
         endpoints = []
         for path in api["paths"]:
             for method in api["paths"][path]:
@@ -262,7 +306,10 @@ class MatrixUnits(Units):
                             if is_array_of_objects:
                                 req_obj = req_obj["items"]
 
-                            req_tables = get_json_schema_object_fields(req_obj)
+                            req_tables = get_json_schema_object_fields(
+                                resolve_references(filepath, req_obj),
+                                include_parents=True,
+                            )
 
                             if req_tables > 1:
                                 for table in req_tables[1:]:
@@ -379,7 +426,10 @@ class MatrixUnits(Units):
                     elif res_type and Units.prop(good_response, "schema/properties"):
                         # response is an object:
                         schema = good_response["schema"]
-                        res_tables = get_json_schema_object_fields(schema)
+                        res_tables = get_json_schema_object_fields(
+                            resolve_references(filepath, schema),
+                            include_parents=True,
+                        )
                         for table in res_tables:
                             if "no-table" not in table:
                                 endpoint["res_tables"].append(table)
@@ -445,13 +495,16 @@ class MatrixUnits(Units):
                 if not filename.endswith(".yaml"):
                     continue
                 self.log("Reading swagger API: %s" % filename)
-                with open(os.path.join(path, filename), "r") as f:
+                filepath = os.path.join(path, filename)
+                with open(filepath, "r") as f:
                     # strip .yaml
                     group_name = filename[:-5].replace("-", "_")
                     if is_v2:
                         group_name = "v2_" + group_name
                     api = yaml.load(f.read())
-                    api["__meta"] = self._load_swagger_meta(api, group_name)
+                    api["__meta"] = self._load_swagger_meta(
+                        filepath, api, group_name
+                    )
                     apis[group_name] = api
         return apis