From 5aef545128b08b12c44a73fa92b903caf162db48 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 1 Aug 2018 13:31:23 -0600 Subject: [PATCH 1/4] Improve documentation for backfilling rooms There's not a whole lot to improve here - most of the changes are about reorganization and minor clarifications. --- api/server-server/backfill.yaml | 77 +++++++++++++++++++++++++++++ api/server-server/events.yaml | 32 ------------ specification/server_server_api.rst | 27 +++------- 3 files changed, 83 insertions(+), 53 deletions(-) create mode 100644 api/server-server/backfill.yaml diff --git a/api/server-server/backfill.yaml b/api/server-server/backfill.yaml new file mode 100644 index 00000000..f83057ae --- /dev/null +++ b/api/server-server/backfill.yaml @@ -0,0 +1,77 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Events API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/backfill/{roomId}": + get: + summary: Retrieves the events which precede the given event + description: |- + Retreives a sliding-window history of previous PDUs that occurred in the given room. + Starting from the PDU ID(s) given in the ``v`` argument, the PDUs that preceded it + are retrieved, up to the total number given by the ``limit``. + operationId: backfillRoom + parameters: + - in: path + name: roomId + type: string + description: The room ID to backfill. + required: true + x-example: "!SomeRoom:matrix.org" + - in: query + name: v + type: array + items: + type: string + description: The event IDs to backfill from. + required: true + x-example: ["$abc123:matrix.org"] + - in: query + name: limit + type: integer + description: The maximum number of PDUs to retrieve, including the given events. + required: true + x-example: 2 + responses: + 200: + description: |- + A transaction containing the PDUs that preceded the given event(s), including the given + event(s), up to the given limit. + schema: + $ref: "definitions/transaction.yaml" + # Override the example to show the response of the request a bit better + examples: + application/json: { + "$ref": "examples/transaction.json", + "pdus": [ + { + "$ref": "pdu.json", + "room_id": "!SomeRoom:matrix.org", + "event_id": "$abc123:matrix.org" + }, + { + "$ref": "pdu.json", + "room_id": "!SomeRoom:matrix.org" + }, + ] + } diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index d540085d..3b48f15c 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -59,35 +59,3 @@ paths: description: A transaction containing a single PDU which is the event requested. schema: $ref: "definitions/transaction.yaml" - "/backfill/{roomId}": - get: - summary: Retrieves the events which precede the given event - description: |- - Retreives a sliding-window history of previous PDUs that occurred in the given room. - Starting from the PDU ID(s) given in the ``v`` argument, the PDUs that preceded it - are retrieved, up to the total number given by the ``limit``. - operationId: backfillRoom - parameters: - - in: path - name: roomId - type: string - description: The room ID to backfill. - required: true - x-example: "!abc123:matrix.org" - - in: query - name: v - type: string # TODO: The description says this is plural - figure out how to specify multiple, and spec it - description: The event ID to backfill from. - required: true - x-example: "$abc123:matrix.org" - - in: query - name: limit - type: integer - description: The maximum number of events to retrieve. - required: true - x-example: 10 - responses: - 200: - description: A transaction containing the PDUs that preceded the given event(s). - schema: - $ref: "definitions/transaction.yaml" diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f1825f27..090885bc 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -850,32 +850,17 @@ To cover this case, the federation API provides a server-to-server analog of the ``/messages`` client API, allowing one homeserver to fetch history from another. This is the ``/backfill`` API. -To request more history, the requesting homeserver picks another homeserver -that it thinks may have more (most likely this should be a homeserver for some -of the existing users in the room at the earliest point in history it has -currently), and makes a ``/backfill`` request. The parameters of this request -give an event ID that the requesting homeserver wishes to obtain, and a number -specifying how many more events of history before that one to return at most. - -The response to this request is an object with the following keys: - -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``pdus`` List A list of events. -``origin`` String The name of the resident homeserver. -``origin_server_ts`` Integer A timestamp added by the resident - homeserver. -======================== ============ ========================================= - -The list of events given in ``pdus`` is returned in reverse chronological -order; having the most recent event first (i.e. the event whose event ID is -that requested by the requester in the ``v`` parameter). +To request more history, the requesting homeserver picks another homeserver +that it thinks may have more (most likely this should be a homeserver for +some of the existing users in the room at the earliest point in history it +has currently), and makes a ``/backfill`` request. .. TODO-spec Specify (or remark that it is unspecified) how the server handles divergent history. DFS? BFS? Anything weirder? +{{backfill_ss_http_api}} + Inviting to a room ------------------ From cda88f3b3debfcacb4ea2ec37ed8309d3090dd8e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 23:18:08 -0600 Subject: [PATCH 2/4] Document /get_missing_events Fixes https://github.com/matrix-org/matrix-doc/issues/1385 --- api/server-server/backfill.yaml | 66 ++++++++++++++++++++++++++++- specification/server_server_api.rst | 8 +++- 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/api/server-server/backfill.yaml b/api/server-server/backfill.yaml index f83057ae..072b28d1 100644 --- a/api/server-server/backfill.yaml +++ b/api/server-server/backfill.yaml @@ -27,7 +27,7 @@ paths: get: summary: Retrieves the events which precede the given event description: |- - Retreives a sliding-window history of previous PDUs that occurred in the given room. + Retrieves a sliding-window history of previous PDUs that occurred in the given room. Starting from the PDU ID(s) given in the ``v`` argument, the PDUs that preceded it are retrieved, up to the total number given by the ``limit``. operationId: backfillRoom @@ -75,3 +75,67 @@ paths: }, ] } + "/get_missing_events/{roomId}": + post: + summary: Retrieves events that the sender is missing + description: |- + Retrieves the previous events that the sender is missing for a given set of + events. + operationId: getMissingPreviousEvents + parameters: + - in: path + name: roomId + type: string + description: The room ID to search in. + required: true + x-example: "!SomeRoom:matrix.org" + - in: body + name: body + schema: + type: object + properties: + limit: + type: integer + description: The maximum number of events to retrieve. Defaults to 10. + example: 10 + min_depth: + type: integer + description: The minimum depth of events to retrieve. Defaults to 0. + example: 0 + earliest_events: + type: array + description: |- + The events the sender is missing. The response will only contain events + that didn't happen before these events. + items: + type: string + example: ["$missing_event:domain.com"] + latest_events: + type: array + description: |- + The events that the sender has, but does not have previous events for. + items: + type: string + example: ["$event_that_has_the_missing_event_as_a_previous_event:domain.com"] + required: ['earliest_events', 'latest_events'] + responses: + 200: + description: |- + The events that happened after the ``earliest_events`` which are also previous + to the ``latest_events`` and have the minimum depth specified. Does not include + any of the events in the request. + schema: + type: object + properties: + events: + type: array + description: The missing events. + items: + $ref: definitions/pdu.yaml + required: ['events'] + examples: + application/json: { + "events": [ + {"$ref": "examples/pdu.json"} + ] + } diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 090885bc..d7a22e72 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -836,8 +836,8 @@ following keys: here. What purpose does it serve expanding them out in full, when surely they'll appear in the state anyway? -Backfilling ------------ +Backfilling and retrieving missing events +----------------------------------------- Once a homeserver has joined a room, it receives all the events emitted by other homeservers in that room, and is thus aware of the entire history of the @@ -855,6 +855,10 @@ that it thinks may have more (most likely this should be a homeserver for some of the existing users in the room at the earliest point in history it has currently), and makes a ``/backfill`` request. +Similar to backfilling a room's history, a server may not have all the events +in the graph. That server may use the ``/get_missing_events`` API to acquire +the events it is missing. + .. TODO-spec Specify (or remark that it is unspecified) how the server handles divergent history. DFS? BFS? Anything weirder? From 6ac25d167c192d3be5a081170c2d67703cf02bdf Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 3 Aug 2018 13:57:13 -0600 Subject: [PATCH 3/4] get_missing_events consumes JSON --- api/server-server/backfill.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/server-server/backfill.yaml b/api/server-server/backfill.yaml index 072b28d1..d4dad2ec 100644 --- a/api/server-server/backfill.yaml +++ b/api/server-server/backfill.yaml @@ -20,6 +20,8 @@ host: localhost:8448 schemes: - https basePath: /_matrix/federation/v1 +consumes: + - application/json produces: - application/json paths: From 721be47eb1070d9e11bb999353c49a4a307f1d6d Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 8 Aug 2018 08:39:59 -0600 Subject: [PATCH 4/4] Describe how get_missing_events actually works The original text wasn't far off, except for being wrong about what the fields actually do. This commit also adds a bit of clarity for how the server is expected to behave. --- api/server-server/backfill.yaml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/api/server-server/backfill.yaml b/api/server-server/backfill.yaml index d4dad2ec..f9f105e2 100644 --- a/api/server-server/backfill.yaml +++ b/api/server-server/backfill.yaml @@ -81,8 +81,9 @@ paths: post: summary: Retrieves events that the sender is missing description: |- - Retrieves the previous events that the sender is missing for a given set of - events. + Retrieves previous events that the sender is missing. This is done by doing a breadth-first + walk of the ``prev_events`` for the ``latest_events``, ignoring any events in ``earliest_events`` + and stopping at the ``limit``. operationId: getMissingPreviousEvents parameters: - in: path @@ -107,15 +108,14 @@ paths: earliest_events: type: array description: |- - The events the sender is missing. The response will only contain events - that didn't happen before these events. + The latest events that the sender already has. These are skipped when retrieving + the previous events of ``latest_events``. items: type: string example: ["$missing_event:domain.com"] latest_events: type: array - description: |- - The events that the sender has, but does not have previous events for. + description: The events to retrieve the previous events for. items: type: string example: ["$event_that_has_the_missing_event_as_a_previous_event:domain.com"] @@ -123,9 +123,8 @@ paths: responses: 200: description: |- - The events that happened after the ``earliest_events`` which are also previous - to the ``latest_events`` and have the minimum depth specified. Does not include - any of the events in the request. + The previous events for ``latest_events``, excluding any ``earliest_events``, up to the + provided ``limit``. schema: type: object properties: