From 87b6dd845e0e6c685800bbc13f9459a7d21ad59e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 1 Oct 2015 17:55:16 +0100 Subject: [PATCH 1/4] Flesh out content repo; modify templating to support headers Edit content-repo.yaml to include examples and headers. Restructure content module to conform to the module template. Adjust the HTTP API template to give 1 more char to the response param to fit "Content-Disposition" correctly. Edit the templating system to support displaying enums for swagger APIs (before it was just JSON schema). Also add support for introspecting headers from swagger. Finally, replace - with _ when forming the {{ template_var }} else things whine. --- api/client-server/v1/content-repo.yaml | 49 +++++++++++++++++-- specification/modules/content_repo.rst | 30 ++++++++++-- .../matrix_templates/templates/http-api.tmpl | 12 ++--- templating/matrix_templates/units.py | 22 +++++++-- 4 files changed, 96 insertions(+), 17 deletions(-) diff --git a/api/client-server/v1/content-repo.yaml b/api/client-server/v1/content-repo.yaml index fe3d1dc3..aa47dc18 100644 --- a/api/client-server/v1/content-repo.yaml +++ b/api/client-server/v1/content-repo.yaml @@ -15,16 +15,22 @@ paths: summary: Upload some content to the content repository. produces: ["application/json"] parameters: + - in: header + name: Content-Type + type: string + description: The content type of the file being uploaded + x-example: "Content-Type: audio/mpeg" - in: body - name: content + name: "" description: The content to be uploaded. required: true schema: type: string + example: "" format: byte responses: 200: - description: Information about the uploaded content. + description: The MXC URI for the uploaded content. schema: type: object required: ["content_uri"] @@ -32,6 +38,11 @@ paths: content_uri: type: string description: "The MXC URI to the uploaded content." + examples: + "application/json": |- + { + "content_uri": "mxc://example.com/AQwafuaFswefuhsfAFAgsw" + } "/download/{serverName}/{mediaId}": get: summary: "Download content from the content repository." @@ -40,20 +51,32 @@ paths: - in: path type: string name: serverName + x-example: matrix.org required: true description: | The server name from the ``mxc://`` URI (the authoritory component) - in: path type: string name: mediaId + x-example: ascERGshawAWawugaAcauga required: true description: | The media ID from the ``mxc://`` URI (the path component) responses: 200: - description: "The content downloaded." + description: "The content that was previously uploaded." + headers: + Content-Type: + description: "The content type of the file that was previously uploaded." + x-example: "audio/mpeg" + type: "string" + Content-Disposition: + description: "The name of the file that was previously uploaded, if set." + x-example: "attachment;filename=03-cool.mp3" + type: "string" schema: type: file + name: "" "/thumbnail/{serverName}/{mediaId}": get: summary: "Download a thumbnail of the content from the content repository." @@ -63,31 +86,47 @@ paths: type: string name: serverName required: true + x-example: matrix.org description: | The server name from the ``mxc://`` URI (the authoritory component) - in: path type: string name: mediaId + x-example: ascERGshawAWawugaAcauga required: true description: | The media ID from the ``mxc://`` URI (the path component) - in: query type: integer + x-example: 64 name: width - description: The desired width of the thumbnail. + description: |- + The *desired* width of the thumbnail. The actual thumbnail may not + match the size specified. - in: query type: integer + x-example: 64 name: height - description: The desired height of the thumbnail. + description: |- + The *desired* height of the thumbnail. The actual thumbnail may not + match the size specified. - in: query type: string enum: ["crop", "scale"] name: method + x-example: "scale" description: The desired resizing method. responses: 200: description: "A thumbnail of the requested content." + headers: + Content-Type: + description: "The content type of the thumbnail." + x-example: "image/jpeg" + type: "string" + enum: ["image/jpeg", "image/png"] schema: type: file + name: "" diff --git a/specification/modules/content_repo.rst b/specification/modules/content_repo.rst index 83333f37..de464e20 100644 --- a/specification/modules/content_repo.rst +++ b/specification/modules/content_repo.rst @@ -3,8 +3,23 @@ Content repository .. _module:content: -HTTP API --------- +This module allows users to upload content to their homeserver which is +retrievable from other homeservers. Its' purpose is to allow users to share +attachments in a room. Content locations are represented as Matrix Content (MXC) +URIs. They look like:: + + mxc:/// + + : The name of the homeserver where this content can be found, e.g. matrix.org + : An opaque ID which identifies the content. + +Client behaviour +---------------- + +Clients can upload and download content using the following HTTP APIs. + +{{content_repo_http_api}} + Uploads are POSTed to a resource which returns a token which is used to GET the download. Uploads are POSTed to the sender's local homeserver, but are @@ -49,6 +64,9 @@ width and height are close to the requested size and the aspect matches the requested size. The client should scale the image if it needs to fit within a given rectangle. +Server behaviour +---------------- + Homeservers may generate thumbnails for content uploaded to remote homeservers themselves or may rely on the remote homeserver to thumbnail the content. Homeservers may return thumbnails of a different size to that @@ -58,13 +76,19 @@ Homeservers must never upscale images. Security considerations ----------------------- +The HTTP GET endpoint does not require any authentication. Knowing the URL of +the content is sufficient to retrieve the content, even if the entity isn't in +the room. + +Homeservers have additional concerns: + - Clients may try to upload very large files. Homeservers should not store files that are too large and should not serve them to clients. - Clients may try to upload very large images. Homeservers should not attempt to generate thumbnails for images that are too large. - - Remote homeservers may host very large files or images. Homeserver should not + - Remote homeservers may host very large files or images. Homeservers should not proxy or thumbnail large files or images from remote homeservers. - Clients may try to upload a large number of files. Homeservers should limit the diff --git a/templating/matrix_templates/templates/http-api.tmpl b/templating/matrix_templates/templates/http-api.tmpl index eb3f3e64..472c9d7a 100644 --- a/templating/matrix_templates/templates/http-api.tmpl +++ b/templating/matrix_templates/templates/http-api.tmpl @@ -31,18 +31,18 @@ Response format: {% for table in endpoint.res_tables -%} {{"``"+table.title+"``" if table.title else "" }} -================== ================= =========================================== +=================== ================= ========================================== Param Type Description -================== ================= =========================================== +=================== ================= ========================================== {% for row in table.rows -%} {# -#} -{# Row type needs to prepend spaces to line up with the type column (19 ch) -#} +{# Row type needs to prepend spaces to line up with the type column (20 ch) -#} {# Desc needs to prepend the required text (maybe) and prepend spaces too -#} -{# It also needs to then wrap inside the desc col (43 ch width) -#} +{# It also needs to then wrap inside the desc col (42 ch width) -#} {# -#} -{{row.key}}{{row.type|indent(19-row.key|length)}}{{row.desc|wrap(43,row.req_str | indent(18 - (row.type|length))) |indent_block(37)}} +{{row.key}}{{row.type|indent(20-row.key|length)}}{{row.desc|wrap(42,row.req_str | indent(18 - (row.type|length))) |indent_block(38)}} {% endfor -%} -================== ================= =========================================== +=================== ================= ========================================== {% endfor %} {% endif -%} diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 50fa784e..04f3bae1 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -151,6 +151,13 @@ class MatrixUnits(Units): # assign value expected for this param val_type = param.get("type") # integer/string + + if param.get("enum"): + val_type = "enum" + desc += ( + " One of: %s" % json.dumps(param.get("enum")) + ) + refType = Units.prop(param, "schema/$ref/") # Error,Event schemaFmt = Units.prop(param, "schema/format") # bytes e.g. uploads if not val_type and refType: @@ -255,7 +262,7 @@ class MatrixUnits(Units): res_type = Units.prop(good_response, "schema/type") if res_type and res_type not in ["object", "array"]: # response is a raw string or something like that - endpoint["res_tables"].append({ + good_table = { "title": None, "rows": [{ "key": good_response["schema"].get("name", ""), @@ -263,7 +270,16 @@ class MatrixUnits(Units): "desc": res.get("description", ""), "req_str": "" }] - }) + } + if good_response.get("headers"): + for (header_name, header) in good_response.get("headers").iteritems(): + good_table["rows"].append({ + "key": header_name, + "type": "Header<" + header["type"] + ">", + "desc": header["description"], + "req_str": "" + }) + endpoint["res_tables"].append(good_table) elif res_type and Units.prop(good_response, "schema/properties"): # response is an object: schema = good_response["schema"] @@ -328,7 +344,7 @@ class MatrixUnits(Units): self.log("Reading swagger API: %s" % filename) with open(os.path.join(path, filename), "r") as f: # strip .yaml - group_name = filename[:-5] + group_name = filename[:-5].replace("-", "_") api = yaml.load(f.read()) api["__meta"] = self._load_swagger_meta(api, group_name) apis[group_name] = api From 8c4d7f50510edb2a8e0932248efed1949bf1715a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 1 Oct 2015 18:03:34 +0100 Subject: [PATCH 2/4] Do not try to parse non-json request examples as json --- api/check_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/check_examples.py b/api/check_examples.py index a0cd0658..f08b2dc1 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -34,7 +34,7 @@ def check_parameter(filepath, request, parameter): example = None try: example_json = schema.get('example') - if example_json: + if example_json and not schema.get("format") == "byte": example = json.loads(example_json) except Exception as e: raise ValueError("Error parsing JSON example request for %r" % ( From 3d9dbe42e691f446830e348eb680297728dadede Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 2 Oct 2015 10:21:48 +0100 Subject: [PATCH 3/4] Bump to swagger-parser 3.2.1 - remove x- keys on headers Removed x- keys due to https://github.com/BigstickCarpet/swagger-parser/issues/23 --- api/client-server/v1/content-repo.yaml | 5 ----- api/package.json | 2 +- api/validator.js | 10 +++++----- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/api/client-server/v1/content-repo.yaml b/api/client-server/v1/content-repo.yaml index aa47dc18..8e6e8d1a 100644 --- a/api/client-server/v1/content-repo.yaml +++ b/api/client-server/v1/content-repo.yaml @@ -68,15 +68,12 @@ paths: headers: Content-Type: description: "The content type of the file that was previously uploaded." - x-example: "audio/mpeg" type: "string" Content-Disposition: description: "The name of the file that was previously uploaded, if set." - x-example: "attachment;filename=03-cool.mp3" type: "string" schema: type: file - name: "" "/thumbnail/{serverName}/{mediaId}": get: summary: "Download a thumbnail of the content from the content repository." @@ -122,11 +119,9 @@ paths: headers: Content-Type: description: "The content type of the thumbnail." - x-example: "image/jpeg" type: "string" enum: ["image/jpeg", "image/png"] schema: type: file - name: "" diff --git a/api/package.json b/api/package.json index 15193493..84b9dd7b 100644 --- a/api/package.json +++ b/api/package.json @@ -10,6 +10,6 @@ "license": "ISC", "dependencies": { "nopt": "^3.0.2", - "swagger-parser": "^2.4.1" + "swagger-parser": "^3.2.1" } } diff --git a/api/validator.js b/api/validator.js index 3b89a5a3..0d76c09d 100644 --- a/api/validator.js +++ b/api/validator.js @@ -26,11 +26,10 @@ if (!opts.schema) { } -var errFn = function(err, api, metadata) { +var errFn = function(err, api) { if (!err) { return; } - console.log(metadata); console.error(err); process.exit(1); }; @@ -46,11 +45,12 @@ if (isDir) { files.forEach(function(f) { var suffix = ".yaml"; if (f.indexOf(suffix, f.length - suffix.length) > 0) { - parser.parse(path.join(opts.schema, f), function(err, api, metadata) { + parser.validate(path.join(opts.schema, f), function(err, api, metadata) { if (!err) { console.log("%s is valid.", f); } else { + console.error("%s is not valid.", f); errFn(err, api, metadata); } }); @@ -59,12 +59,12 @@ if (isDir) { }); } else{ - parser.parse(opts.schema, function(err, api, metadata) { + parser.validate(opts.schema, function(err, api) { if (!err) { console.log("%s is valid", opts.schema); } else { - errFn(err, api, metadata); + errFn(err, api); } }); }; From 4dabcd112ef787c2f09ddbd61415e145e1b146e3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 2 Oct 2015 10:44:50 +0100 Subject: [PATCH 4/4] Remove redundant info now we have the http api template. Minor tweaks to display of schema with no names but a type --- specification/modules/content_repo.rst | 46 +++++--------------------- templating/matrix_templates/units.py | 3 +- 2 files changed, 11 insertions(+), 38 deletions(-) diff --git a/specification/modules/content_repo.rst b/specification/modules/content_repo.rst index de464e20..9ac5e199 100644 --- a/specification/modules/content_repo.rst +++ b/specification/modules/content_repo.rst @@ -10,9 +10,15 @@ URIs. They look like:: mxc:/// - : The name of the homeserver where this content can be found, e.g. matrix.org + : The name of the homeserver where this content originated, e.g. matrix.org : An opaque ID which identifies the content. +Uploads are POSTed to a resource on the user's local homeserver which returns a +token which is used to GET the download. Content is downloaded from the +recipient's local homeserver, which must first transfer the content from the +origin homeserver using the same API (unless the origin and destination +homeservers are the same). + Client behaviour ---------------- @@ -20,42 +26,8 @@ Clients can upload and download content using the following HTTP APIs. {{content_repo_http_api}} - -Uploads are POSTed to a resource which returns a token which is used to GET -the download. Uploads are POSTed to the sender's local homeserver, but are -downloaded from the recipient's local homeserver, which must thus first transfer -the content from the origin homeserver using the same API (unless the origin -and destination homeservers are the same). The upload/download API is:: - - => POST /_matrix/media/v1/upload HTTP/1.1 - Content-Type: - - - - <= HTTP/1.1 200 OK - Content-Type: application/json - - { "content-uri": "mxc:///" } - - => GET /_matrix/media/v1/download// HTTP/1.1 - - <= HTTP/1.1 200 OK - Content-Type: - Content-Disposition: attachment;filename= - - - -Clients can get thumbnails by supplying a desired width and height and -thumbnailing method:: - - => GET /_matrix/media/v1/thumbnail/ - /?width=&height=&method= HTTP/1.1 - - <= HTTP/1.1 200 OK - Content-Type: image/jpeg or image/png - - - +Thumbnails +~~~~~~~~~~ The thumbnail methods are "crop" and "scale". "scale" tries to return an image where either the width or the height is smaller than the requested size. The client should then scale and letterbox the image if it needs to diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 04f3bae1..1e449b5d 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -260,12 +260,13 @@ class MatrixUnits(Units): if good_response: self.log("Found a 200 response for this API") res_type = Units.prop(good_response, "schema/type") + res_name = Units.prop(good_response, "schema/name") if res_type and res_type not in ["object", "array"]: # response is a raw string or something like that good_table = { "title": None, "rows": [{ - "key": good_response["schema"].get("name", ""), + "key": "<" + res_type + ">" if not res_name else res_name, "type": res_type, "desc": res.get("description", ""), "req_str": ""