diff --git a/.gitignore b/.gitignore index 84ac4951..a850d2fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ /api/node_modules /assets /assets.tar.gz -/env +/env* /scripts/gen /scripts/continuserv/continuserv /scripts/speculator/speculator @@ -10,3 +10,4 @@ /templating/out *.pyc *.swp +_rendered.rst diff --git a/.travis.yml b/.travis.yml index 7d84f237..9e9363af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,15 +2,15 @@ language: go go: - 1.8 -sudo: false +sudo: required # we only need a single git commit git: depth: 1 -# test-and-build does the installation, so tell travis to skip the -# installation step -install: true +install: + - sudo apt-get update + - sudo apt-get install python3 python3-dev script: - ./scripts/test-and-build.sh diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 2a7416b1..c592cf02 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -69,6 +69,41 @@ For such changes, please do just open a `pull request`_. .. _pull request: https://help.github.com/articles/about-pull-requests + +Adding to the changelog +~~~~~~~~~~~~~~~~~~~~~~~ + +Currently only changes to the client-server API need to end up in a changelog. The +other APIs are not yet stable and therefore do not have a changelog. Adding to the +changelog can only be done after you've opened your pull request, so be sure to do +that first. + +The changelog is managed by Towncrier (https://github.com/hawkowl/towncrier) in the +form of "news fragments". The news fragments for the client-server API are stored +under ``changelogs/client_server/newsfragments``. + +To create a changelog entry, create a file named in the format ``prNumber.type`` in +the ``newsfragments`` directory. The ``type`` can be one of the following: + +* ``new`` - Used when adding new endpoints. Please have the file contents be the + method and route being added, surrounded in RST code tags. For example: ``POST + /accounts/whoami`` + +* ``feature`` - Used when adding backwards-compatible changes to the API. + +* ``clarification`` - Used when an area of the spec is being improved upon and does + not change or introduce any functionality. + +* ``breaking`` - Used when the change is not backwards compatible. + +* ``deprecation`` - Used when deprecating something + +All news fragments must have a brief summary explaining the change in the contents +of the file. + +Changes that do not change the spec, such as changes to the build script, formatting, +CSS, etc should not get a news fragment. + Sign off -------- diff --git a/README.rst b/README.rst index 6c87201c..b8847bfb 100644 --- a/README.rst +++ b/README.rst @@ -41,9 +41,9 @@ specs and event schemas in this repository. Preparation ----------- -To use the scripts, it is best to create a Python 2.x virtualenv as follows:: +To use the scripts, it is best to create a Python 3.4+ virtualenv as follows:: - virtualenv env + virtualenv -p python3 env env/bin/pip install -r scripts/requirements.txt (Benjamin Synders has contributed a script for `Nix`_ users, which can be diff --git a/changelogs/README.md b/changelogs/README.md new file mode 100644 index 00000000..a5fb1fb7 --- /dev/null +++ b/changelogs/README.md @@ -0,0 +1,55 @@ + + +# Changelogs + +[Towncrier](https://github.com/hawkowl/towncrier) is used to manage the changelog and +keep it up to date. Because of this, updating a changelog is really easy. + +## How to update a changelog when releasing an API + +1. Ensure you're in your Python 3 virtual environment +2. `cd` your way to the API you're releasing (eg: `cd changelogs/client_server`) +3. Run `towncrier --version "r0.4.0" --name "client-server" --yes` substituting the + variables as approprite. Note that `--name` is required although the value is ignored. +4. Commit the changes and finish the release process. + +## How to prepare a changelog for a new API + +For this example, we're going to pretend that the `server_server` API doesn't exist. + +1. Create the file `changelogs/server_server.rst` +2. Create the folder `changelogs/server_server` +3. In the new folder, create a `pyproject.toml` file with these contents: + ```toml + [tool.towncrier] + filename = "../server_server.rst" + directory = "newsfragments" + issue_format = "`#{issue} `_" + title_format = "{version}" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true + ``` +4. Create a `.gitignore` in `changelogs/server_server/newsfragments` with the contents `!.gitignore` diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index c32cd639..d2ef7f2d 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -1,89 +1,3 @@ -Unreleased changes -================== - -- Changes to the API which will be backwards-compatible for clients: - - - New endpoints: - - - ``POST /user_directory/search`` - (`#1096 `_). - - ``GET /rooms/{roomId}/event/{eventId}`` - (`#1110 `_). - - ``POST /delete_devices`` - (`#1239 `_). - - ``GET /thirdparty/protocol/{protocol}`` - (`#1353 `_). - - ``GET /thirdparty/location/{protocol}`` - (`#1353 `_). - - ``GET /thirdparty/user/{protocol}`` - (`#1353 `_). - - ``GET /thirdparty/location`` - (`#1353 `_). - - ``GET /thirdparty/user`` - (`#1353 `_). - - - Sticker messages: - - Add sticker message event definition. - (`#1158 `_). - - Document the ``server_name`` parameter on ``/join/{roomIdOrAlias}`` - (`#1364 `_). - - Add 'token' parameter to /keys/query endpoint - (`#1104 `_). - - Add the room visibility options for the room directory - (`#1141 `_). - - Add spec for ignoring users - (`#1142 `_). - - Add the ``/register/available`` endpoint for username availability - (`#1151 `_). - - Add ``allow_remote`` to the content repo to avoid routing loops - (`#1265 `_). - - Add report content API - (`#1264 `_). - - Document ``/logout/all`` endpoint - (`#1263 `_). - - Document `highlights` field in /search response - (`#1274 `_). - - Document the GET version of ``/login`` - (`#1361 `_). - - Document guest access in ``/createRoom`` presets - (`#1379 `_). - - Document the CORS/preflight headers - (`#1365 `_). - -- Spec clarifications: - - - Update ``ImageInfo`` and ``ThumbnailInfo`` dimension schema descriptions - to clarify that they relate to intended display size, as opposed to the - intrinsic size of the image file. - (`#1158 `_). - - Mark ``home_server`` return field for ``/login`` and ``/register`` - endpoints as deprecated - (`#1097 `_). - - Fix response format of ``/keys/changes`` endpoint - (`#1106 `_). - - Clarify default values for some fields on the /search API - (`#1109 `_). - - Fix the representation of ``m.presence`` events - (`#1137 `_). - - Clarify that ``m.tag`` ordering is done with numbers, not strings - (`#1139 `_). - - Clarify that ``/account/whoami`` should consider application services - (`#1152 `_). - - Mark ``GET /rooms/{roomId}/members`` as requiring authentication - (`#1245 `_). - - Define what a ``RoomEvent`` is on ``/rooms/{roomId}/messages`` - (`#1380 `_). - - Mark ``GET /presence/{userId}/status`` as requiring authentication - (`#1371 `_). - - Describe ``StateEvent`` for ``/createRoom`` - (`#1329 `_). - - Describe how the ``reason`` is handled for kicks/bans - (`#1362 `_). - - Clarify that clients must leave rooms before forgetting them - (`#1378 `_). - - Clarify the request and result types on ``/search`` - (`#1381 `_). - r0.3.0 ====== diff --git a/changelogs/client_server/newsfragments/.gitignore b/changelogs/client_server/newsfragments/.gitignore new file mode 100644 index 00000000..b722e9e1 --- /dev/null +++ b/changelogs/client_server/newsfragments/.gitignore @@ -0,0 +1 @@ +!.gitignore \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1096.new b/changelogs/client_server/newsfragments/1096.new new file mode 100644 index 00000000..50d86879 --- /dev/null +++ b/changelogs/client_server/newsfragments/1096.new @@ -0,0 +1 @@ +``POST /user_directory/search`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1097.clarification b/changelogs/client_server/newsfragments/1097.clarification new file mode 100644 index 00000000..2a7cb93a --- /dev/null +++ b/changelogs/client_server/newsfragments/1097.clarification @@ -0,0 +1 @@ +Mark ``home_server`` return field for ``/login`` and ``/register`` endpoints as deprecated \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1104.feature b/changelogs/client_server/newsfragments/1104.feature new file mode 100644 index 00000000..9b85343f --- /dev/null +++ b/changelogs/client_server/newsfragments/1104.feature @@ -0,0 +1 @@ +Add ``token`` parameter to the ``/keys/query`` endpoint \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1106.clarification b/changelogs/client_server/newsfragments/1106.clarification new file mode 100644 index 00000000..f7a1fe3e --- /dev/null +++ b/changelogs/client_server/newsfragments/1106.clarification @@ -0,0 +1 @@ +Fix response format of ``/keys/changes`` endpoint \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1109.clarification b/changelogs/client_server/newsfragments/1109.clarification new file mode 100644 index 00000000..176d9403 --- /dev/null +++ b/changelogs/client_server/newsfragments/1109.clarification @@ -0,0 +1 @@ +Clarify default values for some fields on the ``/search`` API \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1110.new b/changelogs/client_server/newsfragments/1110.new new file mode 100644 index 00000000..e1b80b8b --- /dev/null +++ b/changelogs/client_server/newsfragments/1110.new @@ -0,0 +1 @@ +``GET /rooms/{roomId}/event/{eventId}`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1137.clarification b/changelogs/client_server/newsfragments/1137.clarification new file mode 100644 index 00000000..5ad8ec34 --- /dev/null +++ b/changelogs/client_server/newsfragments/1137.clarification @@ -0,0 +1 @@ +Fix the representation of ``m.presence`` events \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1139.clarification b/changelogs/client_server/newsfragments/1139.clarification new file mode 100644 index 00000000..b5193ad3 --- /dev/null +++ b/changelogs/client_server/newsfragments/1139.clarification @@ -0,0 +1 @@ +Clarify that ``m.tag`` ordering is done with numbers, not strings \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1141.feature b/changelogs/client_server/newsfragments/1141.feature new file mode 100644 index 00000000..da041819 --- /dev/null +++ b/changelogs/client_server/newsfragments/1141.feature @@ -0,0 +1 @@ +Add the room visibility options for the room directory \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1142.feature b/changelogs/client_server/newsfragments/1142.feature new file mode 100644 index 00000000..0a0842c4 --- /dev/null +++ b/changelogs/client_server/newsfragments/1142.feature @@ -0,0 +1 @@ +Add spec for ignoring users \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1151.feature b/changelogs/client_server/newsfragments/1151.feature new file mode 100644 index 00000000..8875812b --- /dev/null +++ b/changelogs/client_server/newsfragments/1151.feature @@ -0,0 +1 @@ +Add the ``/register/available`` endpoint for username availability \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1152.clarification b/changelogs/client_server/newsfragments/1152.clarification new file mode 100644 index 00000000..bbecc9b2 --- /dev/null +++ b/changelogs/client_server/newsfragments/1152.clarification @@ -0,0 +1 @@ +Clarify that ``/account/whoami`` should consider application services \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1158.clarification b/changelogs/client_server/newsfragments/1158.clarification new file mode 100644 index 00000000..dc1f6d14 --- /dev/null +++ b/changelogs/client_server/newsfragments/1158.clarification @@ -0,0 +1,3 @@ +Update ``ImageInfo`` and ``ThumbnailInfo`` dimension schema descriptions +to clarify that they relate to intended display size, as opposed to the +intrinsic size of the image file. \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1158.feature b/changelogs/client_server/newsfragments/1158.feature new file mode 100644 index 00000000..a55df4fb --- /dev/null +++ b/changelogs/client_server/newsfragments/1158.feature @@ -0,0 +1 @@ +Add sticker messages \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1239.new b/changelogs/client_server/newsfragments/1239.new new file mode 100644 index 00000000..9bcf357d --- /dev/null +++ b/changelogs/client_server/newsfragments/1239.new @@ -0,0 +1 @@ +``POST /delete_devices`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1245.clarification b/changelogs/client_server/newsfragments/1245.clarification new file mode 100644 index 00000000..e0a63834 --- /dev/null +++ b/changelogs/client_server/newsfragments/1245.clarification @@ -0,0 +1 @@ +Mark ``GET /rooms/{roomId}/members`` as requiring authentication \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1263.feature b/changelogs/client_server/newsfragments/1263.feature new file mode 100644 index 00000000..04964a7d --- /dev/null +++ b/changelogs/client_server/newsfragments/1263.feature @@ -0,0 +1 @@ +Document ``/logout/all`` endpoint \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1264.feature b/changelogs/client_server/newsfragments/1264.feature new file mode 100644 index 00000000..9cb06f71 --- /dev/null +++ b/changelogs/client_server/newsfragments/1264.feature @@ -0,0 +1 @@ +Add report content API \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1265.feature b/changelogs/client_server/newsfragments/1265.feature new file mode 100644 index 00000000..1e270fa8 --- /dev/null +++ b/changelogs/client_server/newsfragments/1265.feature @@ -0,0 +1 @@ +Add ``allow_remote`` to the content repo to avoid routing loops \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1274.feature b/changelogs/client_server/newsfragments/1274.feature new file mode 100644 index 00000000..d4958131 --- /dev/null +++ b/changelogs/client_server/newsfragments/1274.feature @@ -0,0 +1 @@ +Document `highlights` field in /search response \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1329.clarification b/changelogs/client_server/newsfragments/1329.clarification new file mode 100644 index 00000000..970d3d94 --- /dev/null +++ b/changelogs/client_server/newsfragments/1329.clarification @@ -0,0 +1 @@ +Describe ``StateEvent`` for ``/createRoom`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1353.new b/changelogs/client_server/newsfragments/1353.new new file mode 100644 index 00000000..0af0c520 --- /dev/null +++ b/changelogs/client_server/newsfragments/1353.new @@ -0,0 +1 @@ +``GET /thirdparty/*`` Endpoints diff --git a/changelogs/client_server/newsfragments/1361.feature b/changelogs/client_server/newsfragments/1361.feature new file mode 100644 index 00000000..b1d4c2f1 --- /dev/null +++ b/changelogs/client_server/newsfragments/1361.feature @@ -0,0 +1 @@ +Document the GET version of ``/login`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1362.clarification b/changelogs/client_server/newsfragments/1362.clarification new file mode 100644 index 00000000..1deb4500 --- /dev/null +++ b/changelogs/client_server/newsfragments/1362.clarification @@ -0,0 +1 @@ +Describe how the ``reason`` is handled for kicks/bans \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1364.feature b/changelogs/client_server/newsfragments/1364.feature new file mode 100644 index 00000000..733d6a1f --- /dev/null +++ b/changelogs/client_server/newsfragments/1364.feature @@ -0,0 +1 @@ +Document the ``server_name`` parameter on ``/join/{roomIdOrAlias}`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1365.feature b/changelogs/client_server/newsfragments/1365.feature new file mode 100644 index 00000000..d2864e96 --- /dev/null +++ b/changelogs/client_server/newsfragments/1365.feature @@ -0,0 +1 @@ +Document the CORS/preflight headers \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1371.clarification b/changelogs/client_server/newsfragments/1371.clarification new file mode 100644 index 00000000..88552fcd --- /dev/null +++ b/changelogs/client_server/newsfragments/1371.clarification @@ -0,0 +1 @@ +Mark ``GET /presence/{userId}/status`` as requiring authentication \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1378.clarification b/changelogs/client_server/newsfragments/1378.clarification new file mode 100644 index 00000000..f952428b --- /dev/null +++ b/changelogs/client_server/newsfragments/1378.clarification @@ -0,0 +1 @@ +Clarify that clients must leave rooms before forgetting them \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1379.clarification b/changelogs/client_server/newsfragments/1379.clarification new file mode 100644 index 00000000..122b3900 --- /dev/null +++ b/changelogs/client_server/newsfragments/1379.clarification @@ -0,0 +1 @@ +Document guest access in ``/createRoom`` presets diff --git a/changelogs/client_server/newsfragments/1380.clarification b/changelogs/client_server/newsfragments/1380.clarification new file mode 100644 index 00000000..490a9a43 --- /dev/null +++ b/changelogs/client_server/newsfragments/1380.clarification @@ -0,0 +1 @@ +Define what a ``RoomEvent`` is on ``/rooms/{roomId}/messages`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1381.clarification b/changelogs/client_server/newsfragments/1381.clarification new file mode 100644 index 00000000..e5e599ac --- /dev/null +++ b/changelogs/client_server/newsfragments/1381.clarification @@ -0,0 +1 @@ +Clarify the request and result types on ``/search`` \ No newline at end of file diff --git a/changelogs/client_server/pyproject.toml b/changelogs/client_server/pyproject.toml new file mode 100644 index 00000000..8fa3f6b5 --- /dev/null +++ b/changelogs/client_server/pyproject.toml @@ -0,0 +1,30 @@ +[tool.towncrier] + filename = "../client_server.rst" + directory = "newsfragments" + issue_format = "`#{issue} `_" + title_format = "{version}" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true diff --git a/scripts/dump-swagger.py b/scripts/dump-swagger.py index 110c4446..7994324f 100755 --- a/scripts/dump-swagger.py +++ b/scripts/dump-swagger.py @@ -103,7 +103,7 @@ for filename in os.listdir(cs_api_dir): output["paths"][path] = {} output["paths"][path][method] = spec -print "Generating %s" % output_file +print("Generating %s" % output_file) try: os.makedirs(os.path.dirname(output_file)) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 2b35f801..16c40af5 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -31,6 +31,7 @@ script_dir = os.path.dirname(os.path.abspath(__file__)) docs_dir = os.path.dirname(script_dir) spec_dir = os.path.join(docs_dir, "specification") tmp_dir = os.path.join(script_dir, "tmp") +changelog_dir = os.path.join(docs_dir, "changelogs") VERBOSE = False @@ -151,7 +152,7 @@ def is_title_line(prev_line, line, title_styles): def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): # string are file paths to RST blobs - if isinstance(file_info, basestring): + if isinstance(file_info, str): log("%s %s" % (">" * (1 + title_level), file_info)) with open(os.path.join(spec_dir, file_info), "r") as f: rst = None @@ -194,7 +195,7 @@ def build_spec(target, out_filename): spec_dir=spec_dir, adjust_titles=True ) - outfile.write(section) + outfile.write(section.encode('UTF-8')) """ @@ -279,15 +280,16 @@ def rst2html(i, o, stylesheets): def addAnchors(path): log("add anchors %s" % path) - with open(path, "r") as f: + with open(path, "rb") as f: lines = f.readlines() - replacement = replacement = r'

\n\1' - with open(path, "w") as f: + replacement = r'

\n\1' + with open(path, "wb") as f: for line in lines: + line = line.decode("UTF-8") line = re.sub(r'()', replacement, line.rstrip()) - line = re.sub(r'(
)', replacement, line.rstrip()) - f.write(line + "\n") + line = re.sub(r'(
)', replacement, line.rstrip()) + f.write((line + "\n").encode('UTF-8')) def run_through_template(input_files, set_verbose, substitutions): @@ -364,7 +366,7 @@ def get_build_target(all_targets, target_name): resolved_files = [] for file_entry in target["files"]: # file_entry is a group id - if isinstance(file_entry, basestring) and file_entry.startswith("group:"): + if isinstance(file_entry, str) and file_entry.startswith("group:"): group = get_group(file_entry, 0) # The group may be resolved to a list of file entries, in which case # we want to extend the array to insert each of them rather than @@ -376,8 +378,8 @@ def get_build_target(all_targets, target_name): # file_entry is a dict which has more file entries as values elif isinstance(file_entry, dict): resolved_entry = {} - for (depth, entry) in file_entry.iteritems(): - if not isinstance(entry, basestring): + for (depth, entry) in file_entry.items(): + if not isinstance(entry, str): raise Exception( "Double-nested depths are not supported. Entry: %s" % (file_entry,) ) @@ -395,11 +397,11 @@ def get_build_target(all_targets, target_name): return build_target def log(line): - print "gendoc: %s" % line + print("gendoc: %s" % line) def logv(line): if VERBOSE: - print "gendoc:V: %s" % line + print("gendoc:V: %s" % line) def cleanup_env(): @@ -445,7 +447,7 @@ def main(targets, dest_dir, keep_intermediates, substitutions): stylesheets = glob.glob(os.path.join(script_dir, "css", "*.css")) - for target_name, templated_file in templated_files.iteritems(): + for target_name, templated_file in templated_files.items(): target = target_defs["targets"].get(target_name) version_label = None if target: @@ -480,7 +482,7 @@ def list_targets(): with open(os.path.join(spec_dir, "targets.yaml"), "r") as targ_file: target_defs = yaml.load(targ_file.read()) targets = target_defs["targets"].keys() - print "\n".join(targets) + print("\n".join(targets)) def extract_major(s): diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 37d99dd3..2a7d7ff8 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -7,3 +7,5 @@ Jinja2 >= 2.9.6 jsonschema >= 2.6.0 PyYAML >= 3.12 requests >= 2.18.4 +towncrier == 18.6.0 +six >= 1.11.0 \ No newline at end of file diff --git a/scripts/swagger-http-server.py b/scripts/swagger-http-server.py index 5ec00101..06d764aa 100755 --- a/scripts/swagger-http-server.py +++ b/scripts/swagger-http-server.py @@ -19,14 +19,14 @@ import argparse import os -import SimpleHTTPServer -import SocketServer +import http.server +import socketserver # Thanks to http://stackoverflow.com/a/13354482 -class MyHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): +class MyHTTPRequestHandler(http.server.SimpleHTTPRequestHandler): def end_headers(self): self.send_my_headers() - SimpleHTTPServer.SimpleHTTPRequestHandler.end_headers(self) + http.server.SimpleHTTPRequestHandler.end_headers(self) def send_my_headers(self): self.send_header("Access-Control-Allow-Origin", "*") @@ -49,7 +49,7 @@ if __name__ == '__main__': os.chdir(args.swagger_dir) - httpd = SocketServer.TCPServer(("localhost", args.port), + httpd = socketserver.TCPServer(("localhost", args.port), MyHTTPRequestHandler) - print "Serving at http://localhost:%i/api-docs.json" % args.port + print("Serving at http://localhost:%i/api-docs.json" % args.port) httpd.serve_forever() diff --git a/scripts/templating/batesian/__init__.py b/scripts/templating/batesian/__init__.py index da41b31b..e901590f 100644 --- a/scripts/templating/batesian/__init__.py +++ b/scripts/templating/batesian/__init__.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from sets import Set class AccessKeyStore(object): @@ -22,7 +21,7 @@ class AccessKeyStore(object): if not existing_data: existing_data = {} self.data = existing_data - self.accessed_set = Set() + self.accessed_set = set() def keys(self): return self.data.keys() @@ -35,5 +34,5 @@ class AccessKeyStore(object): return self.data[key] def get_unaccessed_set(self): - data_list = Set(self.data.keys()) + data_list = set(self.data.keys()) return data_list - self.accessed_set \ No newline at end of file diff --git a/scripts/templating/batesian/sections.py b/scripts/templating/batesian/sections.py index c541d771..18a622f6 100644 --- a/scripts/templating/batesian/sections.py +++ b/scripts/templating/batesian/sections.py @@ -29,7 +29,7 @@ class Sections(object): def log(self, text): if self.debug: - print "batesian:sections: %s" % text + print("batesian:sections: %s" % text) def get_sections(self): render_list = inspect.getmembers(self, predicate=inspect.ismethod) @@ -40,7 +40,7 @@ class Sections(object): section_key = func_name[len("render_"):] self.log("Generating section '%s'" % section_key) section = func() - if isinstance(section, basestring): + if isinstance(section, str): if section_key in section_dict: raise Exception( ("%s : Section %s already exists. It must have been " + @@ -54,8 +54,8 @@ class Sections(object): ) elif isinstance(section, dict): self.log(" Generated multiple sections:") - for (k, v) in section.iteritems(): - if not isinstance(k, basestring) or not isinstance(v, basestring): + for (k, v) in section.items(): + if not isinstance(k, str) or not isinstance(v, str): raise Exception( ("Method %s returned multiple sections as a dict but " + "expected the dict elements to be strings but they aren't.") % diff --git a/scripts/templating/batesian/units.py b/scripts/templating/batesian/units.py index 8f748f6d..82cc52f9 100644 --- a/scripts/templating/batesian/units.py +++ b/scripts/templating/batesian/units.py @@ -41,7 +41,7 @@ class Units(object): trace = inspect.stack() if len(trace) > 1 and len(trace[1]) > 2: func_name = trace[1][3] + ":" - print "batesian:units:%s %s" % (func_name, text) + print("batesian:units:%s %s" % (func_name, text)) def get_units(self, debug=False): unit_list = inspect.getmembers(self, predicate=inspect.ismethod) @@ -50,7 +50,7 @@ class Units(object): if not func_name.startswith("load_"): continue unit_key = func_name[len("load_"):] - if len(inspect.getargs(func.func_code).args) > 1: + if len(inspect.getargs(func.__code__).args) > 1: unit_dict[unit_key] = func(self.substitutions) else: unit_dict[unit_key] = func() diff --git a/scripts/templating/build.py b/scripts/templating/build.py index d18569b6..fae4db56 100755 --- a/scripts/templating/build.py +++ b/scripts/templating/build.py @@ -63,6 +63,7 @@ import sys from textwrap import TextWrapper from matrix_templates.units import TypeTableRow +from functools import reduce def create_from_template(template, sections): @@ -138,7 +139,7 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={} return reduce(max, rowwidths, default if default is not None else default_width) - results = map(colwidth, keys, defaults) + results = list(map(colwidth, keys, defaults)) return results # make Jinja aware of the templates and filters @@ -167,7 +168,7 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={} # print out valid section keys if no file supplied if not files: - print "\nValid template variables:" + print("\nValid template variables:") for key in sections.keys(): sec_text = "" if (len(sections[key]) > 75) else ( "(Value: '%s')" % sections[key] @@ -175,8 +176,8 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={} sec_info = "%s characters" % len(sections[key]) if sections[key].count("\n") > 0: sec_info += ", %s lines" % sections[key].count("\n") - print " %s" % key - print " %s %s" % (sec_info, sec_text) + print(" %s" % key) + print(" %s %s" % (sec_info, sec_text)) return # check the input files and substitute in sections where required @@ -190,8 +191,8 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={} def process_file(env, sections, filename, output_filename): log("Parsing input template: %s" % filename) - with open(filename, "r") as file_stream: - temp_str = file_stream.read().decode("utf-8") + with open(filename, "rb") as file_stream: + temp_str = file_stream.read().decode('UTF-8') # do sanity checking on the template to make sure they aren't reffing things # which will never be replaced with a section. @@ -213,13 +214,13 @@ def process_file(env, sections, filename, output_filename): for old, new in substitutions.items(): output = output.replace(old, new) - with open(output_filename, "w") as f: - f.write(output.encode("utf-8")) + with open(output_filename, "wb") as f: + f.write(output.encode('UTF-8')) log("Output file for: %s" % output_filename) def log(line): - print "batesian: %s" % line + print("batesian: %s" % line) if __name__ == '__main__': parser = ArgumentParser( diff --git a/scripts/templating/matrix_templates/__init__.py b/scripts/templating/matrix_templates/__init__.py index 6b46192c..b81c5a30 100644 --- a/scripts/templating/matrix_templates/__init__.py +++ b/scripts/templating/matrix_templates/__init__.py @@ -11,8 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from sections import MatrixSections -from units import MatrixUnits +from .sections import MatrixSections +from .units import MatrixUnits import os exports = { diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index dac183d1..cd563346 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -29,8 +29,9 @@ import os.path import re import subprocess import sys -import urllib import yaml +from functools import reduce +from six.moves.urllib.parse import urlencode, quote matrix_doc_dir=reduce(lambda acc,_: os.path.dirname(acc), range(1, 5), os.path.abspath(__file__)) @@ -147,7 +148,7 @@ def inherit_parents(obj): # settings defined in the child take priority over the parents, so we # iterate through the parents first, and then overwrite with the settings # from the child. - for p in map(inherit_parents, parents) + [obj]: + for p in list(map(inherit_parents, parents)) + [obj]: # child blats out type, title and description for key in ('type', 'title', 'description'): if p.get(key): @@ -250,12 +251,12 @@ def get_json_schema_object_fields(obj, enforce_title=False): tables.extend(res["tables"]) logger.debug("Done property %s" % key_name) - except Exception, e: + except Exception as e: e2 = Exception("Error reading property %s.%s: %s" % (obj_title, key_name, str(e))) # throw the new exception with the old stack trace, so that # we don't lose information about where the error occurred. - raise e2, None, sys.exc_info()[2] + raise e2.with_traceback(sys.exc_info()[2]) tables.insert(0, TypeTable(title=obj_title, rows=first_table_rows)) @@ -380,7 +381,7 @@ def get_example_for_schema(schema): if 'properties' not in schema: raise Exception('"object" property has neither properties nor example') res = OrderedDict() - for prop_name, prop in schema['properties'].iteritems(): + for prop_name, prop in schema['properties'].items(): logger.debug("Parsing property %r" % prop_name) prop_example = get_example_for_schema(prop) res[prop_name] = prop_example @@ -523,7 +524,7 @@ class MatrixUnits(Units): if param_loc == "path": path_template = path_template.replace( - "{%s}" % param_name, urllib.quote(example) + "{%s}" % param_name, quote(example) ) elif param_loc == "query": if type(example) == list: @@ -532,7 +533,7 @@ class MatrixUnits(Units): else: example_query_params.append((param_name, example)) - except Exception, e: + except Exception as e: raise Exception("Error handling parameter %s" % param_name, e) # endfor[param] good_response = None @@ -556,14 +557,14 @@ class MatrixUnits(Units): ) if "headers" in good_response: headers = TypeTable() - for (header_name, header) in good_response["headers"].iteritems(): + for (header_name, header) in good_response["headers"].items(): headers.add_row( TypeTableRow(key=header_name, title=header["type"], desc=header["description"]), ) endpoint["res_headers"] = headers query_string = "" if len( - example_query_params) == 0 else "?" + urllib.urlencode( + example_query_params) == 0 else "?" + urlencode( example_query_params) if example_body: endpoint["example"][ @@ -605,12 +606,12 @@ class MatrixUnits(Units): body_tables = req_body_tables[1:] endpoint_data['req_body_tables'].extend(body_tables) - except Exception, e: + except Exception as e: e2 = Exception( "Error decoding body of API endpoint %s %s: %s" % (endpoint_data["method"], endpoint_data["path"], e) ) - raise e2, None, sys.exc_info()[2] + raise e2.with_traceback(sys.exc_info()[2]) def load_swagger_apis(self): @@ -711,12 +712,12 @@ class MatrixUnits(Units): if filename != event_name: examples[event_name] = examples.get(event_name, []) examples[event_name].append(example) - except Exception, e: + except Exception as e: e2 = Exception("Error reading event example "+filepath+": "+ str(e)) # throw the new exception with the old stack trace, so that # we don't lose information about where the error occurred. - raise e2, None, sys.exc_info()[2] + raise e2.with_traceback(sys.exc_info()[2]) return examples @@ -730,12 +731,12 @@ class MatrixUnits(Units): filepath = os.path.join(path, filename) try: schemata[filename] = self.read_event_schema(filepath) - except Exception, e: + except Exception as e: e2 = Exception("Error reading event schema "+filepath+": "+ str(e)) # throw the new exception with the old stack trace, so that # we don't lose information about where the error occurred. - raise e2, None, sys.exc_info()[2] + raise e2.with_traceback(sys.exc_info()[2]) return schemata @@ -831,12 +832,42 @@ class MatrixUnits(Units): path = os.path.join(CHANGELOG_DIR, f) name = f[:-4] + # If there's a directory with the same name, we'll try to generate + # a towncrier changelog and prepend it to the general changelog. + tc_path = os.path.join(CHANGELOG_DIR, name) + tc_lines = [] + if os.path.isdir(tc_path): + logger.info("Generating towncrier changelog for: %s" % name) + p = subprocess.Popen( + ['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'], + cwd=tc_path, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + stdout, stderr = p.communicate() + if p.returncode != 0: + # Something broke - dump as much information as we can + logger.error("Towncrier exited with code %s" % p.returncode) + logger.error(stdout.decode('UTF-8')) + logger.error(stderr.decode('UTF-8')) + raw_log = "" + else: + raw_log = stdout.decode('UTF-8') + + # This is a bit of a hack, but it does mean that the log at least gets *something* + # to tell us it broke + if not raw_log.startswith("Unreleased Changes"): + logger.error("Towncrier appears to have failed to generate a changelog") + logger.error(raw_log) + raw_log = "" + tc_lines = raw_log.splitlines() + title_part = None changelog_lines = [] with open(path, "r") as f: lines = f.readlines() prev_line = None - for line in lines: + for line in (tc_lines + lines): if prev_line is None: prev_line = line continue @@ -852,7 +883,10 @@ class MatrixUnits(Units): # then bail out. changelog_lines.pop() break - changelog_lines.append(" " + line) + # Don't generate subheadings (we'll keep the title though) + if re.match("^[-]{3,}$", line.strip()): + continue + changelog_lines.append(" " + line + '\n') changelogs[name] = "".join(changelog_lines) return changelogs @@ -871,7 +905,7 @@ class MatrixUnits(Units): ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], stderr=null, cwd=cwd, - ).strip() + ).strip().decode('UTF-8') except subprocess.CalledProcessError: git_branch = "" try: @@ -879,7 +913,7 @@ class MatrixUnits(Units): ['git', 'describe', '--exact-match'], stderr=null, cwd=cwd, - ).strip() + ).strip().decode('UTF-8') git_tag = "tag=" + git_tag except subprocess.CalledProcessError: git_tag = "" @@ -888,7 +922,7 @@ class MatrixUnits(Units): ['git', 'rev-parse', '--short', 'HEAD'], stderr=null, cwd=cwd, - ).strip() + ).strip().decode('UTF-8') except subprocess.CalledProcessError: git_commit = "" try: @@ -897,7 +931,7 @@ class MatrixUnits(Units): ['git', 'describe', '--dirty=' + dirty_string, "--all"], stderr=null, cwd=cwd, - ).strip().endswith(dirty_string) + ).strip().decode('UTF-8').endswith(dirty_string) git_dirty = "dirty" if is_dirty else "" except subprocess.CalledProcessError: git_dirty = "" @@ -908,7 +942,7 @@ class MatrixUnits(Units): s for s in (git_branch, git_tag, git_commit, git_dirty,) if s - ).encode("ascii") + ).encode("ascii").decode('ascii') return { "string": git_version, "revision": git_commit diff --git a/scripts/test-and-build.sh b/scripts/test-and-build.sh index 7794f826..710b03dd 100755 --- a/scripts/test-and-build.sh +++ b/scripts/test-and-build.sh @@ -4,8 +4,13 @@ set -ex cd `dirname $0`/.. -virtualenv env +virtualenv -p python3 env . env/bin/activate + +# Print out the python versions for debugging purposes +python --version +pip --version + pip install -r scripts/requirements.txt # do sanity checks on the examples and swagger