Merge pull request #195 from matrix-org/daniel/versionnumbers

Embed client and server release numbers

scripts/gendoc.py

@@ -250,7 +250,7 @@ def addAnchors(path):
             f.write(line + "\n")
 
 
-def run_through_template(input, set_verbose):
+def run_through_template(input, set_verbose, substitutions):
     tmpfile = './tmp/output'
     try:
         with open(tmpfile, 'w') as out:
@@ -260,6 +260,9 @@ def run_through_template(input, set_verbose):
                 "-o", "../scripts/tmp",
                 "../scripts/"+input
             ]
+            for k, v in substitutions.items():
+                args.append("--substitution=%s=%s" % (k, v))
+
             if set_verbose:
                 args.insert(2, "-v")
             log("EXEC: %s" % " ".join(args))
@@ -392,7 +395,7 @@ def cleanup_env():
     shutil.rmtree("./tmp")
 
 
-def main(requested_target_name, keep_intermediates):
+def main(requested_target_name, keep_intermediates, substitutions):
     prepare_env()
     log("Building spec [target=%s]" % requested_target_name)
 
@@ -407,7 +410,7 @@ def main(requested_target_name, keep_intermediates):
 
         target = get_build_target("../specification/targets.yaml", target_name)
         build_spec(target=target, out_filename=templated_file)
-        run_through_template(templated_file, VERBOSE)
+        run_through_template(templated_file, VERBOSE, substitutions)
         fix_relative_titles(
             target=target, filename=templated_file,
             out_filename=rst_file,
@@ -417,13 +420,21 @@ def main(requested_target_name, keep_intermediates):
 
     if requested_target_name == "all":
         shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst")
-        run_through_template("tmp/howto.rst", False)  # too spammy to mark -v on this
+        run_through_template("tmp/howto.rst", False, substitutions)  # too spammy to mark -v on this
         rst2html("tmp/howto.rst", "gen/howtos.html")
 
     if not keep_intermediates:
         cleanup_env()
 
 
+def extract_major(s):
+    major_version = s
+    match = re.match("^(r\d)+(\.\d+)?$", s)
+    if match:
+        major_version = match.group(1)
+    return major_version
+
+
 if __name__ == '__main__':
     parser = ArgumentParser(
         "gendoc.py - Generate the Matrix specification as HTML to the gen/ folder."
@@ -441,9 +452,23 @@ if __name__ == '__main__':
         "--verbose", "-v", action="store_true",
         help="Turn on verbose mode."
     )
+    parser.add_argument(
+        "--client_release", "-c", action="store", default="unstable",
+        help="The client-server release tag to generate, e.g. r1.2"
+    )
+    parser.add_argument(
+        "--server_release", "-s", action="store", default="unstable",
+        help="The server-server release tag to generate, e.g. r1.2"
+    )
     args = parser.parse_args()
     if not args.target:
         parser.print_help()
         sys.exit(1)
     VERBOSE = args.verbose
-    main(args.target, args.nodelete)
+    substitutions = {
+        "%CLIENT_RELEASE_LABEL%": args.client_release,
+        "%CLIENT_MAJOR_VERSION%": extract_major(args.client_release),
+        "%SERVER_RELEASE_LABEL%": args.server_release,
+        "%SERVER_MAJOR_VERSION%": extract_major(args.server_release),
+    }
+    main(args.target, args.nodelete, substitutions)

scripts/generate-http-docs.sh

@@ -4,6 +4,17 @@
 # It takes all of the swagger YAML files for the client-server API, and turns
 # them into API docs, with none of the narrative found in the rst files which
 # normally wrap these API docs.
+#
+# Optionally takes one positional argument, the label of the release, e.g. r1.2.
+# This falls back to "unstable" if unspecified.
+
+if [[ $# == 1 ]]; then
+  client_release=$1
+else
+  client_release="unstable"
+fi
+
+client_major_version="$(echo "${client_release}" | perl -pe 's/^(r\d+)\..*$/$1/g')"
 
 cd "$(dirname $0)"
 
@@ -23,5 +34,5 @@ for f in ../api/client-server/*.yaml; do
   echo "{{${f/-/_}}}" >> tmp/http_apis
 done
 
-(cd ../templating ; python build.py -i matrix_templates -o ../scripts/gen ../scripts/tmp/http_apis)
+(cd ../templating ; python build.py -i matrix_templates -o ../scripts/gen ../scripts/tmp/http_apis --substitution=%CLIENT_RELEASE_LABEL%="${client_release}" --substitution=%CLIENT_MAJOR_VERSION%="${client_major_version}")
 rst2html.py --stylesheet-path=$(echo css/*.css | tr ' ' ',') gen/http_apis > gen/http_apis.html

specification/intro.rst

@@ -1,30 +1,20 @@
 Matrix Specification
 ====================
 
-Version: {{spec_version}}
------------------------------
 This specification has been generated from
 https://github.com/matrix-org/matrix-doc using
 https://github.com/matrix-org/matrix-doc/blob/master/scripts/gendoc.py as of
-revision ``{{git_version}}`` - https://github.com/matrix-org/matrix-doc/tree/{{git_rev}}
+revision ``{{git_version}}`` -
+https://github.com/matrix-org/matrix-doc/tree/{{git_rev}}.
 
-APIs
-----
 The following APIs are documented in this specification:
 
-- `Client-Server API <client_server.html>`_ for writing Matrix clients.
+- `Client-Server API <client_server.html>`_ version %CLIENT_RELEASE_LABEL% for writing Matrix clients.
-- `Server-Server API <server_server.html>`_ for writing servers which can federate with Matrix.
+- `Server-Server API <server_server.html>`_ version %SERVER_RELEASE_LABEL% for writing servers which can federate with Matrix.
-- `Application Service API <application_service.html>`_ for writing privileged plugins to servers.
+- `Application Service API <application_service.html>`_ version %CLIENT_RELEASE_LABEL% for writing privileged plugins to servers.
 
 There are also some `appendices <appendices.html>`_.
 
-Changelog
----------
-{{spec_changelog}}
-
-For a full changelog, see 
-https://github.com/matrix-org/matrix-doc/blob/master/CHANGELOG.rst
-
 .. contents:: Table of Contents
 .. sectnum::
 

templating/build.py

@@ -44,6 +44,7 @@ import importlib
 import json
 import logging
 import os
+import re
 import sys
 from textwrap import TextWrapper
 
@@ -56,7 +57,7 @@ def check_unaccessed(name, store):
         log("Found %s unused %s keys." % (len(unaccessed_keys), name))
         log(unaccessed_keys)
 
-def main(input_module, file_stream=None, out_dir=None, verbose=False):
+def main(input_module, file_stream=None, out_dir=None, verbose=False, substitutions={}):
     if out_dir and not os.path.exists(out_dir):
         os.makedirs(out_dir)
 
@@ -167,6 +168,12 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False):
     temp = Template(temp_str)
     log("Creating output for: %s" % file_stream.name)
     output = create_from_template(temp, sections)
+
+    # Do these substitutions outside of the ordinary templating system because
+    # we want them to apply to things like the underlying swagger used to
+    # generate the templates, not just the top-level sections.
+    for old, new in substitutions.items():
+        output = output.replace(old, new)
     with open(
             os.path.join(out_dir, os.path.basename(file_stream.name)), "w"
             ) as f:
@@ -209,6 +216,10 @@ if __name__ == '__main__':
         "--verbose", "-v", action="store_true",
         help="Turn on verbose mode."
     )
+    parser.add_argument(
+        "--substitution", action="append",
+        help="Substitutions to apply to the generated output, of form NEEDLE=REPLACEMENT."
+    )
     args = parser.parse_args()
 
     if args.verbose:
@@ -226,7 +237,14 @@ if __name__ == '__main__':
         parser.print_help()
         sys.exit(1)
 
+    substitutions = {}
+    for substitution in args.substitution:
+        parts = substitution.split("=", 1)
+        if len(parts) != 2:
+            raise Exception("Invalid substitution")
+        substitutions[parts[0]] = parts[1]
+
     main(
         args.input, file_stream=args.file, out_dir=args.out_directory,
-        verbose=args.verbose
+        substitutions=substitutions, verbose=args.verbose
     )

templating/matrix_templates/sections.py

@@ -15,10 +15,6 @@ class MatrixSections(Sections):
     def render_git_rev(self):
         return self.units.get("git_version")["revision"]
 
-    def render_spec_version(self):
-        spec_meta = self.units.get("spec_meta")
-        return spec_meta["version"]
-
     def render_spec_changelog(self):
         spec_meta = self.units.get("spec_meta")
         return spec_meta["changelog"]

templating/matrix_templates/units.py

@@ -18,8 +18,8 @@ import urllib
 import yaml
 
 HTTP_APIS = "../api/client-server"
-V1_EVENT_EXAMPLES = "../event-schemas/examples"
+EVENT_EXAMPLES = "../event-schemas/examples"
-V1_EVENT_SCHEMA = "../event-schemas/schema"
+EVENT_SCHEMA = "../event-schemas/schema"
 CORE_EVENT_SCHEMA = "../event-schemas/schema/core-event-schema"
 CHANGELOG = "../CHANGELOG.rst"
 TARGETS = "../specification/targets.yaml"
@@ -605,7 +605,7 @@ class MatrixUnits(Units):
         return event_types
 
     def load_event_examples(self):
-        path = V1_EVENT_EXAMPLES
+        path = EVENT_EXAMPLES
         examples = {}
         for filename in os.listdir(path):
             if not filename.startswith("m."):
@@ -622,7 +622,7 @@ class MatrixUnits(Units):
         return examples
 
     def load_event_schemas(self):
-        path = V1_EVENT_SCHEMA
+        path = EVENT_SCHEMA
         schemata = {}
 
         for filename in os.listdir(path):
@@ -714,7 +714,6 @@ class MatrixUnits(Units):
     def load_spec_meta(self):
         path = CHANGELOG
         title_part = None
-        version = None
         changelog_lines = []
         with open(path, "r") as f:
             prev_line = None
@@ -738,19 +737,11 @@ class MatrixUnits(Units):
                         break
                     changelog_lines.append(line)
 
-        # parse out version from title
+        self.log("Title part: %s Changelog line count: %s" % (
-        for word in title_part.split():
+            title_part, len(changelog_lines)
-            if re.match("^v[0-9\.]+$", word):
-                version = word[1:]  # strip the 'v'
-
-        self.log("Version: %s Title part: %s Changelog line count: %s" % (
-            version, title_part, len(changelog_lines)
         ))
-        if not version or len(changelog_lines) == 0:
-            raise Exception("Failed to parse CHANGELOG.rst")
 
         return {
-            "version": version,
             "changelog": "".join(changelog_lines)
         }