Put each bit of spec in its own directory

I want to change the URLs for the spec sections on the website from
<version>/<section>.html to <section>/<version>.html, to better reflect how we
do the versioning.

This puts each bit of spec in its own directory, updates the index to point to
the right place, and fixes continuserv to deal with directories as well as
files.

This will probably require fixes to the speculator too, but I'll have to come
back to that.

scripts/continuserv/main.go

@@ -103,7 +103,7 @@ func makeWalker(base string, w *fsnotify.Watcher) filepath.WalkFunc {
 
 		// log.Printf("Adding watch on %s", path)
 		if err := w.Add(path); err != nil {
-			log.Fatalf("Failed to add watch: %v", err)
+			log.Fatalf("Failed to add watch on %s: %v", path, err)
 		}
 		return nil
 	}
@@ -166,19 +166,34 @@ func populateOnce(dir string) {
 		toServe.Store(bytesOrErr{nil, fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())})
 		return
 	}
-	fis, err := ioutil.ReadDir(path.Join(dir, "scripts", "gen"))
-	if err != nil {
-		toServe.Store(bytesOrErr{nil, err})
-		return
-	}
+
 	files := make(map[string][]byte)
-	for _, fi := range fis {
-		bytes, err := ioutil.ReadFile(path.Join(dir, "scripts", "gen", fi.Name()))
+	base := path.Join(dir, "scripts", "gen")
+	walker := func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+
+		rel, err := filepath.Rel(base, path)
 		if err != nil {
-			toServe.Store(bytesOrErr{nil, fmt.Errorf("error reading spec: %v", err)})
-			return
+			return fmt.Errorf("Failed to get relative path of %s: %v", path, err)
 		}
-		files[fi.Name()] = bytes
+
+		bytes, err := ioutil.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		files[rel] = bytes
+		return nil
+	}
+
+	err = filepath.Walk(base, walker)
+	if err != nil {
+		toServe.Store(bytesOrErr{nil, fmt.Errorf("error reading spec: %v", err)})
+		return
 	}
 	toServe.Store(bytesOrErr{files, nil})
 }

scripts/gendoc.py

@@ -6,6 +6,7 @@ import copy
 import fileinput
 import glob
 import os
+import os.path
 import re
 import shutil
 import subprocess
@@ -112,7 +113,7 @@ def load_with_adjusted_titles(filename, file_stream, title_level, title_styles):
             line_title_style,
             title_styles[adjusted_level]
         ))
-            
+
     return "".join(rst_lines)
 
 
@@ -277,16 +278,10 @@ def run_through_template(input_files, set_verbose, substitutions):
         cwd="../templating"
     )
 
-def get_build_targets(targets_listing):
-    with open(targets_listing, "r") as targ_file:
-        all_targets = yaml.load(targ_file.read())
-    return all_targets["targets"].keys()
-
-
 """
 Extract and resolve groups for the given target in the given targets listing.
 Args:
-    targets_listing (str): The path to a YAML file containing a list of targets
+    all_targets (dict): The parsed YAML file containing a list of targets
     target_name (str): The name of the target to extract from the listings.
 Returns:
     dict: Containing "filees" (a list of file paths), "relative_title_styles"
@@ -294,14 +289,12 @@ Returns:
           (a list of characters which represent the global title style to follow,
            with the top section title first, the second section second, and so on.)
 """
-def get_build_target(targets_listing, target_name):
+def get_build_target(all_targets, target_name):
     build_target = {
         "title_styles": [],
         "relative_title_styles": {},
         "files": []
     }
-    with open(targets_listing, "r") as targ_file:
-        all_targets = yaml.load(targ_file.read())
 
     build_target["title_styles"] = all_targets["title_styles"]
     build_target["relative_title_styles"] = all_targets["relative_title_styles"]
@@ -396,8 +389,11 @@ def cleanup_env():
 def main(targets, keep_intermediates, substitutions):
     prepare_env()
 
+    with open("../specification/targets.yaml", "r") as targ_file:
+        target_defs = yaml.load(targ_file.read())
+
     if targets == ["all"]:
-        targets = get_build_targets("../specification/targets.yaml") + ["howtos"]
+        targets = target_defs["targets"].keys() + ["howtos"]
 
     log("Building spec [target=%s]" % targets)
 
@@ -408,7 +404,7 @@ def main(targets, keep_intermediates, substitutions):
         if target_name == "howtos":
             shutil.copy("../supporting-docs/howtos/client-server.rst", templated_file)
         else:
-            target = get_build_target("../specification/targets.yaml", target_name)
+            target = get_build_target(target_defs, target_name)
             build_spec(target=target, out_filename=templated_file)
         templated_files.append(templated_file)
 
@@ -416,12 +412,26 @@ def main(targets, keep_intermediates, substitutions):
     run_through_template(templated_files, VERBOSE, substitutions)
 
     for target_name in targets:
+        target = target_defs["targets"].get(target_name)
+        version_label = None
+        if target:
+            version_label = target.get("version_label")
+            if version_label:
+                for old, new in substitutions.items():
+                    version_label = version_label.replace(old, new)
+
         templated_file = "tmp/templated_%s.rst" % (target_name,)
         rst_file = "tmp/spec_%s.rst" % (target_name,)
-        html_file = "gen/%s.html" % (target_name,)
+        if version_label:
+            d = os.path.join("gen", target_name)
+            if not os.path.exists(d):
+                os.mkdir(d)
+            html_file = os.path.join(d, "%s.html" % version_label)
+        else:
+            html_file = "gen/%s.html" % (target_name, )
 
         fix_relative_titles(
-            target=target, filename=templated_file,
+            target=target_defs, filename=templated_file,
             out_filename=rst_file,
         )
         rst2html(rst_file, html_file)

specification/targets.yaml

@@ -12,19 +12,24 @@ targets:
       - { 1: modules.rst }
       - { 2: feature_profiles.rst }
       - { 2: "group:modules" }  # reference a group of files
+    version_label: "%CLIENT_RELEASE_LABEL%"
   application_service:
     files:
       - application_service_api.rst
+    version_label: unstable
   server_server:
     files:
       - server_server_api.rst
       - { 1: event_signing.rst }
+    version_label: "%SERVER_RELEASE_LABEL%"
   identity_service:
     files:
       - identity_service_api.rst
+    version_label: unstable
   push_gateway:
     files:
       - push_gateway.rst
+    version_label: unstable
   appendices:
     files:
       - appendices.rst

templating/matrix_templates/units.py

@@ -541,25 +541,27 @@ class MatrixUnits(Units):
         return event_types
 
     def load_apis(self, substitutions):
+        cs_ver = substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable")
+        fed_ver = substitutions.get("%SERVER_RELEASE_LABEL%", "unstable")
         return {
             "rows": [{
-                "key": "`Client-Server API <client_server.html>`_",
-                "type": substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable"),
+                "key": "`Client-Server API <client_server/"+cs_ver+".html>`_",
+                "type": cs_ver,
                 "desc": "Interaction between clients and servers",
             }, {
-                "key": "`Server-Server API <server_server.html>`_",
-                "type": substitutions.get("%SERVER_RELEASE_LABEL%", "unstable"),
+                "key": "`Server-Server API <server_server/"+fed_ver+".html>`_",
+                "type": fed_ver,
                 "desc": "Federation between servers",
             }, {
-                "key": "`Application Service API <application_service.html>`_",
-                "type": substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable"),
+                "key": "`Application Service API <application_service/unstable.html>`_",
+                "type": "unstable",
                 "desc": "Privileged server plugins",
             }, {
-                "key": "`Identity Service API <identity_service.html>`_",
+                "key": "`Identity Service API <identity_service/unstable.html>`_",
                 "type": "unstable",
                 "desc": "Mapping of third party IDs with Matrix ID",
             }, {
-                "key": "`Push Gateway API <push_gateway.html>`_",
+                "key": "`Push Gateway API <push_gateway/unstable.html>`_",
                 "type": "unstable",
                 "desc": "Push notifications for Matrix events",
             }]