Merge pull request #326 from matrix-org/rav/more_spec_versioning

Put each bit of spec in its own directory

scripts/continuserv/main.go

@@ -103,7 +103,7 @@ func makeWalker(base string, w *fsnotify.Watcher) filepath.WalkFunc {
 
 		// log.Printf("Adding watch on %s", path)
 		if err := w.Add(path); err != nil {
-			log.Fatalf("Failed to add watch: %v", err)
+			log.Fatalf("Failed to add watch on %s: %v", path, err)
 		}
 		return nil
 	}
@@ -166,19 +166,34 @@ func populateOnce(dir string) {
 		toServe.Store(bytesOrErr{nil, fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())})
 		return
 	}
-	fis, err := ioutil.ReadDir(path.Join(dir, "scripts", "gen"))
-	if err != nil {
-		toServe.Store(bytesOrErr{nil, err})
-		return
-	}
+
 	files := make(map[string][]byte)
-	for _, fi := range fis {
-		bytes, err := ioutil.ReadFile(path.Join(dir, "scripts", "gen", fi.Name()))
+	base := path.Join(dir, "scripts", "gen")
+	walker := func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+
+		rel, err := filepath.Rel(base, path)
 		if err != nil {
-			toServe.Store(bytesOrErr{nil, fmt.Errorf("error reading spec: %v", err)})
-			return
+			return fmt.Errorf("Failed to get relative path of %s: %v", path, err)
 		}
-		files[fi.Name()] = bytes
+
+		bytes, err := ioutil.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		files[rel] = bytes
+		return nil
+	}
+
+	err = filepath.Walk(base, walker)
+	if err != nil {
+		toServe.Store(bytesOrErr{nil, fmt.Errorf("error reading spec: %v", err)})
+		return
 	}
 	toServe.Store(bytesOrErr{files, nil})
 }

scripts/gendoc.py

@@ -6,6 +6,7 @@ import copy
 import fileinput
 import glob
 import os
+import os.path
 import re
 import shutil
 import subprocess
@@ -112,7 +113,7 @@ def load_with_adjusted_titles(filename, file_stream, title_level, title_styles):
             line_title_style,
             title_styles[adjusted_level]
         ))
-            
+
     return "".join(rst_lines)
 
 
@@ -277,16 +278,10 @@ def run_through_template(input_files, set_verbose, substitutions):
         cwd="../templating"
     )
 
-def get_build_targets(targets_listing):
-    with open(targets_listing, "r") as targ_file:
-        all_targets = yaml.load(targ_file.read())
-    return all_targets["targets"].keys()
-
-
 """
 Extract and resolve groups for the given target in the given targets listing.
 Args:
-    targets_listing (str): The path to a YAML file containing a list of targets
+    all_targets (dict): The parsed YAML file containing a list of targets
     target_name (str): The name of the target to extract from the listings.
 Returns:
     dict: Containing "filees" (a list of file paths), "relative_title_styles"
@@ -294,14 +289,12 @@ Returns:
           (a list of characters which represent the global title style to follow,
            with the top section title first, the second section second, and so on.)
 """
-def get_build_target(targets_listing, target_name):
+def get_build_target(all_targets, target_name):
     build_target = {
         "title_styles": [],
         "relative_title_styles": {},
         "files": []
     }
-    with open(targets_listing, "r") as targ_file:
-        all_targets = yaml.load(targ_file.read())
 
     build_target["title_styles"] = all_targets["title_styles"]
     build_target["relative_title_styles"] = all_targets["relative_title_styles"]
@@ -396,8 +389,11 @@ def cleanup_env():
 def main(targets, keep_intermediates, substitutions):
     prepare_env()
 
+    with open("../specification/targets.yaml", "r") as targ_file:
+        target_defs = yaml.load(targ_file.read())
+
     if targets == ["all"]:
-        targets = get_build_targets("../specification/targets.yaml") + ["howtos"]
+        targets = target_defs["targets"].keys() + ["howtos"]
 
     log("Building spec [target=%s]" % targets)
 
@@ -408,7 +404,7 @@ def main(targets, keep_intermediates, substitutions):
         if target_name == "howtos":
             shutil.copy("../supporting-docs/howtos/client-server.rst", templated_file)
         else:
-            target = get_build_target("../specification/targets.yaml", target_name)
+            target = get_build_target(target_defs, target_name)
             build_spec(target=target, out_filename=templated_file)
         templated_files.append(templated_file)
 
@@ -416,12 +412,26 @@ def main(targets, keep_intermediates, substitutions):
     run_through_template(templated_files, VERBOSE, substitutions)
 
     for target_name in targets:
+        target = target_defs["targets"].get(target_name)
+        version_label = None
+        if target:
+            version_label = target.get("version_label")
+            if version_label:
+                for old, new in substitutions.items():
+                    version_label = version_label.replace(old, new)
+
         templated_file = "tmp/templated_%s.rst" % (target_name,)
         rst_file = "tmp/spec_%s.rst" % (target_name,)
-        html_file = "gen/%s.html" % (target_name,)
+        if version_label:
+            d = os.path.join("gen", target_name)
+            if not os.path.exists(d):
+                os.mkdir(d)
+            html_file = os.path.join(d, "%s.html" % version_label)
+        else:
+            html_file = "gen/%s.html" % (target_name, )
 
         fix_relative_titles(
-            target=target, filename=templated_file,
+            target=target_defs, filename=templated_file,
             out_filename=rst_file,
         )
         rst2html(rst_file, html_file)

scripts/speculator/main.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
@@ -353,29 +354,44 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) {
 			return
 		}
 
-		if styleLikeMatrixDotOrg {
-			cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir)
-			cmd.Dir = path.Join(dst, "scripts")
-			var b bytes.Buffer
-			cmd.Stderr = &b
-			if err := cmd.Run(); err != nil {
-				writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String()))
-				return
+		pathToContent = make(map[string][]byte)
+		scriptsdir := path.Join(dst, "scripts")
+		base := path.Join(scriptsdir, "gen")
+		walker := func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				return nil
 			}
-		}
 
-		fis, err := ioutil.ReadDir(path.Join(dst, "scripts", "gen"))
-		if err != nil {
-			writeError(w, 500, fmt.Errorf("Error reading directory: %v", err))
-		}
-		pathToContent = make(map[string][]byte)
-		for _, fi := range fis {
-			b, err := ioutil.ReadFile(path.Join(dst, "scripts", "gen", fi.Name()))
+			rel, err := filepath.Rel(base, path)
+			if err != nil {
+				return fmt.Errorf("Failed to get relative path of %s: %v", path, err)
+			}
+
+			if styleLikeMatrixDotOrg {
+				cmd := exec.Command("./add-matrix-org-stylings.pl", *includesDir, path)
+				cmd.Dir = scriptsdir
+				var b bytes.Buffer
+				cmd.Stderr = &b
+				if err := cmd.Run(); err != nil {
+					return fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String())
+				}
+			}
+
+			bytes, err := ioutil.ReadFile(path)
 			if err != nil {
-				writeError(w, 500, fmt.Errorf("Error reading spec: %v", err))
-				return
+				return fmt.Errorf("Error reading spec: %v", err)
 			}
-			pathToContent[fi.Name()] = b
+			pathToContent[rel] = bytes
+			return nil
+		}
+
+		err = filepath.Walk(base, walker)
+		if err != nil {
+			writeError(w, 500, err)
+			return
 		}
 		cache.Add(sha, pathToContent)
 	}
@@ -499,13 +515,15 @@ func (s *server) serveHTMLDiff(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 	cmd := exec.Command(htmlDiffer, path.Join(base, "scripts", "gen", requestedPath), path.Join(head, "scripts", "gen", requestedPath))
-	var b bytes.Buffer
-	cmd.Stdout = &b
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
 	if err := cmd.Run(); err != nil {
-		writeError(w, 500, fmt.Errorf("error running HTML differ: %v", err))
+		writeError(w, 500, fmt.Errorf("error running HTML differ: %v\nOutput:\n%v", err, stderr.String()))
 		return
 	}
-	w.Write(b.Bytes())
+	w.Write(stdout.Bytes())
 }
 
 func findHTMLDiffer() (string, error) {

specification/targets.yaml

@@ -12,19 +12,24 @@ targets:
       - { 1: modules.rst }
       - { 2: feature_profiles.rst }
       - { 2: "group:modules" }  # reference a group of files
+    version_label: "%CLIENT_RELEASE_LABEL%"
   application_service:
     files:
       - application_service_api.rst
+    version_label: unstable
   server_server:
     files:
       - server_server_api.rst
       - { 1: event_signing.rst }
+    version_label: "%SERVER_RELEASE_LABEL%"
   identity_service:
     files:
       - identity_service_api.rst
+    version_label: unstable
   push_gateway:
     files:
       - push_gateway.rst
+    version_label: unstable
   appendices:
     files:
       - appendices.rst

templating/matrix_templates/units.py

@@ -541,25 +541,27 @@ class MatrixUnits(Units):
         return event_types
 
     def load_apis(self, substitutions):
+        cs_ver = substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable")
+        fed_ver = substitutions.get("%SERVER_RELEASE_LABEL%", "unstable")
         return {
             "rows": [{
-                "key": "`Client-Server API <client_server.html>`_",
-                "type": substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable"),
+                "key": "`Client-Server API <client_server/"+cs_ver+".html>`_",
+                "type": cs_ver,
                 "desc": "Interaction between clients and servers",
             }, {
-                "key": "`Server-Server API <server_server.html>`_",
-                "type": substitutions.get("%SERVER_RELEASE_LABEL%", "unstable"),
+                "key": "`Server-Server API <server_server/"+fed_ver+".html>`_",
+                "type": fed_ver,
                 "desc": "Federation between servers",
             }, {
-                "key": "`Application Service API <application_service.html>`_",
-                "type": substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable"),
+                "key": "`Application Service API <application_service/unstable.html>`_",
+                "type": "unstable",
                 "desc": "Privileged server plugins",
             }, {
-                "key": "`Identity Service API <identity_service.html>`_",
+                "key": "`Identity Service API <identity_service/unstable.html>`_",
                 "type": "unstable",
                 "desc": "Mapping of third party IDs with Matrix ID",
             }, {
-                "key": "`Push Gateway API <push_gateway.html>`_",
+                "key": "`Push Gateway API <push_gateway/unstable.html>`_",
                 "type": "unstable",
                 "desc": "Push notifications for Matrix events",
             }]