Update the speculator to understand spec subdirs

Fix the speculator so that it doesn't blow up when it finds subdirs in the gen
directory.

(It doesn't handle the html diff very well in the case that the subdirs don't
match, but it's hard to do much about that)

scripts/speculator/main.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
@@ -353,30 +354,45 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) {
 			return
 		}
 
+		pathToContent = make(map[string][]byte)
+		scriptsdir := path.Join(dst, "scripts")
+		base := path.Join(scriptsdir, "gen")
+		walker := func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				return nil
+			}
+
+			rel, err := filepath.Rel(base, path)
+			if err != nil {
+				return fmt.Errorf("Failed to get relative path of %s: %v", path, err)
+			}
+
 			if styleLikeMatrixDotOrg {
-			cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir)
-			cmd.Dir = path.Join(dst, "scripts")
+				cmd := exec.Command("./add-matrix-org-stylings.pl", *includesDir, path)
+				cmd.Dir = scriptsdir
 				var b bytes.Buffer
 				cmd.Stderr = &b
 				if err := cmd.Run(); err != nil {
-				writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String()))
-				return
+					return fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String())
 				}
 			}
 
-		fis, err := ioutil.ReadDir(path.Join(dst, "scripts", "gen"))
+			bytes, err := ioutil.ReadFile(path)
 			if err != nil {
-			writeError(w, 500, fmt.Errorf("Error reading directory: %v", err))
+				return fmt.Errorf("Error reading spec: %v", err)
 			}
-		pathToContent = make(map[string][]byte)
-		for _, fi := range fis {
-			b, err := ioutil.ReadFile(path.Join(dst, "scripts", "gen", fi.Name()))
+			pathToContent[rel] = bytes
+			return nil
+		}
+
+		err = filepath.Walk(base, walker)
 		if err != nil {
-				writeError(w, 500, fmt.Errorf("Error reading spec: %v", err))
+			writeError(w, 500, err)
 			return
 		}
-			pathToContent[fi.Name()] = b
-		}
 		cache.Add(sha, pathToContent)
 	}
 
@@ -499,13 +515,15 @@ func (s *server) serveHTMLDiff(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 	cmd := exec.Command(htmlDiffer, path.Join(base, "scripts", "gen", requestedPath), path.Join(head, "scripts", "gen", requestedPath))
-	var b bytes.Buffer
-	cmd.Stdout = &b
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
 	if err := cmd.Run(); err != nil {
-		writeError(w, 500, fmt.Errorf("error running HTML differ: %v", err))
+		writeError(w, 500, fmt.Errorf("error running HTML differ: %v\nOutput:\n%v", err, stderr.String()))
 		return
 	}
-	w.Write(b.Bytes())
+	w.Write(stdout.Bytes())
 }
 
 func findHTMLDiffer() (string, error) {