From 01f8173c84be2fb647c341add15b533eb7f3e867 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 5 May 2016 18:26:17 +0100 Subject: [PATCH] Put each bit of spec in its own directory I want to change the URLs for the spec sections on the website from /
.html to
/.html, to better reflect how we do the versioning. This puts each bit of spec in its own directory, updates the index to point to the right place, and fixes continuserv to deal with directories as well as files. This will probably require fixes to the speculator too, but I'll have to come back to that. --- scripts/continuserv/main.go | 37 +++++++++++++++++-------- scripts/gendoc.py | 40 +++++++++++++++++----------- specification/targets.yaml | 5 ++++ templating/matrix_templates/units.py | 18 +++++++------ 4 files changed, 66 insertions(+), 34 deletions(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 4d97d790..59a4cce2 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -103,7 +103,7 @@ func makeWalker(base string, w *fsnotify.Watcher) filepath.WalkFunc { // log.Printf("Adding watch on %s", path) if err := w.Add(path); err != nil { - log.Fatalf("Failed to add watch: %v", err) + log.Fatalf("Failed to add watch on %s: %v", path, err) } return nil } @@ -166,19 +166,34 @@ func populateOnce(dir string) { toServe.Store(bytesOrErr{nil, fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())}) return } - fis, err := ioutil.ReadDir(path.Join(dir, "scripts", "gen")) - if err != nil { - toServe.Store(bytesOrErr{nil, err}) - return - } + files := make(map[string][]byte) - for _, fi := range fis { - bytes, err := ioutil.ReadFile(path.Join(dir, "scripts", "gen", fi.Name())) + base := path.Join(dir, "scripts", "gen") + walker := func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + rel, err := filepath.Rel(base, path) if err != nil { - toServe.Store(bytesOrErr{nil, fmt.Errorf("error reading spec: %v", err)}) - return + return fmt.Errorf("Failed to get relative path of %s: %v", path, err) } - files[fi.Name()] = bytes + + bytes, err := ioutil.ReadFile(path) + if err != nil { + return err + } + files[rel] = bytes + return nil + } + + err = filepath.Walk(base, walker) + if err != nil { + toServe.Store(bytesOrErr{nil, fmt.Errorf("error reading spec: %v", err)}) + return } toServe.Store(bytesOrErr{files, nil}) } diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 7792308b..4aeb153a 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -6,6 +6,7 @@ import copy import fileinput import glob import os +import os.path import re import shutil import subprocess @@ -112,7 +113,7 @@ def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): line_title_style, title_styles[adjusted_level] )) - + return "".join(rst_lines) @@ -277,16 +278,10 @@ def run_through_template(input_files, set_verbose, substitutions): cwd="../templating" ) -def get_build_targets(targets_listing): - with open(targets_listing, "r") as targ_file: - all_targets = yaml.load(targ_file.read()) - return all_targets["targets"].keys() - - """ Extract and resolve groups for the given target in the given targets listing. Args: - targets_listing (str): The path to a YAML file containing a list of targets + all_targets (dict): The parsed YAML file containing a list of targets target_name (str): The name of the target to extract from the listings. Returns: dict: Containing "filees" (a list of file paths), "relative_title_styles" @@ -294,14 +289,12 @@ Returns: (a list of characters which represent the global title style to follow, with the top section title first, the second section second, and so on.) """ -def get_build_target(targets_listing, target_name): +def get_build_target(all_targets, target_name): build_target = { "title_styles": [], "relative_title_styles": {}, "files": [] } - with open(targets_listing, "r") as targ_file: - all_targets = yaml.load(targ_file.read()) build_target["title_styles"] = all_targets["title_styles"] build_target["relative_title_styles"] = all_targets["relative_title_styles"] @@ -396,8 +389,11 @@ def cleanup_env(): def main(targets, keep_intermediates, substitutions): prepare_env() + with open("../specification/targets.yaml", "r") as targ_file: + target_defs = yaml.load(targ_file.read()) + if targets == ["all"]: - targets = get_build_targets("../specification/targets.yaml") + ["howtos"] + targets = target_defs["targets"].keys() + ["howtos"] log("Building spec [target=%s]" % targets) @@ -408,7 +404,7 @@ def main(targets, keep_intermediates, substitutions): if target_name == "howtos": shutil.copy("../supporting-docs/howtos/client-server.rst", templated_file) else: - target = get_build_target("../specification/targets.yaml", target_name) + target = get_build_target(target_defs, target_name) build_spec(target=target, out_filename=templated_file) templated_files.append(templated_file) @@ -416,12 +412,26 @@ def main(targets, keep_intermediates, substitutions): run_through_template(templated_files, VERBOSE, substitutions) for target_name in targets: + target = target_defs["targets"].get(target_name) + version_label = None + if target: + version_label = target.get("version_label") + if version_label: + for old, new in substitutions.items(): + version_label = version_label.replace(old, new) + templated_file = "tmp/templated_%s.rst" % (target_name,) rst_file = "tmp/spec_%s.rst" % (target_name,) - html_file = "gen/%s.html" % (target_name,) + if version_label: + d = os.path.join("gen", target_name) + if not os.path.exists(d): + os.mkdir(d) + html_file = os.path.join(d, "%s.html" % version_label) + else: + html_file = "gen/%s.html" % (target_name, ) fix_relative_titles( - target=target, filename=templated_file, + target=target_defs, filename=templated_file, out_filename=rst_file, ) rst2html(rst_file, html_file) diff --git a/specification/targets.yaml b/specification/targets.yaml index fb18a78b..2555b36e 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -12,19 +12,24 @@ targets: - { 1: modules.rst } - { 2: feature_profiles.rst } - { 2: "group:modules" } # reference a group of files + version_label: "%CLIENT_RELEASE_LABEL%" application_service: files: - application_service_api.rst + version_label: unstable server_server: files: - server_server_api.rst - { 1: event_signing.rst } + version_label: "%SERVER_RELEASE_LABEL%" identity_service: files: - identity_service_api.rst + version_label: unstable push_gateway: files: - push_gateway.rst + version_label: unstable appendices: files: - appendices.rst diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index 81489f63..df347d40 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -541,25 +541,27 @@ class MatrixUnits(Units): return event_types def load_apis(self, substitutions): + cs_ver = substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable") + fed_ver = substitutions.get("%SERVER_RELEASE_LABEL%", "unstable") return { "rows": [{ - "key": "`Client-Server API `_", - "type": substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable"), + "key": "`Client-Server API `_", + "type": cs_ver, "desc": "Interaction between clients and servers", }, { - "key": "`Server-Server API `_", - "type": substitutions.get("%SERVER_RELEASE_LABEL%", "unstable"), + "key": "`Server-Server API `_", + "type": fed_ver, "desc": "Federation between servers", }, { - "key": "`Application Service API `_", - "type": substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable"), + "key": "`Application Service API `_", + "type": "unstable", "desc": "Privileged server plugins", }, { - "key": "`Identity Service API `_", + "key": "`Identity Service API `_", "type": "unstable", "desc": "Mapping of third party IDs with Matrix ID", }, { - "key": "`Push Gateway API `_", + "key": "`Push Gateway API `_", "type": "unstable", "desc": "Push notifications for Matrix events", }]