From e171acf01ffecc4a931712928ffca024cbdade8a Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 14:21:32 +0000 Subject: [PATCH 1/2] Split spec into page-per-section --- scripts/gendoc.py | 52 +++++++++++++------ specification/intro.rst | 10 ++++ .../modules/typing_notifications.rst | 6 +-- specification/targets.yaml | 11 +++- 4 files changed, 57 insertions(+), 22 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 28a11528..9b87cb30 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -276,6 +276,12 @@ def run_through_template(input, set_verbose): raise +def get_build_targets(targets_listing): + with open(targets_listing, "r") as targ_file: + all_targets = yaml.load(targ_file.read()) + return all_targets["targets"].keys() + + """ Extract and resolve groups for the given target in the given targets listing. Args: @@ -386,21 +392,34 @@ def cleanup_env(): shutil.rmtree("./tmp") -def main(target_name, keep_intermediates): +def main(requested_target_name, keep_intermediates): prepare_env() - log("Building spec [target=%s]" % target_name) - target = get_build_target("../specification/targets.yaml", target_name) - build_spec(target=target, out_filename="tmp/templated_spec.rst") - run_through_template("tmp/templated_spec.rst", VERBOSE) - fix_relative_titles( - target=target, filename="tmp/templated_spec.rst", - out_filename="tmp/full_spec.rst" - ) - shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") - run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this - rst2html("tmp/full_spec.rst", "gen/specification.html") - addAnchors("gen/specification.html") - rst2html("tmp/howto.rst", "gen/howtos.html") + log("Building spec [target=%s]" % requested_target_name) + + targets = [requested_target_name] + if requested_target_name == "all": + targets = get_build_targets("../specification/targets.yaml") + + for target_name in targets: + templated_file = "tmp/templated_%s.rst" % (target_name,) + rst_file = "tmp/spec_%s.rst" % (target_name,) + html_file = "gen/%s.html" % (target_name,) + + target = get_build_target("../specification/targets.yaml", target_name) + build_spec(target=target, out_filename=templated_file) + run_through_template(templated_file, VERBOSE) + fix_relative_titles( + target=target, filename=templated_file, + out_filename=rst_file, + ) + rst2html(rst_file, html_file) + addAnchors(html_file) + + if requested_target_name == "all": + shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") + run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this + rst2html("tmp/howto.rst", "gen/howtos.html") + if not keep_intermediates: cleanup_env() @@ -414,8 +433,9 @@ if __name__ == '__main__': help="Do not delete intermediate files. They will be found in tmp/" ) parser.add_argument( - "--target", "-t", default="main", - help="Specify the build target to build from specification/targets.yaml" + "--target", "-t", default="all", + help="Specify the build target to build from specification/targets.yaml. " + + "The value 'all' will build all of the targets therein." ) parser.add_argument( "--verbose", "-v", action="store_true", diff --git a/specification/intro.rst b/specification/intro.rst index 6ff7946c..8c08bf24 100644 --- a/specification/intro.rst +++ b/specification/intro.rst @@ -8,6 +8,16 @@ https://github.com/matrix-org/matrix-doc using https://github.com/matrix-org/matrix-doc/blob/master/scripts/gendoc.py as of revision ``{{git_version}}`` - https://github.com/matrix-org/matrix-doc/tree/{{git_rev}} +APIs +~~~~ +The following APIs are documented in this specification: + +- `Client-Server API `_ for writing Matrix clients. +- `Server-Server API `_ for writing servers which can federate with Matrix. +- `Application Service API `_ for writing privileged plugins to servers. + +There are also some `appendices `_. + Changelog ~~~~~~~~~ {{spec_changelog}} diff --git a/specification/modules/typing_notifications.rst b/specification/modules/typing_notifications.rst index d2253632..da383e73 100644 --- a/specification/modules/typing_notifications.rst +++ b/specification/modules/typing_notifications.rst @@ -5,10 +5,8 @@ Typing Notifications Users may wish to be informed when another user is typing in a room. This can be achieved using typing notifications. These are ephemeral events scoped to a -``room_id``. This means they do not form part of the `Event Graph`_ but still -have a ``room_id`` key. - -.. _Event Graph: `sect:event-graph`_ +``room_id``. This means they do not form part of the +`Event Graph `_ but still have a ``room_id`` key. Events ------ diff --git a/specification/targets.yaml b/specification/targets.yaml index 8e6a2ce0..cd0f6f41 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -1,16 +1,23 @@ targets: - main: # arbitrary name to identify this build target + index: files: # the sort order of files to cat - intro.rst + client_server: + files: - client_server_api.rst - { 1: events.rst } - { 1: event_signing.rst } - modules.rst - { 1: feature_profiles.rst } - { 1: "group:modules" } # reference a group of files + application_service: + files: - application_service_api.rst + server_server: + files: - server_server_api.rst - - identity_servers.rst + appendices: + files: - appendices.rst groups: # reusable blobs of files when prefixed with 'group:' modules: From 30ed918633cf47ceb2d5dac793b1e3a3e9a0d80f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 27 Nov 2015 14:53:58 +0000 Subject: [PATCH 2/2] speculator: Allow spec viewing for multi-page spec --- scripts/speculator/main.go | 94 +++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index abe77c74..3736672f 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -57,8 +57,8 @@ var ( port = flag.Int("port", 9000, "Port on which to listen for HTTP") includesDir = flag.String("includes_dir", "", "Directory containing include files for styling like matrix.org") allowedMembers map[string]bool - specCache *lru.Cache // string -> []byte - styledSpecCache *lru.Cache // string -> []byte + specCache *lru.Cache // string -> map[string][]byte filename -> contents + styledSpecCache *lru.Cache // string -> map[string][]byte filename -> contents ) func (u *User) IsTrusted() bool { @@ -105,10 +105,7 @@ func lookupPullRequest(url url.URL, pathPrefix string) (*PullRequest, error) { if !strings.HasPrefix(url.Path, pathPrefix+"/") { return nil, fmt.Errorf("invalid path passed: %s expect %s/123", url.Path, pathPrefix) } - prNumber := url.Path[len(pathPrefix)+1:] - if strings.Contains(prNumber, "/") { - return nil, fmt.Errorf("invalid path passed: %s expect %s/123", url.Path, pathPrefix) - } + prNumber := strings.Split(url.Path[len(pathPrefix)+1:], "/")[0] resp, err := http.Get(fmt.Sprintf("%s/%s", pullsPrefix, prNumber)) defer resp.Body.Close() @@ -196,6 +193,17 @@ func (s *server) getSHAOf(ref string) (string, error) { return strings.TrimSpace(b.String()), nil } +func extractPath(path, base string) string { + // Assume exactly one flat directory + max := strings.Count(base, "/") + 2 + parts := strings.SplitN(path, "/", max) + + if len(parts) < max || parts[max-1] == "" { + return "index.html" + } + return parts[max-1] +} + func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { var sha string @@ -206,7 +214,7 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { return } - if strings.ToLower(req.URL.Path) == "/spec/head" { + if strings.HasPrefix(strings.ToLower(req.URL.Path), "/spec/head") { // err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring. // This is to deal with cases like where github is down but we still want to serve the spec. if headSha, err := s.lookupHeadSHA(); headSha == "" { @@ -236,36 +244,59 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) { cache = styledSpecCache } + var pathToContent map[string][]byte + if cached, ok := cache.Get(sha); ok { - w.Write(cached.([]byte)) - return - } + pathToContent = cached.(map[string][]byte) + } else { + dst, err := s.generateAt(sha) + defer os.RemoveAll(dst) + if err != nil { + writeError(w, 500, err) + return + } - dst, err := s.generateAt(sha) - defer os.RemoveAll(dst) - if err != nil { - writeError(w, 500, err) - return - } + if styleLikeMatrixDotOrg { + cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir) + cmd.Dir = path.Join(dst, "scripts") + var b bytes.Buffer + cmd.Stderr = &b + if err := cmd.Run(); err != nil { + writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String())) + return + } + } - if styleLikeMatrixDotOrg { - cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir) - cmd.Dir = path.Join(dst, "scripts") - var b bytes.Buffer - cmd.Stderr = &b - if err := cmd.Run(); err != nil { - writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String())) - return + fis, err := ioutil.ReadDir(path.Join(dst, "scripts", "gen")) + if err != nil { + writeError(w, 500, fmt.Errorf("Error reading directory: %v", err)) + } + pathToContent = make(map[string][]byte) + for _, fi := range fis { + b, err := ioutil.ReadFile(path.Join(dst, "scripts", "gen", fi.Name())) + if err != nil { + writeError(w, 500, fmt.Errorf("Error reading spec: %v", err)) + return + } + pathToContent[fi.Name()] = b } + cache.Add(sha, pathToContent) } - b, err := ioutil.ReadFile(path.Join(dst, "scripts/gen/specification.html")) - if err != nil { - writeError(w, 500, fmt.Errorf("Error reading spec: %v", err)) + requestedPath := extractPath(req.URL.Path, "/spec/pr") + if b, ok := pathToContent[requestedPath]; ok { + w.Write(b) return } - w.Write(b) - cache.Add(sha, b) + if requestedPath == "index.html" { + // Fall back to single-page spec for old PRs + if b, ok := pathToContent["specification.html"]; ok { + w.Write(b) + return + } + } + w.WriteHeader(404) + w.Write([]byte("Not found")) } // lookupHeadSHA looks up what origin/master's HEAD SHA is. @@ -322,7 +353,7 @@ func (s *server) serveRSTDiff(w http.ResponseWriter, req *http.Request) { return } - diffCmd := exec.Command("diff", "-u", path.Join(base, "scripts", "tmp", "full_spec.rst"), path.Join(head, "scripts", "tmp", "full_spec.rst")) + diffCmd := exec.Command("diff", "-r", "-u", path.Join(base, "scripts", "tmp"), path.Join(head, "scripts", "tmp")) var diff bytes.Buffer diffCmd.Stdout = &diff if err := ignoreExitCodeOne(diffCmd.Run()); err != nil { @@ -366,7 +397,8 @@ func (s *server) serveHTMLDiff(w http.ResponseWriter, req *http.Request) { return } - cmd := exec.Command(htmlDiffer, path.Join(base, "scripts", "gen", "specification.html"), path.Join(head, "scripts", "gen", "specification.html")) + requestedPath := extractPath(req.URL.Path, "/diff/spec/pr") + cmd := exec.Command(htmlDiffer, path.Join(base, "scripts", "gen", requestedPath), path.Join(head, "scripts", "gen", requestedPath)) var b bytes.Buffer cmd.Stdout = &b if err := cmd.Run(); err != nil {