From 34091038f39a332c70ac188992437ae7a2e79003 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 14:19:06 +0100 Subject: [PATCH 1/4] speculator: Tool to preview spec pull requests Not the prettiest thing in the world, and horribly unperformant, but useful :) --- scripts/speculator/README | 4 + scripts/speculator/main.go | 220 +++++++++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 scripts/speculator/README create mode 100644 scripts/speculator/main.go diff --git a/scripts/speculator/README b/scripts/speculator/README new file mode 100644 index 00000000..1da99cfb --- /dev/null +++ b/scripts/speculator/README @@ -0,0 +1,4 @@ +speculator allows you to preview pull requests to the matrix.org specification. + +To run it, you must install the `go` tool, and run: + `go run main.go` diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go new file mode 100644 index 00000000..59047e68 --- /dev/null +++ b/scripts/speculator/main.go @@ -0,0 +1,220 @@ +// speculator allows you to preview pull requests to the matrix.org specification. +// It serves two HTTP endpoints: +// - /spec/123 which renders the spec as html at pull request 123. +// - /diff/rst/123 which gives a diff of the spec's rst at pull request 123. +// It is currently woefully inefficient, and there is a lot of low hanging fruit for improvement. +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "math/rand" + "net/http" + "os" + "os/exec" + "path" + "strconv" + "strings" + "syscall" +) + +type PullRequest struct { + Base Commit + Head Commit + User User +} + +type Commit struct { + SHA string + Repo RequestRepo +} + +type RequestRepo struct { + CloneURL string `json:"clone_url"` +} + +type User struct { + Login string +} + +var ( + port = flag.Int("port", 9000, "Port on which to listen for HTTP") + allowedMembers map[string]bool +) + +func gitClone(url string) (string, error) { + dst := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10)) + cmd := exec.Command("git", "clone", url, dst) + err := cmd.Run() + if err != nil { + return "", fmt.Errorf("error cloning repo: %v", err) + } + return dst, nil +} + +func gitCheckout(path, sha string) error { + cmd := exec.Command("git", "checkout", sha) + cmd.Dir = path + err := cmd.Run() + if err != nil { + return fmt.Errorf("error checking out repo: %v", err) + } + return nil +} + +func lookupPullRequest(prNumber string) (PullRequest, error) { + resp, _ := http.Get("https://api.github.com/repos/matrix-org/matrix-doc/pulls/" + prNumber) + defer resp.Body.Close() + dec := json.NewDecoder(resp.Body) + var pr PullRequest + _ = dec.Decode(&pr) + return pr, nil +} + +func generate(dir string) error { + cmd := exec.Command("python", "gendoc.py", "--nodelete") + cmd.Dir = path.Join(dir, "scripts") + var b bytes.Buffer + cmd.Stderr = &b + err := cmd.Run() + if err != nil { + return fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()) + } + return nil +} + +func writeError(w http.ResponseWriter, err error) { + w.WriteHeader(500) + io.WriteString(w, fmt.Sprintf("%v\n", err)) +} + +// generateAt generates spec from repo at sha. +// Returns the path where the generation was done. +func generateAt(repo, sha string) (dst string, err error) { + dst, err = gitClone(repo) + if err != nil { + return + } + + if err = gitCheckout(dst, sha); err != nil { + return + } + + err = generate(dst) + return +} + +func serveSpec(w http.ResponseWriter, req *http.Request) { + parts := strings.Split(req.URL.Path, "/") + if len(parts) != 3 { + w.WriteHeader(400) + io.WriteString(w, fmt.Sprintf("Invalid path passed: %v expect /pull/123", req.URL.Path)) + return + } + + pr, err := lookupPullRequest(parts[2]) + if err != nil { + writeError(w, err) + return + } + + if !allowedMembers[pr.User.Login] { + w.WriteHeader(403) + io.WriteString(w, fmt.Sprintf("%q is not a trusted pull requester", pr.User.Login)) + return + } + + dst, err := generateAt(pr.Head.Repo.CloneURL, pr.Head.SHA) + defer os.RemoveAll(dst) + if err != nil { + writeError(w, err) + return + } + + b, err := ioutil.ReadFile(path.Join(dst, "scripts/gen/specification.html")) + if err != nil { + writeError(w, fmt.Errorf("Error reading spec: %v", err)) + return + } + w.Write(b) +} + +func serveRstDiff(w http.ResponseWriter, req *http.Request) { + parts := strings.Split(req.URL.Path, "/") + if len(parts) != 4 { + w.WriteHeader(400) + io.WriteString(w, fmt.Sprintf("Invalid path passed: %v expect /diff/rst/123", req.URL.Path)) + return + } + + pr, err := lookupPullRequest(parts[3]) + if err != nil { + writeError(w, err) + return + } + + if !allowedMembers[pr.User.Login] { + w.WriteHeader(403) + io.WriteString(w, fmt.Sprintf("%q is not a trusted pull requester", pr.User.Login)) + return + } + + base, err := generateAt(pr.Base.Repo.CloneURL, pr.Base.SHA) + defer os.RemoveAll(base) + if err != nil { + writeError(w, err) + return + } + + head, err := generateAt(pr.Head.Repo.CloneURL, pr.Head.SHA) + defer os.RemoveAll(head) + if err != nil { + writeError(w, err) + return + } + + diffCmd := exec.Command("diff", path.Join(base, "scripts", "tmp", "full_spec.rst"), path.Join(head, "scripts", "tmp", "full_spec.rst")) + var diff bytes.Buffer + diffCmd.Stdout = &diff + if err := ignoreExitCodeOne(diffCmd.Run()); err != nil { + writeError(w, fmt.Errorf("error running diff: %v", err)) + return + } + w.Write(diff.Bytes()) +} + +func ignoreExitCodeOne(err error) error { + if err == nil { + return err + } + + if exiterr, ok := err.(*exec.ExitError); ok { + if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { + if status.ExitStatus() == 1 { + return nil + } + } + } + return err +} + +func main() { + flag.Parse() + // It would be great to read this from github, but there's no convenient way to do so. + // Most of these memberships are "private", so would require some kind of auth. + allowedMembers = map[string]bool{ + "dbkr": true, + "erikjohnston": true, + "illicitonion": true, + "Kegsay": true, + "NegativeMjark": true, + } + http.HandleFunc("/spec/", serveSpec) + http.HandleFunc("/diff/rst/", serveRstDiff) + log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) +} From 708f28127057d26e7000f24752dd1a111e1c1d75 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 17:09:44 +0100 Subject: [PATCH 2/4] Add comments about auth --- scripts/speculator/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 59047e68..22fb2ff2 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -123,6 +123,8 @@ func serveSpec(w http.ResponseWriter, req *http.Request) { return } + // We're going to run whatever Python is specified in the pull request, which + // may do bad things, so only trust people we trust. if !allowedMembers[pr.User.Login] { w.WriteHeader(403) io.WriteString(w, fmt.Sprintf("%q is not a trusted pull requester", pr.User.Login)) @@ -158,6 +160,8 @@ func serveRstDiff(w http.ResponseWriter, req *http.Request) { return } + // We're going to run whatever Python is specified in the pull request, which + // may do bad things, so only trust people we trust. if !allowedMembers[pr.User.Login] { w.WriteHeader(403) io.WriteString(w, fmt.Sprintf("%q is not a trusted pull requester", pr.User.Login)) From ca7dd49dc600a8d2f3cc0dbccbfd7f81e15185a2 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 17:14:53 +0100 Subject: [PATCH 3/4] speculator: Add /healthz endpoint --- scripts/speculator/main.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 22fb2ff2..3775ad86 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -220,5 +220,12 @@ func main() { } http.HandleFunc("/spec/", serveSpec) http.HandleFunc("/diff/rst/", serveRstDiff) + http.HandleFunc("/healthz", serveText("ok")) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) } + +func serveText(s string) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, req *http.Request) { + io.WriteString(w, s) + } +} From bf318ffcd53e4b01ea20c4a3bbe566a653360815 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 11 Sep 2015 12:42:41 +0100 Subject: [PATCH 4/4] Add HTTP endpoints to README --- scripts/speculator/README | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/speculator/README b/scripts/speculator/README index 1da99cfb..ae00c55a 100644 --- a/scripts/speculator/README +++ b/scripts/speculator/README @@ -1,4 +1,8 @@ speculator allows you to preview pull requests to the matrix.org specification. +It serves two HTTP endpoints: + - /spec/123 which renders the spec as html at pull request 123. + - /diff/rst/123 which gives a diff of the spec's rst at pull request 123. + To run it, you must install the `go` tool, and run: `go run main.go`