You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
489 lines
13 KiB
Go
489 lines
13 KiB
Go
// speculator allows you to preview pull requests to the matrix.org specification.
|
|
// It serves the following HTTP endpoints:
|
|
// - / lists open pull requests
|
|
// - /spec/123 which renders the spec as html at pull request 123.
|
|
// - /diff/rst/123 which gives a diff of the spec's rst at pull request 123.
|
|
// - /diff/html/123 which gives a diff of the spec's HTML at pull request 123.
|
|
// It is currently woefully inefficient, and there is a lot of low hanging fruit for improvement.
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"log"
|
|
"math/rand"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"os/exec"
|
|
"path"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/hashicorp/golang-lru"
|
|
)
|
|
|
|
type PullRequest struct {
|
|
Number int
|
|
Base Commit
|
|
Head Commit
|
|
Title string
|
|
User User
|
|
HTMLURL string `json:"html_url"`
|
|
}
|
|
|
|
type Commit struct {
|
|
SHA string
|
|
Repo RequestRepo
|
|
}
|
|
|
|
type RequestRepo struct {
|
|
CloneURL string `json:"clone_url"`
|
|
}
|
|
|
|
type User struct {
|
|
Login string
|
|
HTMLURL string `json:"html_url"`
|
|
}
|
|
|
|
var (
|
|
port = flag.Int("port", 9000, "Port on which to listen for HTTP")
|
|
includesDir = flag.String("includes_dir", "", "Directory containing include files for styling like matrix.org")
|
|
allowedMembers map[string]bool
|
|
specCache *lru.Cache // string -> []byte
|
|
styledSpecCache *lru.Cache // string -> []byte
|
|
)
|
|
|
|
func (u *User) IsTrusted() bool {
|
|
return allowedMembers[u.Login]
|
|
}
|
|
|
|
const (
|
|
pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls"
|
|
matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git"
|
|
permissionsOwnerFull = 0700
|
|
)
|
|
|
|
func gitClone(url string, shared bool) (string, error) {
|
|
directory := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10))
|
|
if err := os.MkdirAll(directory, permissionsOwnerFull); err != nil {
|
|
return "", fmt.Errorf("error making directory %s: %v", directory, err)
|
|
}
|
|
args := []string{"clone", url, directory}
|
|
if shared {
|
|
args = append(args, "--shared")
|
|
}
|
|
if err := runGitCommand(directory, args); err != nil {
|
|
return "", err
|
|
}
|
|
return directory, nil
|
|
}
|
|
|
|
func gitCheckout(path, sha string) error {
|
|
return runGitCommand(path, []string{"checkout", sha})
|
|
}
|
|
|
|
func runGitCommand(path string, args []string) error {
|
|
cmd := exec.Command("git", args...)
|
|
cmd.Dir = path
|
|
var b bytes.Buffer
|
|
cmd.Stderr = &b
|
|
if err := cmd.Run(); err != nil {
|
|
return fmt.Errorf("error running %q: %v (stderr: %s)", strings.Join(cmd.Args, " "), err, b.String())
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func lookupPullRequest(url url.URL, pathPrefix string) (*PullRequest, error) {
|
|
if !strings.HasPrefix(url.Path, pathPrefix+"/") {
|
|
return nil, fmt.Errorf("invalid path passed: %s expect %s/123", url.Path, pathPrefix)
|
|
}
|
|
prNumber := url.Path[len(pathPrefix)+1:]
|
|
if strings.Contains(prNumber, "/") {
|
|
return nil, fmt.Errorf("invalid path passed: %s expect %s/123", url.Path, pathPrefix)
|
|
}
|
|
|
|
resp, err := http.Get(fmt.Sprintf("%s/%s", pullsPrefix, prNumber))
|
|
defer resp.Body.Close()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error getting pulls: %v", err)
|
|
}
|
|
dec := json.NewDecoder(resp.Body)
|
|
var pr PullRequest
|
|
if err := dec.Decode(&pr); err != nil {
|
|
return nil, fmt.Errorf("error decoding pulls: %v", err)
|
|
}
|
|
return &pr, nil
|
|
}
|
|
|
|
func generate(dir string) error {
|
|
cmd := exec.Command("python", "gendoc.py", "--nodelete")
|
|
cmd.Dir = path.Join(dir, "scripts")
|
|
var b bytes.Buffer
|
|
cmd.Stderr = &b
|
|
if err := cmd.Run(); err != nil {
|
|
return fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func writeError(w http.ResponseWriter, code int, err error) {
|
|
w.Header().Set("Content-Type", "text/plain")
|
|
w.WriteHeader(code)
|
|
io.WriteString(w, fmt.Sprintf("%v\n", err))
|
|
}
|
|
|
|
type server struct {
|
|
mu sync.Mutex // Must be locked around any git command on matrixDocCloneURL
|
|
matrixDocCloneURL string
|
|
}
|
|
|
|
func (s *server) updateBase() error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return runGitCommand(s.matrixDocCloneURL, []string{"fetch"})
|
|
}
|
|
|
|
// canCheckout returns whether a given sha can currently be checked out from s.matrixDocCloneURL.
|
|
func (s *server) canCheckout(sha string) bool {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return runGitCommand(s.matrixDocCloneURL, []string{"cat-file", "-e", sha + "^{commit}"}) == nil
|
|
}
|
|
|
|
// generateAt generates spec from repo at sha.
|
|
// Returns the path where the generation was done.
|
|
func (s *server) generateAt(sha string) (dst string, err error) {
|
|
if !s.canCheckout(sha) {
|
|
err = s.updateBase()
|
|
if err != nil {
|
|
return
|
|
}
|
|
}
|
|
s.mu.Lock()
|
|
dst, err = gitClone(s.matrixDocCloneURL, true)
|
|
s.mu.Unlock()
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if err = gitCheckout(dst, sha); err != nil {
|
|
return
|
|
}
|
|
|
|
err = generate(dst)
|
|
return
|
|
}
|
|
|
|
func (s *server) getSHAOf(ref string) (string, error) {
|
|
cmd := exec.Command("git", "rev-list", ref, "-n1")
|
|
cmd.Dir = path.Join(s.matrixDocCloneURL)
|
|
var b bytes.Buffer
|
|
cmd.Stdout = &b
|
|
s.mu.Lock()
|
|
err := cmd.Run()
|
|
s.mu.Unlock()
|
|
if err != nil {
|
|
return "", fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())
|
|
}
|
|
return strings.TrimSpace(b.String()), nil
|
|
}
|
|
|
|
func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) {
|
|
var sha string
|
|
|
|
var styleLikeMatrixDotOrg = req.URL.Query().Get("matrixdotorgstyle") != ""
|
|
|
|
if styleLikeMatrixDotOrg && *includesDir == "" {
|
|
writeError(w, 500, fmt.Errorf("Cannot style like matrix.org - no include dir specified"))
|
|
return
|
|
}
|
|
|
|
if strings.ToLower(req.URL.Path) == "/spec/head" {
|
|
// err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring.
|
|
// This is to deal with cases like where github is down but we still want to serve the spec.
|
|
if headSha, err := s.lookupHeadSHA(); headSha == "" {
|
|
writeError(w, 500, err)
|
|
return
|
|
} else {
|
|
sha = headSha
|
|
}
|
|
} else {
|
|
pr, err := lookupPullRequest(*req.URL, "/spec")
|
|
if err != nil {
|
|
writeError(w, 400, err)
|
|
return
|
|
}
|
|
|
|
// We're going to run whatever Python is specified in the pull request, which
|
|
// may do bad things, so only trust people we trust.
|
|
if err := checkAuth(pr); err != nil {
|
|
writeError(w, 403, err)
|
|
return
|
|
}
|
|
sha = pr.Head.SHA
|
|
}
|
|
|
|
var cache = specCache
|
|
if styleLikeMatrixDotOrg {
|
|
cache = styledSpecCache
|
|
}
|
|
|
|
if cached, ok := cache.Get(sha); ok {
|
|
w.Write(cached.([]byte))
|
|
return
|
|
}
|
|
|
|
dst, err := s.generateAt(sha)
|
|
defer os.RemoveAll(dst)
|
|
if err != nil {
|
|
writeError(w, 500, err)
|
|
return
|
|
}
|
|
|
|
if styleLikeMatrixDotOrg {
|
|
cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir)
|
|
cmd.Dir = path.Join(dst, "scripts")
|
|
var b bytes.Buffer
|
|
cmd.Stderr = &b
|
|
if err := cmd.Run(); err != nil {
|
|
writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String()))
|
|
return
|
|
}
|
|
}
|
|
|
|
b, err := ioutil.ReadFile(path.Join(dst, "scripts/gen/specification.html"))
|
|
if err != nil {
|
|
writeError(w, 500, fmt.Errorf("Error reading spec: %v", err))
|
|
return
|
|
}
|
|
w.Write(b)
|
|
cache.Add(sha, b)
|
|
}
|
|
|
|
// lookupHeadSHA looks up what origin/master's HEAD SHA is.
|
|
// It attempts to `git fetch` before doing so.
|
|
// If this fails, it may still return a stale sha, but will also return an error.
|
|
func (s *server) lookupHeadSHA() (sha string, retErr error) {
|
|
retErr = s.updateBase()
|
|
if retErr != nil {
|
|
log.Printf("Error fetching: %v, attempting to fall back to current known value", retErr)
|
|
}
|
|
originHead, err := s.getSHAOf("origin/master")
|
|
if err != nil {
|
|
retErr = err
|
|
}
|
|
sha = originHead
|
|
if retErr != nil && originHead != "" {
|
|
log.Printf("Successfully fell back to possibly stale sha: %s", sha)
|
|
}
|
|
return
|
|
}
|
|
|
|
func checkAuth(pr *PullRequest) error {
|
|
if !pr.User.IsTrusted() {
|
|
return fmt.Errorf("%q is not a trusted pull requester", pr.User.Login)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *server) serveRSTDiff(w http.ResponseWriter, req *http.Request) {
|
|
pr, err := lookupPullRequest(*req.URL, "/diff/rst")
|
|
if err != nil {
|
|
writeError(w, 400, err)
|
|
return
|
|
}
|
|
|
|
// We're going to run whatever Python is specified in the pull request, which
|
|
// may do bad things, so only trust people we trust.
|
|
if err := checkAuth(pr); err != nil {
|
|
writeError(w, 403, err)
|
|
return
|
|
}
|
|
|
|
base, err := s.generateAt(pr.Base.SHA)
|
|
defer os.RemoveAll(base)
|
|
if err != nil {
|
|
writeError(w, 500, err)
|
|
return
|
|
}
|
|
|
|
head, err := s.generateAt(pr.Head.SHA)
|
|
defer os.RemoveAll(head)
|
|
if err != nil {
|
|
writeError(w, 500, err)
|
|
return
|
|
}
|
|
|
|
diffCmd := exec.Command("diff", "-u", path.Join(base, "scripts", "tmp", "full_spec.rst"), path.Join(head, "scripts", "tmp", "full_spec.rst"))
|
|
var diff bytes.Buffer
|
|
diffCmd.Stdout = &diff
|
|
if err := ignoreExitCodeOne(diffCmd.Run()); err != nil {
|
|
writeError(w, 500, fmt.Errorf("error running diff: %v", err))
|
|
return
|
|
}
|
|
w.Write(diff.Bytes())
|
|
}
|
|
|
|
func (s *server) serveHTMLDiff(w http.ResponseWriter, req *http.Request) {
|
|
pr, err := lookupPullRequest(*req.URL, "/diff/html")
|
|
if err != nil {
|
|
writeError(w, 400, err)
|
|
return
|
|
}
|
|
|
|
// We're going to run whatever Python is specified in the pull request, which
|
|
// may do bad things, so only trust people we trust.
|
|
if err := checkAuth(pr); err != nil {
|
|
writeError(w, 403, err)
|
|
return
|
|
}
|
|
|
|
base, err := s.generateAt(pr.Base.SHA)
|
|
defer os.RemoveAll(base)
|
|
if err != nil {
|
|
writeError(w, 500, err)
|
|
return
|
|
}
|
|
|
|
head, err := s.generateAt(pr.Head.SHA)
|
|
defer os.RemoveAll(head)
|
|
if err != nil {
|
|
writeError(w, 500, err)
|
|
return
|
|
}
|
|
|
|
htmlDiffer, err := findHTMLDiffer()
|
|
if err != nil {
|
|
writeError(w, 500, fmt.Errorf("could not find HTML differ"))
|
|
return
|
|
}
|
|
|
|
cmd := exec.Command(htmlDiffer, path.Join(base, "scripts", "gen", "specification.html"), path.Join(head, "scripts", "gen", "specification.html"))
|
|
var b bytes.Buffer
|
|
cmd.Stdout = &b
|
|
if err := cmd.Run(); err != nil {
|
|
writeError(w, 500, fmt.Errorf("error running HTML differ: %v", err))
|
|
return
|
|
}
|
|
w.Write(b.Bytes())
|
|
}
|
|
|
|
func findHTMLDiffer() (string, error) {
|
|
wd, err := os.Getwd()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
differ := path.Join(wd, "htmldiff.pl")
|
|
if _, err := os.Stat(differ); err == nil {
|
|
return differ, nil
|
|
}
|
|
return "", fmt.Errorf("unable to find htmldiff.pl")
|
|
}
|
|
|
|
func listPulls(w http.ResponseWriter, req *http.Request) {
|
|
resp, err := http.Get(pullsPrefix)
|
|
if err != nil {
|
|
writeError(w, 500, err)
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
dec := json.NewDecoder(resp.Body)
|
|
var pulls []PullRequest
|
|
if err := dec.Decode(&pulls); err != nil {
|
|
writeError(w, 500, err)
|
|
return
|
|
}
|
|
if len(pulls) == 0 {
|
|
io.WriteString(w, "No pull requests found")
|
|
return
|
|
}
|
|
s := "<body><ul>"
|
|
for _, pull := range pulls {
|
|
s += fmt.Sprintf(`<li>%d: <a href="%s">%s</a>: <a href="%s">%s</a>: <a href="spec/%d">spec</a> <a href="diff/html/%d">spec diff</a> <a href="diff/rst/%d">rst diff</a></li>`,
|
|
pull.Number, pull.User.HTMLURL, pull.User.Login, pull.HTMLURL, pull.Title, pull.Number, pull.Number, pull.Number)
|
|
}
|
|
s += `</ul><div><a href="spec/head">View the spec at head</a></div></body>`
|
|
if *includesDir != "" {
|
|
s += `</ul><div><a href="spec/head?matrixdotorgstyle=1">View the spec at head styled like matrix.org</a></div></body>`
|
|
}
|
|
|
|
io.WriteString(w, s)
|
|
}
|
|
|
|
func ignoreExitCodeOne(err error) error {
|
|
if err == nil {
|
|
return err
|
|
}
|
|
|
|
if exiterr, ok := err.(*exec.ExitError); ok {
|
|
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
|
|
if status.ExitStatus() == 1 {
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
|
|
func main() {
|
|
flag.Parse()
|
|
// It would be great to read this from github, but there's no convenient way to do so.
|
|
// Most of these memberships are "private", so would require some kind of auth.
|
|
allowedMembers = map[string]bool{
|
|
"dbkr": true,
|
|
"erikjohnston": true,
|
|
"illicitonion": true,
|
|
"Kegsay": true,
|
|
"NegativeMjark": true,
|
|
"richvdh": true,
|
|
"leonerd": true,
|
|
}
|
|
if err := initCache(); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
rand.Seed(time.Now().Unix())
|
|
masterCloneDir, err := gitClone(matrixDocCloneURL, false)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
s := server{matrixDocCloneURL: masterCloneDir}
|
|
http.HandleFunc("/spec/", forceHTML(s.serveSpec))
|
|
http.HandleFunc("/diff/rst/", s.serveRSTDiff)
|
|
http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff))
|
|
http.HandleFunc("/healthz", serveText("ok"))
|
|
http.HandleFunc("/", forceHTML(listPulls))
|
|
|
|
fmt.Printf("Listening on port %d\n", *port)
|
|
log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil))
|
|
}
|
|
|
|
func forceHTML(h func(w http.ResponseWriter, req *http.Request)) func(w http.ResponseWriter, req *http.Request) {
|
|
return func(w http.ResponseWriter, req *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html")
|
|
h(w, req)
|
|
}
|
|
}
|
|
|
|
func serveText(s string) func(http.ResponseWriter, *http.Request) {
|
|
return func(w http.ResponseWriter, req *http.Request) {
|
|
io.WriteString(w, s)
|
|
}
|
|
}
|
|
|
|
func initCache() error {
|
|
c1, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s)
|
|
specCache = c1
|
|
|
|
c2, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s)
|
|
styledSpecCache = c2
|
|
return err
|
|
}
|