From c4acee3bcbf8fc99d93a912f09c565729f4aad39 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 11:16:31 +0100 Subject: [PATCH 1/5] continuserv: Live serves updates to the spec --- .gitignore | 1 + scripts/continuserv/main.go | 129 ++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 scripts/continuserv/main.go diff --git a/.gitignore b/.gitignore index f4177c79..e2250131 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ scripts/gen +scripts/continuserv/continuserv templating/out *.pyc supporting-docs/_site diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go new file mode 100644 index 00000000..26dfc4a1 --- /dev/null +++ b/scripts/continuserv/main.go @@ -0,0 +1,129 @@ +// continuserv proactively re-generates the spec on filesystem changes, and serves it over HTTP. +// It will always serve the most recent version of the spec, and may block an HTTP request until regeneration is finished. +// It does not currently pre-empt stale generations, but will block until they are complete. +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "sync" + "sync/atomic" + + fsnotify "gopkg.in/fsnotify.v1" +) + +var toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. +var wg sync.WaitGroup // Indicates how many updates are pending. +var mu sync.Mutex // Prevent multiple updates in parallel. + +func main() { + w, err := fsnotify.NewWatcher() + if err != nil { + log.Fatalf("Error making watcher: %v", err) + } + + dir, err := os.Getwd() + if err != nil { + log.Fatalf("Error getting wd: %v", err) + } + for ; !exists(path.Join(dir, ".git")); dir = path.Dir(dir) { + if dir == "/" { + log.Fatalf("Could not find git root") + } + } + + filepath.Walk(dir, makeWalker(w)) + + wg.Add(1) + populateOnce(dir) + + ch := make(chan struct{}, 100) // Buffered to ensure we can multiple-increment wg for pending writes + go doPopulate(ch, dir) + + http.HandleFunc("/", serve) + go http.ListenAndServe(":8000", nil) + + for { + select { + case e := <-w.Events: + if filter(e) { + wg.Add(1) + ch <- struct{}{} + } + } + } +} + +func makeWalker(w *fsnotify.Watcher) filepath.WalkFunc { + return func(path string, _ os.FileInfo, err error) error { + if err != nil { + log.Fatalf("Error walking: %v", err) + } + w.Add(path) + return nil + } +} + +// Return true if event should trigger re-population +func filter(e fsnotify.Event) bool { + // vim is *really* noisy about how it writes files + if e.Op != fsnotify.Write { + return false + } + // Avoid some temp files that vim writes + if strings.HasSuffix(e.Name, "~") || strings.HasSuffix(e.Name, ".swp") || strings.HasPrefix(e.Name, ".") { + return false + } + + // Avoid infinite cycles being caused by writing actual output + if strings.Contains(e.Name, "/tmp/") || strings.Contains(e.Name, "/gen/") { + return false + } + return true +} + +func serve(w http.ResponseWriter, req *http.Request) { + wg.Wait() + b := toServe.Load().([]byte) + w.Write(b) +} + +func populateOnce(dir string) { + defer wg.Done() + mu.Lock() + defer mu.Unlock() + cmd := exec.Command("python", "gendoc.py") + cmd.Dir = path.Join(dir, "scripts") + var b bytes.Buffer + cmd.Stderr = &b + err := cmd.Run() + if err != nil { + toServe.Store([]byte(fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String()).Error())) + return + } + specBytes, err := ioutil.ReadFile(path.Join(dir, "scripts", "gen", "specification.html")) + if err != nil { + toServe.Store([]byte(fmt.Errorf("error reading spec: %v", err).Error())) + return + } + toServe.Store(specBytes) +} + +func doPopulate(ch chan struct{}, dir string) { + for _ = range ch { + populateOnce(dir) + } +} + +func exists(path string) bool { + _, err := os.Stat(path) + return !os.IsNotExist(err) +} From e30272733b48a3afca3c2e5e6fd87ef1a0ff9727 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 11:29:16 +0100 Subject: [PATCH 2/5] continuserv: Make port flag-specified --- scripts/continuserv/main.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 26dfc4a1..c5d623a4 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -5,6 +5,7 @@ package main import ( "bytes" + "flag" "fmt" "io/ioutil" "log" @@ -20,11 +21,17 @@ import ( fsnotify "gopkg.in/fsnotify.v1" ) -var toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. -var wg sync.WaitGroup // Indicates how many updates are pending. -var mu sync.Mutex // Prevent multiple updates in parallel. +var ( + port = flag.Int("port", 8000, "Port on which to serve HTTP") + + toServe atomic.Value // Always contains valid []byte to serve. May be stale unless wg is zero. + wg sync.WaitGroup // Indicates how many updates are pending. + mu sync.Mutex // Prevent multiple updates in parallel. +) func main() { + flag.Parse() + w, err := fsnotify.NewWatcher() if err != nil { log.Fatalf("Error making watcher: %v", err) @@ -48,9 +55,14 @@ func main() { ch := make(chan struct{}, 100) // Buffered to ensure we can multiple-increment wg for pending writes go doPopulate(ch, dir) + go watchFS(ch, w) + http.HandleFunc("/", serve) - go http.ListenAndServe(":8000", nil) + log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) + +} +func watchFS(ch chan struct{}, w *fsnotify.Watcher) { for { select { case e := <-w.Events: From 6e86cb34d2e8864b69c50faa4ab9b165e47857a7 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 14:25:23 +0100 Subject: [PATCH 3/5] continuserv: Adding README --- scripts/continuserv/README | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 scripts/continuserv/README diff --git a/scripts/continuserv/README b/scripts/continuserv/README new file mode 100644 index 00000000..8ce37850 --- /dev/null +++ b/scripts/continuserv/README @@ -0,0 +1,6 @@ +continuserv proactively re-generates the spec on filesystem changes, and serves it over HTTP. + +To run it, you must install the `go` tool. You will also need to install fsnotify by running: + `go get gopkg.in/fsnotify.v1` +You can then run continuserv by running: + `go run main.go` From 6ac519d9dce4016b9911b5f6ddc05889b8c2abd2 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 14:25:37 +0100 Subject: [PATCH 4/5] Log on file updates --- scripts/continuserv/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index c5d623a4..b4224772 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -68,6 +68,7 @@ func watchFS(ch chan struct{}, w *fsnotify.Watcher) { case e := <-w.Events: if filter(e) { wg.Add(1) + fmt.Printf("Noticed change to %s, re-generating spec\n", e.Name) ch <- struct{}{} } } From c29aef362e11ed31300f0f66a54536a4644c9a51 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 10 Sep 2015 15:45:21 +0100 Subject: [PATCH 5/5] Error on failure of Watcher.Add --- scripts/continuserv/main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index b4224772..e7757c06 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -80,7 +80,9 @@ func makeWalker(w *fsnotify.Watcher) filepath.WalkFunc { if err != nil { log.Fatalf("Error walking: %v", err) } - w.Add(path) + if err := w.Add(path); err != nil { + log.Fatalf("Failed to add watch: %v", err) + } return nil } }