From 2612e54ad1e4f1ea10d20717ff41027a2be4cca2 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 21 Feb 2020 09:35:53 -0800 Subject: [PATCH] derp, cmd/derper: add debug handlers, stats Signed-off-by: Brad Fitzpatrick --- cmd/derper/derper.go | 101 ++++++++++++++++++++++++++++++++++++++++--- derp/derp_server.go | 74 +++++++++++++++++++++++++++---- 2 files changed, 160 insertions(+), 15 deletions(-) diff --git a/cmd/derper/derper.go b/cmd/derper/derper.go index 7dd84a86d..ffb0e464f 100644 --- a/cmd/derper/derper.go +++ b/cmd/derper/derper.go @@ -7,30 +7,37 @@ package main // import "tailscale.com/cmd/derper" import ( "encoding/json" + "expvar" + _ "expvar" "flag" + "fmt" "io" "io/ioutil" "log" "net" "net/http" + _ "net/http/pprof" "os" "path/filepath" + "time" "github.com/tailscale/wireguard-go/wgcfg" "golang.org/x/crypto/acme/autocert" "tailscale.com/atomicfile" "tailscale.com/derp" "tailscale.com/derp/derphttp" + "tailscale.com/interfaces" "tailscale.com/logpolicy" "tailscale.com/types/key" ) var ( + dev = flag.Bool("dev", false, "run in localhost development mode") addr = flag.String("a", ":443", "server address") configPath = flag.String("c", "", "config file path") certDir = flag.String("certdir", defaultCertDir(), "directory to store LetsEncrypt certs, if addr's port is :443") hostname = flag.String("hostname", "derp.tailscale.com", "LetsEncrypt host name, if addr's port is :443") - bytesPerSec = flag.Int("mbps", 5, "Mbps (mebibit/s) per-client rate limit; 0 means unlimited") + mbps = flag.Int("mbps", 5, "Mbps (mebibit/s) per-client rate limit; 0 means unlimited") logCollection = flag.String("logcollection", "", "If non-empty, logtail collection to log to") ) @@ -47,6 +54,9 @@ type config struct { } func loadConfig() config { + if *dev { + return config{PrivateKey: mustNewKey()} + } if *configPath == "" { log.Fatalf("derper: -c not specified") } @@ -66,12 +76,16 @@ func loadConfig() config { } } -func writeNewConfig() config { +func mustNewKey() wgcfg.PrivateKey { key, err := wgcfg.NewPrivateKey() if err != nil { log.Fatal(err) } + return key +} +func writeNewConfig() config { + key := mustNewKey() if err := os.MkdirAll(filepath.Dir(*configPath), 0777); err != nil { log.Fatal(err) } @@ -91,6 +105,12 @@ func writeNewConfig() config { func main() { flag.Parse() + if *dev { + *logCollection = "" + *addr = ":3340" // above the keys DERP + log.Printf("Running in dev mode.") + } + var logPol *logpolicy.Policy if *logCollection != "" { logPol = logpolicy.New(*logCollection) @@ -105,16 +125,33 @@ func main() { } s := derp.NewServer(key.Private(cfg.PrivateKey), log.Printf) - if *bytesPerSec != 0 { - s.BytesPerSecond = (*bytesPerSec << 20) / 8 + if *mbps != 0 { + s.BytesPerSecond = (*mbps << 20) / 8 } + expvar.Publish("derp", s.ExpVar()) + expvar.Publish("uptime", uptimeVar{}) + // Create our own mux so we don't expose /debug/ stuff to the world. mux := http.NewServeMux() mux.Handle("/derp", derphttp.Handler(s)) + mux.Handle("/debug/", protected(debugHandler(s))) + mux.Handle("/debug/pprof/", protected(http.DefaultServeMux)) // to net/http/pprof + mux.Handle("/debug/vars", protected(http.DefaultServeMux)) // to expvar mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain") + w.Header().Set("Content-Type", "text/html; charset=utf-8") w.WriteHeader(200) - io.WriteString(w, "Tailscale DERP server.") + io.WriteString(w, ` +

DERP

+

+ This is a + Tailscale + DERP + server. +

+`) + if allowDebugAccess(r) { + io.WriteString(w, "

Debug info at /debug/.

\n") + } })) httpsrv := &http.Server{ @@ -151,3 +188,55 @@ func main() { log.Fatalf("derper: %v", err) } } + +func protected(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !allowDebugAccess(r) { + http.Error(w, "debug access denied", http.StatusForbidden) + return + } + h.ServeHTTP(w, r) + }) +} + +func allowDebugAccess(r *http.Request) bool { + if r.Header.Get("X-Forwarded-For") != "" { + // TODO if/when needed. For now, conservative: + return false + } + ipStr, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return false + } + ip := net.ParseIP(ipStr) + return interfaces.IsTailscaleIP(ip) || ip.IsLoopback() || ipStr == os.Getenv("ALLOW_DEBUG_IP") +} + +func debugHandler(s *derp.Server) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f := func(format string, args ...interface{}) { fmt.Fprintf(w, format, args...) } + f(` +

DERP debug

+
    +`) + f("
  • Hostname: %v
    • \n", *hostname) + f("
  • Rate Limit: %v Mbps
    • \n", *mbps) + f("
  • Uptime: %v
    • \n", uptime().Round(time.Second)) + + f(`
  • /debug/vars
    • +
  • /debug/pprof/
    • +
  • /debug/pprof/goroutine (collapsed)
    • +
  • /debug/pprof/goroutine (full)
    • +
      + +`) + }) +} + +var timeStart = time.Now() + +func uptime() time.Duration { return time.Since(timeStart) } + +type uptimeVar struct{} + +func (uptimeVar) String() string { return fmt.Sprint(int64(uptime().Seconds())) } diff --git a/derp/derp_server.go b/derp/derp_server.go index 74338acd1..e87596b4c 100644 --- a/derp/derp_server.go +++ b/derp/derp_server.go @@ -13,11 +13,13 @@ import ( crand "crypto/rand" "encoding/json" "errors" + "expvar" "fmt" "io" "math/big" "net" "sync" + "sync/atomic" "time" "golang.org/x/crypto/nacl/box" @@ -36,21 +38,28 @@ type Server struct { publicKey key.Public logf logger.Logf - mu sync.Mutex - closed bool - netConns map[net.Conn]chan struct{} // chan is closed when conn closes - clients map[key.Public]*sclient + // Counters: + packetsSent int64 + bytesSent int64 + + mu sync.Mutex + closed bool + accepts int64 + netConns map[net.Conn]chan struct{} // chan is closed when conn closes + clients map[key.Public]*sclient + clientsEver map[key.Public]bool // never deleted from, for stats; fine for now } // NewServer returns a new DERP server. It doesn't listen on its own. // Connections are given to it via Server.Accept. func NewServer(privateKey key.Private, logf logger.Logf) *Server { s := &Server{ - privateKey: privateKey, - publicKey: privateKey.Public(), - logf: logf, - clients: make(map[key.Public]*sclient), - netConns: make(map[net.Conn]chan struct{}), + privateKey: privateKey, + publicKey: privateKey.Public(), + logf: logf, + clients: make(map[key.Public]*sclient), + clientsEver: make(map[key.Public]bool), + netConns: make(map[net.Conn]chan struct{}), } return s } @@ -95,6 +104,7 @@ func (s *Server) Accept(nc net.Conn, brw *bufio.ReadWriter) { closed := make(chan struct{}) s.mu.Lock() + s.accepts++ s.netConns[nc] = closed s.mu.Unlock() @@ -125,6 +135,7 @@ func (s *Server) registerClient(c *sclient) { s.logf("derp: %s: client %x: adding connection, replacing %s", c.nc.RemoteAddr(), c.key, old.nc.RemoteAddr()) } s.clients[c.key] = c + s.clientsEver[c.key] = true } // unregisterClient removes a client from the server. @@ -311,6 +322,8 @@ func (s *Server) recvClientKey(br *bufio.Reader) (clientKey key.Public, info *sc } func (s *Server) sendPacket(bw *bufio.Writer, srcKey key.Public, contents []byte) error { + atomic.AddInt64(&s.packetsSent, 1) + atomic.AddInt64(&s.bytesSent, int64(len(contents))) if err := writeFrameHeader(bw, frameRecvPacket, uint32(len(contents))); err != nil { return err } @@ -397,3 +410,46 @@ type sclientInfo struct { type serverInfo struct { } + +// Stats returns stats about the server. +func (s *Server) Stats() *ServerStats { + s.mu.Lock() + defer s.mu.Unlock() + return &ServerStats{ + BytesPerSecondLimit: s.BytesPerSecond, + CurrentConnections: len(s.netConns), + UniqueClientsEver: len(s.clientsEver), + TotalAccepts: s.accepts, + PacketsSent: atomic.LoadInt64(&s.packetsSent), + BytesSent: atomic.LoadInt64(&s.bytesSent), + } +} + +// ExpVar returns an expvar variable suitable for registering with expvar.Publish. +func (s *Server) ExpVar() expvar.Var { + return expVar{s} +} + +type expVar struct{ *Server } + +// String implements the expvar.Var interface, returning the current server stats as JSON. +func (v expVar) String() string { + ss := v.Server.Stats() + j, err := json.MarshalIndent(ss, "", "\t") + if err != nil { + return "{}" + } + return string(j) +} + +// ServerStats are returned by Server.Stats. +// +// It is JSON-ified by expVar for the expvar package. +type ServerStats struct { + BytesPerSecondLimit int `json:"bytesPerSecondLimit"` + CurrentConnections int `json:"currentClients"` + UniqueClientsEver int `json:"uniqueClientsEver"` + TotalAccepts int64 `json:"totalAccepts"` + PacketsSent int64 `json:"packetsSent"` + BytesSent int64 `json:"bytesSent"` +}