tsweb, derp: add expvar http.Handler for Prometheus's format

And add some opinions.

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

cmd/derper/derper.go

@@ -104,6 +104,7 @@ func main() {
 		*logCollection = ""
 		*addr = ":3340" // above the keys DERP
 		log.Printf("Running in dev mode.")
+		tsweb.DevMode = true
 	}
 
 	var logPol *logpolicy.Policy
@@ -213,13 +214,13 @@ func serveSTUN() {
 	}
 	log.Printf("running STUN server on %v", pc.LocalAddr())
 	var (
-		stunReadErrors       = expvar.NewInt("stun-read-error")
+		stunReadErrors       = expvar.NewInt("stun_read_error")
-		stunWriteErrors      = expvar.NewInt("stun-write-error")
+		stunWriteErrors      = expvar.NewInt("stun_write_error")
-		stunReadNotSTUN      = expvar.NewInt("stun-read-not-stun")
+		stunReadNotSTUN      = expvar.NewInt("stun_read_not_stun")
-		stunReadNotSTUNValid = expvar.NewInt("stun-read-not-stun-valid")
+		stunReadNotSTUNValid = expvar.NewInt("stun_read_not_stun_valid")
-		stunReadIPv4         = expvar.NewInt("stun-read-ipv4")
+		stunReadIPv4         = expvar.NewInt("stun_read_ipv4")
-		stunReadIPv6         = expvar.NewInt("stun-read-ipv6")
+		stunReadIPv6         = expvar.NewInt("stun_read_ipv6")
-		stunWrite            = expvar.NewInt("stun-write")
+		stunWrite            = expvar.NewInt("stun_write")
 	)
 	var buf [64 << 10]byte
 	for {

derp/derp_server.go

@@ -21,11 +21,11 @@ import (
 	"os"
 	"strconv"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/time/rate"
+	"tailscale.com/metrics"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 )
@@ -43,13 +43,13 @@ type Server struct {
 	logf       logger.Logf
 
 	// Counters:
-	packetsSent, bytesSent int64
+	packetsSent, bytesSent expvar.Int
-	packetsRecv, bytesRecv int64
+	packetsRecv, bytesRecv expvar.Int
-	packetsDropped         int64
+	packetsDropped         expvar.Int
+	accepts                expvar.Int
 
 	mu          sync.Mutex
 	closed      bool
-	accepts     int64
 	netConns    map[net.Conn]chan struct{} // chan is closed when conn closes
 	clients     map[key.Public]*sclient
 	clientsEver map[key.Public]bool // never deleted from, for stats; fine for now
@@ -111,8 +111,8 @@ func (s *Server) isClosed() bool {
 func (s *Server) Accept(nc net.Conn, brw *bufio.ReadWriter) {
 	closed := make(chan struct{})
 
+	s.accepts.Add(1)
 	s.mu.Lock()
-	s.accepts++
 	s.netConns[nc] = closed
 	s.mu.Unlock()
 
@@ -229,7 +229,7 @@ func (s *Server) accept(nc net.Conn, brw *bufio.ReadWriter) error {
 		s.mu.Unlock()
 
 		if dst == nil {
-			atomic.AddInt64(&s.packetsDropped, 1)
+			s.packetsDropped.Add(1)
 			if debug {
 				s.logf("derp: %s: client %x: dropping packet for unknown %x", nc.RemoteAddr(), c.key, dstKey)
 			}
@@ -333,8 +333,8 @@ func (s *Server) recvClientKey(br *bufio.Reader) (clientKey key.Public, info *sc
 }
 
 func (s *Server) sendPacket(bw *bufio.Writer, srcKey key.Public, contents []byte) error {
-	atomic.AddInt64(&s.packetsSent, 1)
+	s.packetsSent.Add(1)
-	atomic.AddInt64(&s.bytesSent, int64(len(contents)))
+	s.bytesSent.Add(int64(len(contents)))
 	if err := writeFrameHeader(bw, frameRecvPacket, uint32(len(contents))); err != nil {
 		return err
 	}
@@ -362,8 +362,8 @@ func (s *Server) recvPacket(ctx context.Context, br *bufio.Reader, frameLen uint
 	if _, err := io.ReadFull(br, contents); err != nil {
 		return key.Public{}, nil, err
 	}
-	atomic.AddInt64(&s.packetsRecv, 1)
+	s.packetsRecv.Add(1)
-	atomic.AddInt64(&s.bytesRecv, int64(len(contents)))
+	s.bytesRecv.Add(int64(len(contents)))
 	return dstKey, contents, nil
 }
 
@@ -424,51 +424,24 @@ type sclientInfo struct {
 type serverInfo struct {
 }
 
-// Stats returns stats about the server.
+func (s *Server) expVarFunc(f func() interface{}) expvar.Func {
-func (s *Server) Stats() *ServerStats {
+	return expvar.Func(func() interface{} {
-	s.mu.Lock()
+		s.mu.Lock()
-	defer s.mu.Unlock()
+		defer s.mu.Unlock()
-	return &ServerStats{
+		return f()
-		BytesPerSecondLimit: s.BytesPerSecond,
+	})
-		CurrentConnections:  len(s.netConns),
-		UniqueClientsEver:   len(s.clientsEver),
-		TotalAccepts:        s.accepts,
-		BytesReceived:       atomic.LoadInt64(&s.bytesRecv),
-		BytesSent:           atomic.LoadInt64(&s.bytesSent),
-		PacketsDropped:      atomic.LoadInt64(&s.packetsDropped),
-		PacketsReceived:     atomic.LoadInt64(&s.packetsRecv),
-		PacketsSent:         atomic.LoadInt64(&s.packetsSent),
-	}
 }
 
 // ExpVar returns an expvar variable suitable for registering with expvar.Publish.
 func (s *Server) ExpVar() expvar.Var {
-	return expVar{s}
+	m := new(metrics.Set)
-}
+	m.Set("gauge_current_connnections", s.expVarFunc(func() interface{} { return len(s.netConns) }))
-
+	m.Set("counter_unique_clients_ever", s.expVarFunc(func() interface{} { return len(s.clientsEver) }))
-type expVar struct{ *Server }
+	m.Set("accepts", &s.accepts)
-
+	m.Set("bytes_received", &s.bytesRecv)
-// String implements the expvar.Var interface, returning the current server stats as JSON.
+	m.Set("bytes_sent", &s.bytesSent)
-func (v expVar) String() string {
+	m.Set("packets_dropped", &s.packetsDropped)
-	ss := v.Server.Stats()
+	m.Set("packets_sent", &s.packetsSent)
-	j, err := json.MarshalIndent(ss, "", "\t")
+	m.Set("packets_received", &s.packetsRecv)
-	if err != nil {
+	return m
-		return "{}"
-	}
-	return string(j)
-}
-
-// ServerStats are returned by Server.Stats.
-//
-// It is JSON-ified by expVar for the expvar package.
-type ServerStats struct {
-	BytesPerSecondLimit int   `json:"bytesPerSecondLimit"`
-	CurrentConnections  int   `json:"currentClients"`
-	UniqueClientsEver   int   `json:"uniqueClientsEver"`
-	TotalAccepts        int64 `json:"totalAccepts"`
-	BytesReceived       int64 `json:"bytesReceived"`
-	BytesSent           int64 `json:"bytesSent"`
-	PacketsDropped      int64 `json:"packetsDropped"`
-	PacketsReceived     int64 `json:"packetsReceived"`
-	PacketsSent         int64 `json:"packetsSent"`
 }

metrics/metrics.go

@@ -0,0 +1,23 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package metrics contains expvar & Prometheus types and code used by
+// Tailscale for monitoring.
+package metrics
+
+import "expvar"
+
+// Map is a string-to-Var map variable that satisfies the expvar.Var
+// interface.
+//
+// Semantically, this is mapped by tsweb's Prometheus exporter as a
+// collection of unrelated variables exported with a common prefix.
+//
+// This lets us have tsweb recognize *expvar.Map for different
+// purposes in the future. (Or perhaps all uses of expvar.Map will
+// require explicit types like this one, declaring how we want tsweb
+// to export it to Prometheus.)
+type Set struct {
+	expvar.Map
+}

tsweb/tsweb.go

@@ -18,8 +18,12 @@ import (
 	"time"
 
 	"tailscale.com/interfaces"
+	"tailscale.com/metrics"
 )
 
+// DevMode controls whether extra output in shown, for when the binary is being run in dev mode.
+var DevMode bool
+
 // NewMux returns a new ServeMux with debugHandler registered (and protected) at /debug/.
 func NewMux(debugHandler http.Handler) *http.ServeMux {
 	mux := http.NewServeMux()
@@ -29,9 +33,10 @@ func NewMux(debugHandler http.Handler) *http.ServeMux {
 }
 
 func RegisterCommonDebug(mux *http.ServeMux) {
-	expvar.Publish("uptime", uptimeVar{})
+	expvar.Publish("counter_uptime_sec", expvar.Func(func() interface{} { return int64(Uptime().Seconds()) }))
 	mux.Handle("/debug/pprof/", Protected(http.DefaultServeMux)) // to net/http/pprof
 	mux.Handle("/debug/vars", Protected(http.DefaultServeMux))   // to expvar
+	mux.Handle("/debug/varz", Protected(http.HandlerFunc(varzHandler)))
 }
 
 func DefaultCertDir(leafDir string) string {
@@ -69,7 +74,12 @@ func AllowDebugAccess(r *http.Request) bool {
 func Protected(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if !AllowDebugAccess(r) {
-			http.Error(w, "debug access denied", http.StatusForbidden)
+			msg := "debug access denied"
+			if DevMode {
+				ipStr, _, _ := net.SplitHostPort(r.RemoteAddr)
+				msg += fmt.Sprintf("; to permit access, set ALLOW_DEBUG_IP=%v", ipStr)
+			}
+			http.Error(w, msg, http.StatusForbidden)
 			return
 		}
 		h.ServeHTTP(w, r)
@@ -80,10 +90,6 @@ var timeStart = time.Now()
 
 func Uptime() time.Duration { return time.Since(timeStart).Round(time.Second) }
 
-type uptimeVar struct{}
-
-func (uptimeVar) String() string { return fmt.Sprint(int64(Uptime().Seconds())) }
-
 // Port80Handler is the handler to be given to
 // autocert.Manager.HTTPHandler.  The inner handler is the mux
 // returned by NewMux containing registered /debug handlers.
@@ -114,3 +120,64 @@ func stripPort(hostport string) string {
 	}
 	return net.JoinHostPort(host, "443")
 }
+
+// varzHandler is an HTTP handler to write expvar values into the
+// prometheus export format:
+//
+//   https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md
+//
+// It makes the following assumptions:
+//
+//   * *expvar.Int are counters.
+//   * a *tailscale/metrics.Set is descended into, joining keys with
+//     underscores. So use underscores as your metric names.
+//   * an expvar named starting with "gauge_" or "counter_" is of that
+//     Prometheus type, and has that prefix stripped.
+//   * anything else is untyped and thus not exported.
+//   * expvar.Func can return an int or int64 (for now) and anything else
+//     is not exported.
+//
+// This will evolve over time, or perhaps be replaced.
+func varzHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain; version=0.0.4")
+
+	var dump func(prefix string, kv expvar.KeyValue)
+	dump = func(prefix string, kv expvar.KeyValue) {
+		name := prefix + kv.Key
+		var typ string
+		switch v := kv.Value.(type) {
+		case *expvar.Int:
+			// Fast path for common value type.
+			fmt.Fprintf(w, "# TYPE %s counter\n%s %v\n", name, name, v.Value())
+			return
+		case *metrics.Set:
+			v.Do(func(kv expvar.KeyValue) {
+				dump(name+"_", kv)
+			})
+			return
+		}
+		if strings.HasPrefix(kv.Key, "gauge_") {
+			typ = "gauge"
+			name = prefix + strings.TrimPrefix(kv.Key, "gauge_")
+		} else if strings.HasPrefix(kv.Key, "counter_") {
+			typ = "counter"
+			name = prefix + strings.TrimPrefix(kv.Key, "counter_")
+		}
+		if fn, ok := kv.Value.(expvar.Func); ok {
+			val := fn()
+			switch val.(type) {
+			case int64, int:
+				if typ != "" {
+					fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, val)
+					return
+				}
+			}
+			fmt.Fprintf(w, "# skipping expvar func %q returning unknown type %T\n", name, val)
+			return
+		}
+		fmt.Fprintf(w, "# skipping func %q returning unknown type %T\n", name, kv.Value)
+	}
+	expvar.Do(func(kv expvar.KeyValue) {
+		dump("", kv)
+	})
+}