From f192c054135bd6672064ac3b2259e7643bfa39b1 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Wed, 4 Mar 2020 12:24:07 -0800 Subject: [PATCH] metrics: add a LabelMap type for variables with 1 label dimension. This lets us publish sets of vars that are breakdowns along one dimension in a format that Prometheus and Grafana natively know how to do useful things with. Signed-off-by: David Anderson --- cmd/derper/derper.go | 40 +++++++++++++++++++++++++--------------- metrics/metrics.go | 21 ++++++++++++++++++++- tsweb/tsweb.go | 35 ++++++++++++++++++++++++----------- 3 files changed, 69 insertions(+), 27 deletions(-) diff --git a/cmd/derper/derper.go b/cmd/derper/derper.go index e4fc7a142..cfc07a068 100644 --- a/cmd/derper/derper.go +++ b/cmd/derper/derper.go @@ -28,6 +28,7 @@ import ( "tailscale.com/derp" "tailscale.com/derp/derphttp" "tailscale.com/logpolicy" + "tailscale.com/metrics" "tailscale.com/stun" "tailscale.com/tsweb" "tailscale.com/types/key" @@ -213,51 +214,60 @@ func serveSTUN() { log.Fatalf("failed to open STUN listener: %v", err) } log.Printf("running STUN server on %v", pc.LocalAddr()) + var ( - stunReadErrors = expvar.NewInt("stun_read_error") - stunWriteErrors = expvar.NewInt("stun_write_error") - stunReadNotSTUN = expvar.NewInt("stun_read_not_stun") - stunReadNotSTUNValid = expvar.NewInt("stun_read_not_stun_valid") - stunReadIPv4 = expvar.NewInt("stun_read_ipv4") - stunReadIPv6 = expvar.NewInt("stun_read_ipv6") - stunWrite = expvar.NewInt("stun_write") + stats = new(metrics.Set) + stunDisposition = &metrics.LabelMap{Label: "disposition"} + stunAddrFamily = &metrics.LabelMap{Label: "family"} + + stunReadError = stunDisposition.Get("read_error") + stunNotSTUN = stunDisposition.Get("not_stun") + stunWriteError = stunDisposition.Get("write_error") + stunSuccess = stunDisposition.Get("success") + + stunIPv4 = stunAddrFamily.Get("ipv4") + stunIPv6 = stunAddrFamily.Get("ipv6") ) + stats.Set("counter_requests", stunDisposition) + stats.Set("counter_addrfamily", stunAddrFamily) + expvar.Publish("stun", stats) + var buf [64 << 10]byte for { n, addr, err := pc.ReadFrom(buf[:]) if err != nil { log.Printf("STUN ReadFrom: %v", err) time.Sleep(time.Second) - stunReadErrors.Add(1) + stunReadError.Add(1) continue } ua, ok := addr.(*net.UDPAddr) if !ok { log.Printf("STUN unexpected address %T %v", addr, addr) - stunReadErrors.Add(1) + stunReadError.Add(1) continue } pkt := buf[:n] if !stun.Is(pkt) { - stunReadNotSTUN.Add(1) + stunNotSTUN.Add(1) continue } txid, err := stun.ParseBindingRequest(pkt) if err != nil { - stunReadNotSTUNValid.Add(1) + stunNotSTUN.Add(1) continue } if ua.IP.To4() != nil { - stunReadIPv4.Add(1) + stunIPv4.Add(1) } else { - stunReadIPv6.Add(1) + stunIPv6.Add(1) } res := stun.Response(txid, ua.IP, uint16(ua.Port)) _, err = pc.WriteTo(res, addr) if err != nil { - stunWriteErrors.Add(1) + stunWriteError.Add(1) } else { - stunWrite.Add(1) + stunSuccess.Add(1) } } } diff --git a/metrics/metrics.go b/metrics/metrics.go index bb3c1bac6..0ffc8fbac 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -8,7 +8,7 @@ package metrics import "expvar" -// Map is a string-to-Var map variable that satisfies the expvar.Var +// Set is a string-to-Var map variable that satisfies the expvar.Var // interface. // // Semantically, this is mapped by tsweb's Prometheus exporter as a @@ -21,3 +21,22 @@ import "expvar" type Set struct { expvar.Map } + +// LabelMap is a string-to-Var map variable that satisfies the +// expvar.Var interface. +// +// Semantically, this is mapped by tsweb's Prometheus exporter as a +// collection of variables with the same name, with a varying label +// value. Use this to export things that are intuitively breakdowns +// into different buckets. +type LabelMap struct { + Label string + expvar.Map +} + +// Get returns a direct pointer to the expvar.Int for key, creating it +// if necessary. +func (m *LabelMap) Get(key string) *expvar.Int { + m.Add(key, 0) + return m.Map.Get(key).(*expvar.Int) +} diff --git a/tsweb/tsweb.go b/tsweb/tsweb.go index d9138f263..fc735ca33 100644 --- a/tsweb/tsweb.go +++ b/tsweb/tsweb.go @@ -156,26 +156,39 @@ func varzHandler(w http.ResponseWriter, r *http.Request) { }) return } - if strings.HasPrefix(kv.Key, "gauge_") { + + switch { + case strings.HasPrefix(kv.Key, "gauge_"): typ = "gauge" name = prefix + strings.TrimPrefix(kv.Key, "gauge_") - } else if strings.HasPrefix(kv.Key, "counter_") { + + case strings.HasPrefix(kv.Key, "counter_"): typ = "counter" name = prefix + strings.TrimPrefix(kv.Key, "counter_") + + default: + fmt.Fprintf(w, "# skipping expvar %q with undeclared Prometheus type\n", name) + return } - if fn, ok := kv.Value.(expvar.Func); ok { - val := fn() + + switch v := kv.Value.(type) { + case expvar.Func: + val := v() switch val.(type) { case int64, int: - if typ != "" { - fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, val) - return - } + fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, val) + default: + fmt.Fprintf(w, "# skipping expvar func %q returning unknown type %T\n", name, val) } - fmt.Fprintf(w, "# skipping expvar func %q returning unknown type %T\n", name, val) - return + + case *metrics.LabelMap: + fmt.Fprintf(w, "# TYPE %s %s\n", name, typ) + // IntMap uses expvar.Map on the inside, which presorts + // keys. The output ordering is deterministic. + v.Do(func(kv expvar.KeyValue) { + fmt.Fprintf(w, "%s{%s=%s} %v\n", name, v.Label, kv.Key, kv.Value) + }) } - fmt.Fprintf(w, "# skipping func %q returning unknown type %T\n", name, kv.Value) } expvar.Do(func(kv expvar.KeyValue) { dump("", kv)