diff --git a/cmd/derper/depaware.txt b/cmd/derper/depaware.txt index 30e3c7c45..7f60654b0 100644 --- a/cmd/derper/depaware.txt +++ b/cmd/derper/depaware.txt @@ -24,7 +24,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa L 💣 github.com/mdlayher/netlink/nlenc from github.com/jsimonetti/rtnetlink+ L 💣 github.com/mdlayher/socket from github.com/mdlayher/netlink 💣 github.com/mitchellh/go-ps from tailscale.com/safesocket - 💣 github.com/prometheus/client_golang/prometheus from tailscale.com/tsweb + 💣 github.com/prometheus/client_golang/prometheus from tailscale.com/tsweb/promvarz github.com/prometheus/client_golang/prometheus/internal from github.com/prometheus/client_golang/prometheus github.com/prometheus/client_model/go from github.com/prometheus/client_golang/prometheus+ github.com/prometheus/common/expfmt from github.com/prometheus/client_golang/prometheus+ @@ -104,6 +104,8 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa 💣 tailscale.com/tstime/mono from tailscale.com/tstime/rate tailscale.com/tstime/rate from tailscale.com/wgengine/filter+ tailscale.com/tsweb from tailscale.com/cmd/derper + tailscale.com/tsweb/promvarz from tailscale.com/tsweb + tailscale.com/tsweb/varz from tailscale.com/tsweb+ tailscale.com/types/dnstype from tailscale.com/tailcfg tailscale.com/types/empty from tailscale.com/ipn tailscale.com/types/ipproto from tailscale.com/net/flowtrack+ @@ -232,7 +234,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa net/http from expvar+ net/http/httptrace from net/http+ net/http/internal from net/http - net/http/pprof from tailscale.com/tsweb + net/http/pprof from tailscale.com/tsweb+ net/netip from go4.org/netipx+ net/textproto from golang.org/x/net/http/httpguts+ net/url from crypto/x509+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index abb7783b4..1d53813d0 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -69,8 +69,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de L github.com/aws/smithy-go/transport/http from github.com/aws/aws-sdk-go-v2/aws/middleware+ L github.com/aws/smithy-go/transport/http/internal/io from github.com/aws/smithy-go/transport/http L github.com/aws/smithy-go/waiter from github.com/aws/aws-sdk-go-v2/service/ssm - github.com/beorn7/perks/quantile from github.com/prometheus/client_golang/prometheus - 💣 github.com/cespare/xxhash/v2 from github.com/prometheus/client_golang/prometheus L github.com/coreos/go-iptables/iptables from tailscale.com/wgengine/router LD 💣 github.com/creack/pty from tailscale.com/ssh/tailssh W 💣 github.com/dblohm7/wingoes from github.com/dblohm7/wingoes/com @@ -80,8 +78,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de W 💣 github.com/go-ole/go-ole/oleutil from tailscale.com/wgengine/winnet L 💣 github.com/godbus/dbus/v5 from tailscale.com/net/dns+ github.com/golang/groupcache/lru from tailscale.com/net/dnscache - github.com/golang/protobuf/proto from github.com/matttproud/golang_protobuf_extensions/pbutil+ - github.com/golang/protobuf/ptypes/timestamp from github.com/prometheus/client_model/go github.com/google/btree from gvisor.dev/gvisor/pkg/tcpip/header+ github.com/hdevalence/ed25519consensus from tailscale.com/tka L 💣 github.com/illarion/gonotify from tailscale.com/net/dns @@ -103,7 +99,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd github.com/kortschak/wol from tailscale.com/ipn/ipnlocal LD github.com/kr/fs from github.com/pkg/sftp - github.com/matttproud/golang_protobuf_extensions/pbutil from github.com/prometheus/common/expfmt L github.com/mdlayher/genetlink from tailscale.com/net/tstun L 💣 github.com/mdlayher/netlink from github.com/jsimonetti/rtnetlink+ L 💣 github.com/mdlayher/netlink/nlenc from github.com/jsimonetti/rtnetlink+ @@ -113,15 +108,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de W github.com/pkg/errors from github.com/tailscale/certstore LD github.com/pkg/sftp from tailscale.com/ssh/tailssh LD github.com/pkg/sftp/internal/encoding/ssh/filexfer from github.com/pkg/sftp - 💣 github.com/prometheus/client_golang/prometheus from tailscale.com/tsweb - github.com/prometheus/client_golang/prometheus/internal from github.com/prometheus/client_golang/prometheus - github.com/prometheus/client_model/go from github.com/prometheus/client_golang/prometheus+ - github.com/prometheus/common/expfmt from github.com/prometheus/client_golang/prometheus+ - github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg from github.com/prometheus/common/expfmt - github.com/prometheus/common/model from github.com/prometheus/client_golang/prometheus+ - LD github.com/prometheus/procfs from github.com/prometheus/client_golang/prometheus - LD github.com/prometheus/procfs/internal/fs from github.com/prometheus/procfs - LD github.com/prometheus/procfs/internal/util from github.com/prometheus/procfs W 💣 github.com/tailscale/certstore from tailscale.com/control/controlclient LD github.com/tailscale/golang-x-crypto/chacha20 from github.com/tailscale/golang-x-crypto/ssh LD 💣 github.com/tailscale/golang-x-crypto/internal/subtle from github.com/tailscale/golang-x-crypto/chacha20 @@ -155,34 +141,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de go4.org/netipx from tailscale.com/ipn/ipnlocal+ W 💣 golang.zx2c4.com/wintun from github.com/tailscale/wireguard-go/tun+ W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg from tailscale.com/net/dns+ - google.golang.org/protobuf/encoding/prototext from github.com/golang/protobuf/proto+ - google.golang.org/protobuf/encoding/protowire from github.com/golang/protobuf/proto+ - google.golang.org/protobuf/internal/descfmt from google.golang.org/protobuf/internal/filedesc - google.golang.org/protobuf/internal/descopts from google.golang.org/protobuf/internal/filedesc+ - google.golang.org/protobuf/internal/detrand from google.golang.org/protobuf/internal/descfmt+ - google.golang.org/protobuf/internal/encoding/defval from google.golang.org/protobuf/internal/encoding/tag+ - google.golang.org/protobuf/internal/encoding/messageset from google.golang.org/protobuf/encoding/prototext+ - google.golang.org/protobuf/internal/encoding/tag from google.golang.org/protobuf/internal/impl - google.golang.org/protobuf/internal/encoding/text from google.golang.org/protobuf/encoding/prototext+ - google.golang.org/protobuf/internal/errors from google.golang.org/protobuf/encoding/prototext+ - google.golang.org/protobuf/internal/filedesc from google.golang.org/protobuf/internal/encoding/tag+ - google.golang.org/protobuf/internal/filetype from google.golang.org/protobuf/runtime/protoimpl - google.golang.org/protobuf/internal/flags from google.golang.org/protobuf/encoding/prototext+ - google.golang.org/protobuf/internal/genid from google.golang.org/protobuf/encoding/prototext+ - 💣 google.golang.org/protobuf/internal/impl from google.golang.org/protobuf/internal/filetype+ - google.golang.org/protobuf/internal/order from google.golang.org/protobuf/encoding/prototext+ - google.golang.org/protobuf/internal/pragma from google.golang.org/protobuf/encoding/prototext+ - google.golang.org/protobuf/internal/set from google.golang.org/protobuf/encoding/prototext - 💣 google.golang.org/protobuf/internal/strs from google.golang.org/protobuf/encoding/prototext+ - google.golang.org/protobuf/internal/version from google.golang.org/protobuf/runtime/protoimpl - google.golang.org/protobuf/proto from github.com/golang/protobuf/proto+ - google.golang.org/protobuf/reflect/protodesc from github.com/golang/protobuf/proto - 💣 google.golang.org/protobuf/reflect/protoreflect from github.com/golang/protobuf/proto+ - google.golang.org/protobuf/reflect/protoregistry from github.com/golang/protobuf/proto+ - google.golang.org/protobuf/runtime/protoiface from github.com/golang/protobuf/proto+ - google.golang.org/protobuf/runtime/protoimpl from github.com/golang/protobuf/proto+ - google.golang.org/protobuf/types/descriptorpb from google.golang.org/protobuf/reflect/protodesc - google.golang.org/protobuf/types/known/timestamppb from github.com/golang/protobuf/ptypes/timestamp+ gvisor.dev/gvisor/pkg/atomicbitops from gvisor.dev/gvisor/pkg/tcpip+ gvisor.dev/gvisor/pkg/bits from gvisor.dev/gvisor/pkg/bufferv2 💣 gvisor.dev/gvisor/pkg/bufferv2 from gvisor.dev/gvisor/pkg/tcpip+ @@ -314,7 +272,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/tstime from tailscale.com/wgengine/magicsock 💣 tailscale.com/tstime/mono from tailscale.com/net/tstun+ tailscale.com/tstime/rate from tailscale.com/wgengine/filter+ - tailscale.com/tsweb from tailscale.com/cmd/tailscaled + tailscale.com/tsweb/varz from tailscale.com/cmd/tailscaled tailscale.com/types/dnstype from tailscale.com/ipn/ipnlocal+ tailscale.com/types/empty from tailscale.com/control/controlclient+ tailscale.com/types/flagtype from tailscale.com/cmd/tailscaled @@ -357,7 +315,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/util/sysresources from tailscale.com/wgengine/magicsock tailscale.com/util/systemd from tailscale.com/control/controlclient+ tailscale.com/util/uniq from tailscale.com/wgengine/magicsock+ - tailscale.com/util/vizerror from tailscale.com/tsweb 💣 tailscale.com/util/winutil from tailscale.com/control/controlclient+ W tailscale.com/util/winutil/policy from tailscale.com/ipn/ipnlocal tailscale.com/version from tailscale.com/derp+ @@ -465,7 +422,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de expvar from tailscale.com/derp+ flag from net/http/httptest+ fmt from compress/flate+ - go/token from google.golang.org/protobuf/internal/strs hash from crypto+ hash/adler32 from tailscale.com/ipn/ipnlocal hash/crc32 from compress/gzip+ @@ -504,7 +460,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de regexp from github.com/coreos/go-iptables/iptables+ regexp/syntax from regexp runtime/debug from github.com/klauspost/compress/zstd+ - runtime/metrics from github.com/prometheus/client_golang/prometheus+ runtime/pprof from tailscale.com/log/logheap+ runtime/trace from net/http/pprof sort from compress/flate+ diff --git a/cmd/tailscaled/tailscaled.go b/cmd/tailscaled/tailscaled.go index 32699c006..4cc547aa1 100644 --- a/cmd/tailscaled/tailscaled.go +++ b/cmd/tailscaled/tailscaled.go @@ -49,7 +49,7 @@ import ( "tailscale.com/safesocket" "tailscale.com/smallzstd" "tailscale.com/syncs" - "tailscale.com/tsweb" + "tailscale.com/tsweb/varz" "tailscale.com/types/flagtype" "tailscale.com/types/logger" "tailscale.com/types/logid" @@ -670,7 +670,7 @@ func newDebugMux() *http.ServeMux { func servePrometheusMetrics(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain") - tsweb.VarzHandler(w, r) + varz.Handler(w, r) clientmetric.WritePrometheusExpositionFormat(w) } diff --git a/tstest/integration/tailscaled_deps_test_darwin.go b/tstest/integration/tailscaled_deps_test_darwin.go index c8d6fc23a..edb891ee9 100644 --- a/tstest/integration/tailscaled_deps_test_darwin.go +++ b/tstest/integration/tailscaled_deps_test_darwin.go @@ -36,7 +36,7 @@ import ( _ "tailscale.com/ssh/tailssh" _ "tailscale.com/syncs" _ "tailscale.com/tailcfg" - _ "tailscale.com/tsweb" + _ "tailscale.com/tsweb/varz" _ "tailscale.com/types/flagtype" _ "tailscale.com/types/key" _ "tailscale.com/types/logger" diff --git a/tstest/integration/tailscaled_deps_test_freebsd.go b/tstest/integration/tailscaled_deps_test_freebsd.go index c8d6fc23a..edb891ee9 100644 --- a/tstest/integration/tailscaled_deps_test_freebsd.go +++ b/tstest/integration/tailscaled_deps_test_freebsd.go @@ -36,7 +36,7 @@ import ( _ "tailscale.com/ssh/tailssh" _ "tailscale.com/syncs" _ "tailscale.com/tailcfg" - _ "tailscale.com/tsweb" + _ "tailscale.com/tsweb/varz" _ "tailscale.com/types/flagtype" _ "tailscale.com/types/key" _ "tailscale.com/types/logger" diff --git a/tstest/integration/tailscaled_deps_test_linux.go b/tstest/integration/tailscaled_deps_test_linux.go index c8d6fc23a..edb891ee9 100644 --- a/tstest/integration/tailscaled_deps_test_linux.go +++ b/tstest/integration/tailscaled_deps_test_linux.go @@ -36,7 +36,7 @@ import ( _ "tailscale.com/ssh/tailssh" _ "tailscale.com/syncs" _ "tailscale.com/tailcfg" - _ "tailscale.com/tsweb" + _ "tailscale.com/tsweb/varz" _ "tailscale.com/types/flagtype" _ "tailscale.com/types/key" _ "tailscale.com/types/logger" diff --git a/tstest/integration/tailscaled_deps_test_openbsd.go b/tstest/integration/tailscaled_deps_test_openbsd.go index c8d6fc23a..edb891ee9 100644 --- a/tstest/integration/tailscaled_deps_test_openbsd.go +++ b/tstest/integration/tailscaled_deps_test_openbsd.go @@ -36,7 +36,7 @@ import ( _ "tailscale.com/ssh/tailssh" _ "tailscale.com/syncs" _ "tailscale.com/tailcfg" - _ "tailscale.com/tsweb" + _ "tailscale.com/tsweb/varz" _ "tailscale.com/types/flagtype" _ "tailscale.com/types/key" _ "tailscale.com/types/logger" diff --git a/tstest/integration/tailscaled_deps_test_windows.go b/tstest/integration/tailscaled_deps_test_windows.go index bb69d70ae..ef995d88e 100644 --- a/tstest/integration/tailscaled_deps_test_windows.go +++ b/tstest/integration/tailscaled_deps_test_windows.go @@ -43,7 +43,7 @@ import ( _ "tailscale.com/smallzstd" _ "tailscale.com/syncs" _ "tailscale.com/tailcfg" - _ "tailscale.com/tsweb" + _ "tailscale.com/tsweb/varz" _ "tailscale.com/types/flagtype" _ "tailscale.com/types/key" _ "tailscale.com/types/logger" diff --git a/tsweb/debug.go b/tsweb/debug.go index 7c231aa45..be2f0a0b0 100644 --- a/tsweb/debug.go +++ b/tsweb/debug.go @@ -14,6 +14,8 @@ import ( "os" "runtime" + "tailscale.com/tsweb/promvarz" + "tailscale.com/tsweb/varz" "tailscale.com/version" ) @@ -51,10 +53,10 @@ func Debugger(mux *http.ServeMux) *DebugHandler { // index page. The /pprof/ index already covers it. mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) - ret.KVFunc("Uptime", func() any { return Uptime() }) + ret.KVFunc("Uptime", func() any { return varz.Uptime() }) ret.KV("Version", version.Long()) ret.Handle("vars", "Metrics (Go)", expvar.Handler()) - ret.Handle("varz", "Metrics (Prometheus)", http.HandlerFunc(CombinedVarzHandler)) + ret.Handle("varz", "Metrics (Prometheus)", http.HandlerFunc(promvarz.Handler)) ret.Handle("pprof/", "pprof", http.HandlerFunc(pprof.Index)) ret.URL("/debug/pprof/goroutine?debug=1", "Goroutines (collapsed)") ret.URL("/debug/pprof/goroutine?debug=2", "Goroutines (full)") diff --git a/tsweb/promvarz/promvarz.go b/tsweb/promvarz/promvarz.go new file mode 100644 index 000000000..9740aeeca --- /dev/null +++ b/tsweb/promvarz/promvarz.go @@ -0,0 +1,48 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package promvarz combines Prometheus metrics exported by our expvar converter +// (tsweb/varz) with metrics exported by the official Prometheus client. +package promvarz + +import ( + "fmt" + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/expfmt" + "tailscale.com/tsweb/varz" +) + +// Handler returns Prometheus metrics exported by our expvar converter +// and the official Prometheus client. +func Handler(w http.ResponseWriter, r *http.Request) { + if err := gatherNativePrometheusMetrics(w); err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + varz.Handler(w, r) +} + +// gatherNativePrometheusMetrics writes metrics from the default +// metric registry in text format. +func gatherNativePrometheusMetrics(w http.ResponseWriter) error { + enc := expfmt.NewEncoder(w, expfmt.FmtText) + mfs, err := prometheus.DefaultGatherer.Gather() + if err != nil { + return fmt.Errorf("could not gather metrics from DefaultGatherer: %w", err) + } + + for _, mf := range mfs { + if err := enc.Encode(mf); err != nil { + return fmt.Errorf("could not encode metric %v: %w", mf, err) + } + } + if closer, ok := enc.(expfmt.Closer); ok { + if err := closer.Close(); err != nil { + return err + } + } + return nil +} diff --git a/tsweb/promvarz/promvarz_test.go b/tsweb/promvarz/promvarz_test.go new file mode 100644 index 000000000..28c906c57 --- /dev/null +++ b/tsweb/promvarz/promvarz_test.go @@ -0,0 +1,35 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause +package promvarz + +import ( + "expvar" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestHandler(t *testing.T) { + test1 := expvar.NewInt("gauge_promvarz_test_expvar") + test1.Set(42) + test2 := promauto.NewGauge(prometheus.GaugeOpts{Name: "promvarz_test_native"}) + test2.Set(4242) + + svr := httptest.NewServer(http.HandlerFunc(Handler)) + defer svr.Close() + + want := ` + # TYPE promvarz_test_expvar gauge + promvarz_test_expvar 42 + # TYPE promvarz_test_native gauge + promvarz_test_native 4242 + ` + if err := testutil.ScrapeAndCompare(svr.URL, strings.NewReader(want), "promvarz_test_expvar", "promvarz_test_native"); err != nil { + t.Error(err) + } +} diff --git a/tsweb/tsweb.go b/tsweb/tsweb.go index 3739d18ea..a84aacbc9 100644 --- a/tsweb/tsweb.go +++ b/tsweb/tsweb.go @@ -11,49 +11,25 @@ import ( "errors" "expvar" "fmt" - "io" "net" "net/http" _ "net/http/pprof" "net/netip" "os" "path/filepath" - "reflect" - "runtime" - "sort" "strconv" "strings" "sync" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/expfmt" "go4.org/mem" "tailscale.com/envknob" - "tailscale.com/metrics" "tailscale.com/net/tsaddr" + "tailscale.com/tsweb/varz" "tailscale.com/types/logger" "tailscale.com/util/vizerror" - "tailscale.com/version" ) -func init() { - expvar.Publish("process_start_unix_time", expvar.Func(func() any { return timeStart.Unix() })) - expvar.Publish("version", expvar.Func(func() any { return version.Long() })) - expvar.Publish("go_version", expvar.Func(func() any { return runtime.Version() })) - expvar.Publish("counter_uptime_sec", expvar.Func(func() any { return int64(Uptime().Seconds()) })) - expvar.Publish("gauge_goroutines", expvar.Func(func() any { return runtime.NumGoroutine() })) -} - -const ( - gaugePrefix = "gauge_" - counterPrefix = "counter_" - labelMapPrefix = "labelmap_" -) - -// prefixesToTrim contains key prefixes to remove when exporting and sorting metrics. -var prefixesToTrim = []string{gaugePrefix, counterPrefix, labelMapPrefix} - // DevMode controls whether extra output in shown, for when the binary is being run in dev mode. var DevMode bool @@ -143,10 +119,6 @@ func Protected(h http.Handler) http.Handler { }) } -var timeStart = time.Now() - -func Uptime() time.Duration { return time.Since(timeStart).Round(time.Second) } - // Port80Handler is the handler to be given to // autocert.Manager.HTTPHandler. The inner handler is the mux // returned by NewMux containing registered /debug handlers. @@ -448,364 +420,8 @@ func Error(code int, msg string, err error) HTTPError { return HTTPError{Code: code, Msg: msg, Err: err} } -// WritePrometheusExpvar writes kv to w in Prometheus metrics format. -// -// See VarzHandler for conventions. This is exported primarily for -// people to test their varz. -func WritePrometheusExpvar(w io.Writer, kv expvar.KeyValue) { - writePromExpVar(w, "", kv) -} - -type prometheusMetricDetails struct { - Name string - Type string - Label string -} - -var prometheusMetricCache sync.Map // string => *prometheusMetricDetails - -func prometheusMetric(prefix string, key string) (string, string, string) { - cachekey := prefix + key - if v, ok := prometheusMetricCache.Load(cachekey); ok { - d := v.(*prometheusMetricDetails) - return d.Name, d.Type, d.Label - } - var typ string - var label string - switch { - case strings.HasPrefix(key, gaugePrefix): - typ = "gauge" - key = strings.TrimPrefix(key, gaugePrefix) - - case strings.HasPrefix(key, counterPrefix): - typ = "counter" - key = strings.TrimPrefix(key, counterPrefix) - } - if strings.HasPrefix(key, labelMapPrefix) { - key = strings.TrimPrefix(key, labelMapPrefix) - if a, b, ok := strings.Cut(key, "_"); ok { - label, key = a, b - } - } - d := &prometheusMetricDetails{ - Name: strings.ReplaceAll(prefix+key, "-", "_"), - Type: typ, - Label: label, - } - prometheusMetricCache.Store(cachekey, d) - return d.Name, d.Type, d.Label -} - -func writePromExpVar(w io.Writer, prefix string, kv expvar.KeyValue) { - key := kv.Key - name, typ, label := prometheusMetric(prefix, key) - - switch v := kv.Value.(type) { - case *expvar.Int: - if typ == "" { - typ = "counter" - } - fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, v.Value()) - return - case *expvar.Float: - if typ == "" { - typ = "gauge" - } - fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, v.Value()) - return - case *metrics.Set: - v.Do(func(kv expvar.KeyValue) { - writePromExpVar(w, name+"_", kv) - }) - return - case PrometheusMetricsReflectRooter: - root := v.PrometheusMetricsReflectRoot() - rv := reflect.ValueOf(root) - if rv.Type().Kind() == reflect.Ptr { - if rv.IsNil() { - return - } - rv = rv.Elem() - } - if rv.Type().Kind() != reflect.Struct { - fmt.Fprintf(w, "# skipping expvar %q; unknown root type\n", name) - return - } - foreachExportedStructField(rv, func(fieldOrJSONName, metricType string, rv reflect.Value) { - mname := name + "_" + fieldOrJSONName - switch rv.Kind() { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Int()) - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Uint()) - case reflect.Float32, reflect.Float64: - fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Float()) - case reflect.Struct: - if rv.CanAddr() { - // Slight optimization, not copying big structs if they're addressable: - writePromExpVar(w, name+"_", expvar.KeyValue{Key: fieldOrJSONName, Value: expVarPromStructRoot{rv.Addr().Interface()}}) - } else { - writePromExpVar(w, name+"_", expvar.KeyValue{Key: fieldOrJSONName, Value: expVarPromStructRoot{rv.Interface()}}) - } - } - return - }) - return - } - - if typ == "" { - var funcRet string - if f, ok := kv.Value.(expvar.Func); ok { - v := f() - if ms, ok := v.(runtime.MemStats); ok && name == "memstats" { - writeMemstats(w, &ms) - return - } - if vs, ok := v.(string); ok && strings.HasSuffix(name, "version") { - fmt.Fprintf(w, "%s{version=%q} 1\n", name, vs) - return - } - switch v := v.(type) { - case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, uintptr, float32, float64: - fmt.Fprintf(w, "%s %v\n", name, v) - return - } - funcRet = fmt.Sprintf(" returning %T", v) - } - switch kv.Value.(type) { - default: - fmt.Fprintf(w, "# skipping expvar %q (Go type %T%s) with undeclared Prometheus type\n", name, kv.Value, funcRet) - return - case *metrics.LabelMap, *expvar.Map: - // Permit typeless LabelMap and expvar.Map for - // compatibility with old expvar-registered - // metrics.LabelMap. - } - } - - switch v := kv.Value.(type) { - case expvar.Func: - val := v() - switch val.(type) { - case float64, int64, int: - fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, val) - default: - fmt.Fprintf(w, "# skipping expvar func %q returning unknown type %T\n", name, val) - } - - case *metrics.LabelMap: - if typ != "" { - fmt.Fprintf(w, "# TYPE %s %s\n", name, typ) - } - // IntMap uses expvar.Map on the inside, which presorts - // keys. The output ordering is deterministic. - v.Do(func(kv expvar.KeyValue) { - fmt.Fprintf(w, "%s{%s=%q} %v\n", name, v.Label, kv.Key, kv.Value) - }) - case *expvar.Map: - if label != "" && typ != "" { - fmt.Fprintf(w, "# TYPE %s %s\n", name, typ) - v.Do(func(kv expvar.KeyValue) { - fmt.Fprintf(w, "%s{%s=%q} %v\n", name, label, kv.Key, kv.Value) - }) - } else { - v.Do(func(kv expvar.KeyValue) { - fmt.Fprintf(w, "%s_%s %v\n", name, kv.Key, kv.Value) - }) - } - } -} - -var sortedKVsPool = &sync.Pool{New: func() any { return new(sortedKVs) }} - -// sortedKV is a KeyValue with a sort key. -type sortedKV struct { - expvar.KeyValue - sortKey string // KeyValue.Key with type prefix removed -} - -type sortedKVs struct { - kvs []sortedKV -} - -// VarzHandler is an HTTP handler to write expvar values into the -// prometheus export format: -// -// https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md -// -// It makes the following assumptions: -// -// - *expvar.Int are counters (unless marked as a gauge_; see below) -// - a *tailscale/metrics.Set is descended into, joining keys with -// underscores. So use underscores as your metric names. -// - an expvar named starting with "gauge_" or "counter_" is of that -// Prometheus type, and has that prefix stripped. -// - anything else is untyped and thus not exported. -// - expvar.Func can return an int or int64 (for now) and anything else -// is not exported. -// -// This will evolve over time, or perhaps be replaced. +// VarzHandler writes expvar values as Prometheus metrics. +// TODO: migrate all users to varz.Handler or promvarz.Handler and remove this. func VarzHandler(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain; version=0.0.4") - - s := sortedKVsPool.Get().(*sortedKVs) - defer sortedKVsPool.Put(s) - s.kvs = s.kvs[:0] - expvarDo(func(kv expvar.KeyValue) { - s.kvs = append(s.kvs, sortedKV{kv, removeTypePrefixes(kv.Key)}) - }) - sort.Slice(s.kvs, func(i, j int) bool { - return s.kvs[i].sortKey < s.kvs[j].sortKey - }) - for _, e := range s.kvs { - writePromExpVar(w, "", e.KeyValue) - } -} - -// CombinedVarzHandler is an HTTP handler for Prometheus metrics that -// combines native metrics with the ones converted from expvar. -func CombinedVarzHandler(w http.ResponseWriter, r *http.Request) { - if err := gatherNativePrometheusMetrics(w); err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(err.Error())) - return - } - VarzHandler(w, r) + varz.Handler(w, r) } - -// gatherNativePrometheusMetrics writes metrics from the default -// metric registry in text format. -func gatherNativePrometheusMetrics(w http.ResponseWriter) error { - enc := expfmt.NewEncoder(w, expfmt.FmtText) - mfs, err := prometheus.DefaultGatherer.Gather() - if err != nil { - return fmt.Errorf("could not gather metrics from DefaultGatherer: %w", err) - } - - for _, mf := range mfs { - if err := enc.Encode(mf); err != nil { - return fmt.Errorf("could not encode metric %v: %w", mf, err) - } - } - if closer, ok := enc.(expfmt.Closer); ok { - if err := closer.Close(); err != nil { - return err - } - } - return nil -} - -// PrometheusMetricsReflectRooter is an optional interface that expvar.Var implementations -// can implement to indicate that they should be walked recursively with reflect to find -// sets of fields to export. -type PrometheusMetricsReflectRooter interface { - expvar.Var - - // PrometheusMetricsReflectRoot returns the struct or struct pointer to walk. - PrometheusMetricsReflectRoot() any -} - -var expvarDo = expvar.Do // pulled out for tests - -func writeMemstats(w io.Writer, ms *runtime.MemStats) { - out := func(name, typ string, v uint64, help string) { - if help != "" { - fmt.Fprintf(w, "# HELP memstats_%s %s\n", name, help) - } - fmt.Fprintf(w, "# TYPE memstats_%s %s\nmemstats_%s %v\n", name, typ, name, v) - } - g := func(name string, v uint64, help string) { out(name, "gauge", v, help) } - c := func(name string, v uint64, help string) { out(name, "counter", v, help) } - g("heap_alloc", ms.HeapAlloc, "current bytes of allocated heap objects (up/down smoothly)") - c("total_alloc", ms.TotalAlloc, "cumulative bytes allocated for heap objects") - g("sys", ms.Sys, "total bytes of memory obtained from the OS") - c("mallocs", ms.Mallocs, "cumulative count of heap objects allocated") - c("frees", ms.Frees, "cumulative count of heap objects freed") - c("num_gc", uint64(ms.NumGC), "number of completed GC cycles") -} - -// sortedStructField is metadata about a struct field used both for sorting once -// (by structTypeSortedFields) and at serving time (by -// foreachExportedStructField). -type sortedStructField struct { - Index int // index of struct field in struct - Name string // struct field name, or "json" name - SortName string // Name with "foo_" type prefixes removed - MetricType string // the "metrictype" struct tag - StructFieldType *reflect.StructField -} - -var structSortedFieldsCache sync.Map // reflect.Type => []sortedStructField - -// structTypeSortedFields returns the sorted fields of t, caching as needed. -func structTypeSortedFields(t reflect.Type) []sortedStructField { - if v, ok := structSortedFieldsCache.Load(t); ok { - return v.([]sortedStructField) - } - fields := make([]sortedStructField, 0, t.NumField()) - for i, n := 0, t.NumField(); i < n; i++ { - sf := t.Field(i) - name := sf.Name - if v := sf.Tag.Get("json"); v != "" { - v, _, _ = strings.Cut(v, ",") - if v == "-" { - // Skip it, regardless of its metrictype. - continue - } - if v != "" { - name = v - } - } - fields = append(fields, sortedStructField{ - Index: i, - Name: name, - SortName: removeTypePrefixes(name), - MetricType: sf.Tag.Get("metrictype"), - StructFieldType: &sf, - }) - } - sort.Slice(fields, func(i, j int) bool { - return fields[i].SortName < fields[j].SortName - }) - structSortedFieldsCache.Store(t, fields) - return fields -} - -// removeTypePrefixes returns s with the first "foo_" prefix in prefixesToTrim -// removed. -func removeTypePrefixes(s string) string { - for _, prefix := range prefixesToTrim { - if trimmed, ok := strings.CutPrefix(s, prefix); ok { - return trimmed - } - } - return s -} - -// foreachExportedStructField iterates over the fields in sorted order of -// their name, after removing metric prefixes. This is not necessarily the -// order they were declared in the struct -func foreachExportedStructField(rv reflect.Value, f func(fieldOrJSONName, metricType string, rv reflect.Value)) { - t := rv.Type() - for _, ssf := range structTypeSortedFields(t) { - sf := ssf.StructFieldType - if ssf.MetricType != "" || sf.Type.Kind() == reflect.Struct { - f(ssf.Name, ssf.MetricType, rv.Field(ssf.Index)) - } else if sf.Type.Kind() == reflect.Ptr && sf.Type.Elem().Kind() == reflect.Struct { - fv := rv.Field(ssf.Index) - if !fv.IsNil() { - f(ssf.Name, ssf.MetricType, fv.Elem()) - } - } - } -} - -type expVarPromStructRoot struct{ v any } - -func (r expVarPromStructRoot) PrometheusMetricsReflectRoot() any { return r.v } -func (r expVarPromStructRoot) String() string { panic("unused") } - -var ( - _ PrometheusMetricsReflectRooter = expVarPromStructRoot{} - _ expvar.Var = expVarPromStructRoot{} -) diff --git a/tsweb/tsweb_test.go b/tsweb/tsweb_test.go index cf67d474f..6d72c0d3d 100644 --- a/tsweb/tsweb_test.go +++ b/tsweb/tsweb_test.go @@ -7,21 +7,17 @@ import ( "bufio" "context" "errors" - "expvar" "fmt" "net" "net/http" "net/http/httptest" - "reflect" "strings" "testing" "time" "github.com/google/go-cmp/cmp" - "tailscale.com/metrics" "tailscale.com/tstest" "tailscale.com/util/vizerror" - "tailscale.com/version" ) type noopHijacker struct { @@ -370,323 +366,6 @@ func TestHTTPError_Unwrap(t *testing.T) { } } -func TestVarzHandler(t *testing.T) { - t.Run("globals_log", func(t *testing.T) { - rec := httptest.NewRecorder() - VarzHandler(rec, httptest.NewRequest("GET", "/", nil)) - t.Logf("Got: %s", rec.Body.Bytes()) - }) - - half := new(expvar.Float) - half.Set(0.5) - - tests := []struct { - name string - k string // key name - v expvar.Var - want string - }{ - { - "int", - "foo", - new(expvar.Int), - "# TYPE foo counter\nfoo 0\n", - }, - { - "dash_in_metric_name", - "counter_foo-bar", - new(expvar.Int), - "# TYPE foo_bar counter\nfoo_bar 0\n", - }, - { - "int_with_type_counter", - "counter_foo", - new(expvar.Int), - "# TYPE foo counter\nfoo 0\n", - }, - { - "int_with_type_gauge", - "gauge_foo", - new(expvar.Int), - "# TYPE foo gauge\nfoo 0\n", - }, - { - // For a float = 0.0, Prometheus client_golang outputs "0" - "float_zero", - "foo", - new(expvar.Float), - "# TYPE foo gauge\nfoo 0\n", - }, - { - "float_point_5", - "foo", - half, - "# TYPE foo gauge\nfoo 0.5\n", - }, - { - "float_with_type_counter", - "counter_foo", - half, - "# TYPE foo counter\nfoo 0.5\n", - }, - { - "float_with_type_gauge", - "gauge_foo", - half, - "# TYPE foo gauge\nfoo 0.5\n", - }, - { - "metrics_set", - "s", - &metrics.Set{ - Map: *(func() *expvar.Map { - m := new(expvar.Map) - m.Init() - m.Add("foo", 1) - m.Add("bar", 2) - return m - })(), - }, - "# TYPE s_bar counter\ns_bar 2\n# TYPE s_foo counter\ns_foo 1\n", - }, - { - "metrics_set_TODO_gauge_type", - "gauge_s", // TODO(bradfitz): arguably a bug; should pass down type - &metrics.Set{ - Map: *(func() *expvar.Map { - m := new(expvar.Map) - m.Init() - m.Add("foo", 1) - m.Add("bar", 2) - return m - })(), - }, - "# TYPE s_bar counter\ns_bar 2\n# TYPE s_foo counter\ns_foo 1\n", - }, - { - "expvar_map_untyped", - "api_status_code", - func() *expvar.Map { - m := new(expvar.Map) - m.Init() - m.Add("2xx", 100) - m.Add("5xx", 2) - return m - }(), - "api_status_code_2xx 100\napi_status_code_5xx 2\n", - }, - { - "func_float64", - "counter_x", - expvar.Func(func() any { return float64(1.2) }), - "# TYPE x counter\nx 1.2\n", - }, - { - "func_float64_gauge", - "gauge_y", - expvar.Func(func() any { return float64(1.2) }), - "# TYPE y gauge\ny 1.2\n", - }, - { - "func_float64_untyped", - "z", - expvar.Func(func() any { return float64(1.2) }), - "z 1.2\n", - }, - { - "metrics_label_map", - "counter_m", - &metrics.LabelMap{ - Label: "label", - Map: *(func() *expvar.Map { - m := new(expvar.Map) - m.Init() - m.Add("foo", 1) - m.Add("bar", 2) - return m - })(), - }, - "# TYPE m counter\nm{label=\"bar\"} 2\nm{label=\"foo\"} 1\n", - }, - { - "metrics_label_map_untyped", - "control_save_config", - (func() *metrics.LabelMap { - m := &metrics.LabelMap{Label: "reason"} - m.Add("new", 1) - m.Add("updated", 1) - m.Add("fun", 1) - return m - })(), - "control_save_config{reason=\"fun\"} 1\ncontrol_save_config{reason=\"new\"} 1\ncontrol_save_config{reason=\"updated\"} 1\n", - }, - { - "expvar_label_map", - "counter_labelmap_keyname_m", - func() *expvar.Map { - m := new(expvar.Map) - m.Init() - m.Add("foo", 1) - m.Add("bar", 2) - return m - }(), - "# TYPE m counter\nm{keyname=\"bar\"} 2\nm{keyname=\"foo\"} 1\n", - }, - { - "struct_reflect", - "foo", - someExpVarWithJSONAndPromTypes(), - strings.TrimSpace(` -# TYPE foo_AUint16 counter -foo_AUint16 65535 -# TYPE foo_AnInt8 counter -foo_AnInt8 127 -# TYPE foo_curTemp gauge -foo_curTemp 20.6 -# TYPE foo_curX gauge -foo_curX 3 -# TYPE foo_nestptr_bar counter -foo_nestptr_bar 20 -# TYPE foo_nestptr_foo gauge -foo_nestptr_foo 10 -# TYPE foo_nestvalue_bar counter -foo_nestvalue_bar 2 -# TYPE foo_nestvalue_foo gauge -foo_nestvalue_foo 1 -# TYPE foo_totalY counter -foo_totalY 4 -`) + "\n", - }, - { - "struct_reflect_nil_root", - "foo", - expvarAdapter{(*SomeStats)(nil)}, - "", - }, - { - "func_returning_int", - "num_goroutines", - expvar.Func(func() any { return 123 }), - "num_goroutines 123\n", - }, - { - "string_version_var", - "foo_version", - expvar.Func(func() any { return "1.2.3-foo15" }), - "foo_version{version=\"1.2.3-foo15\"} 1\n", - }, - { - "field_ordering", - "foo", - someExpVarWithFieldNamesSorting(), - strings.TrimSpace(` -# TYPE foo_bar_a gauge -foo_bar_a 1 -# TYPE foo_bar_b counter -foo_bar_b 1 -# TYPE foo_foo_a gauge -foo_foo_a 1 -# TYPE foo_foo_b counter -foo_foo_b 1 -`) + "\n", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tstest.Replace(t, &expvarDo, func(f func(expvar.KeyValue)) { - f(expvar.KeyValue{Key: tt.k, Value: tt.v}) - }) - rec := httptest.NewRecorder() - VarzHandler(rec, httptest.NewRequest("GET", "/", nil)) - if got := rec.Body.Bytes(); string(got) != tt.want { - t.Errorf("mismatch\n got: %q\n%s\nwant: %q\n%s\n", got, got, tt.want, tt.want) - } - }) - } -} - -type SomeNested struct { - FooG int64 `json:"foo" metrictype:"gauge"` - BarC int64 `json:"bar" metrictype:"counter"` - Omit int `json:"-" metrictype:"counter"` -} - -type SomeStats struct { - Nested SomeNested `json:"nestvalue"` - NestedPtr *SomeNested `json:"nestptr"` - NestedNilPtr *SomeNested `json:"nestnilptr"` - CurX int `json:"curX" metrictype:"gauge"` - NoMetricType int `json:"noMetric" metrictype:""` - TotalY int64 `json:"totalY,omitempty" metrictype:"counter"` - CurTemp float64 `json:"curTemp" metrictype:"gauge"` - AnInt8 int8 `metrictype:"counter"` - AUint16 uint16 `metrictype:"counter"` -} - -// someExpVarWithJSONAndPromTypes returns an expvar.Var that -// implements PrometheusMetricsReflectRooter for TestVarzHandler. -func someExpVarWithJSONAndPromTypes() expvar.Var { - st := &SomeStats{ - Nested: SomeNested{ - FooG: 1, - BarC: 2, - Omit: 3, - }, - NestedPtr: &SomeNested{ - FooG: 10, - BarC: 20, - }, - CurX: 3, - TotalY: 4, - CurTemp: 20.6, - AnInt8: 127, - AUint16: 65535, - } - return expvarAdapter{st} -} - -type expvarAdapter struct { - st *SomeStats -} - -func (expvarAdapter) String() string { return "{}" } // expvar JSON; unused in test - -func (a expvarAdapter) PrometheusMetricsReflectRoot() any { - return a.st -} - -// SomeTestOfFieldNamesSorting demonstrates field -// names that are not in sorted in declaration order, to verify -// that we sort based on field name -type SomeTestOfFieldNamesSorting struct { - FooAG int64 `json:"foo_a" metrictype:"gauge"` - BarAG int64 `json:"bar_a" metrictype:"gauge"` - FooBC int64 `json:"foo_b" metrictype:"counter"` - BarBC int64 `json:"bar_b" metrictype:"counter"` -} - -// someExpVarWithFieldNamesSorting returns an expvar.Var that -// implements PrometheusMetricsReflectRooter for TestVarzHandler. -func someExpVarWithFieldNamesSorting() expvar.Var { - st := &SomeTestOfFieldNamesSorting{ - FooAG: 1, - BarAG: 1, - FooBC: 1, - BarBC: 1, - } - return expvarAdapter2{st} -} - -type expvarAdapter2 struct { - st *SomeTestOfFieldNamesSorting -} - -func (expvarAdapter2) String() string { return "{}" } // expvar JSON; unused in test - -func (a expvarAdapter2) PrometheusMetricsReflectRoot() any { - return a.st -} - func TestAcceptsEncoding(t *testing.T) { tests := []struct { in, enc string @@ -756,45 +435,3 @@ func TestPort80Handler(t *testing.T) { }) } } - -func TestSortedStructAllocs(t *testing.T) { - f := reflect.ValueOf(struct { - Foo int - Bar int - Baz int - }{}) - n := testing.AllocsPerRun(1000, func() { - foreachExportedStructField(f, func(fieldOrJSONName, metricType string, rv reflect.Value) { - // Nothing. - }) - }) - if n != 0 { - t.Errorf("allocs = %v; want 0", n) - } -} - -func TestVarzHandlerSorting(t *testing.T) { - tstest.Replace(t, &expvarDo, func(f func(expvar.KeyValue)) { - f(expvar.KeyValue{Key: "counter_zz", Value: new(expvar.Int)}) - f(expvar.KeyValue{Key: "gauge_aa", Value: new(expvar.Int)}) - }) - rec := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/", nil) - VarzHandler(rec, req) - got := rec.Body.Bytes() - const want = "# TYPE aa gauge\naa 0\n# TYPE zz counter\nzz 0\n" - if string(got) != want { - t.Errorf("got %q; want %q", got, want) - } - rec = new(httptest.ResponseRecorder) // without a body - - // Lock in the current number of allocs, to prevent it from growing. - if !version.IsRace() { - allocs := int(testing.AllocsPerRun(1000, func() { - VarzHandler(rec, req) - })) - if max := 13; allocs > max { - t.Errorf("allocs = %v; want max %v", allocs, max) - } - } -} diff --git a/tsweb/varz/varz.go b/tsweb/varz/varz.go new file mode 100644 index 000000000..024d4ea48 --- /dev/null +++ b/tsweb/varz/varz.go @@ -0,0 +1,372 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package varz contains code to export metrics in Prometheus format. +package varz + +import ( + "expvar" + "fmt" + "io" + "net/http" + _ "net/http/pprof" + "reflect" + "runtime" + "sort" + "strings" + "sync" + "time" + + "tailscale.com/metrics" + "tailscale.com/version" +) + +func init() { + expvar.Publish("process_start_unix_time", expvar.Func(func() any { return timeStart.Unix() })) + expvar.Publish("version", expvar.Func(func() any { return version.Long() })) + expvar.Publish("go_version", expvar.Func(func() any { return runtime.Version() })) + expvar.Publish("counter_uptime_sec", expvar.Func(func() any { return int64(Uptime().Seconds()) })) + expvar.Publish("gauge_goroutines", expvar.Func(func() any { return runtime.NumGoroutine() })) +} + +const ( + gaugePrefix = "gauge_" + counterPrefix = "counter_" + labelMapPrefix = "labelmap_" +) + +// prefixesToTrim contains key prefixes to remove when exporting and sorting metrics. +var prefixesToTrim = []string{gaugePrefix, counterPrefix, labelMapPrefix} + +var timeStart = time.Now() + +func Uptime() time.Duration { return time.Since(timeStart).Round(time.Second) } + +// WritePrometheusExpvar writes kv to w in Prometheus metrics format. +// +// See VarzHandler for conventions. This is exported primarily for +// people to test their varz. +func WritePrometheusExpvar(w io.Writer, kv expvar.KeyValue) { + writePromExpVar(w, "", kv) +} + +type prometheusMetricDetails struct { + Name string + Type string + Label string +} + +var prometheusMetricCache sync.Map // string => *prometheusMetricDetails + +func prometheusMetric(prefix string, key string) (string, string, string) { + cachekey := prefix + key + if v, ok := prometheusMetricCache.Load(cachekey); ok { + d := v.(*prometheusMetricDetails) + return d.Name, d.Type, d.Label + } + var typ string + var label string + switch { + case strings.HasPrefix(key, gaugePrefix): + typ = "gauge" + key = strings.TrimPrefix(key, gaugePrefix) + + case strings.HasPrefix(key, counterPrefix): + typ = "counter" + key = strings.TrimPrefix(key, counterPrefix) + } + if strings.HasPrefix(key, labelMapPrefix) { + key = strings.TrimPrefix(key, labelMapPrefix) + if a, b, ok := strings.Cut(key, "_"); ok { + label, key = a, b + } + } + d := &prometheusMetricDetails{ + Name: strings.ReplaceAll(prefix+key, "-", "_"), + Type: typ, + Label: label, + } + prometheusMetricCache.Store(cachekey, d) + return d.Name, d.Type, d.Label +} + +func writePromExpVar(w io.Writer, prefix string, kv expvar.KeyValue) { + key := kv.Key + name, typ, label := prometheusMetric(prefix, key) + + switch v := kv.Value.(type) { + case *expvar.Int: + if typ == "" { + typ = "counter" + } + fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, v.Value()) + return + case *expvar.Float: + if typ == "" { + typ = "gauge" + } + fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, v.Value()) + return + case *metrics.Set: + v.Do(func(kv expvar.KeyValue) { + writePromExpVar(w, name+"_", kv) + }) + return + case PrometheusMetricsReflectRooter: + root := v.PrometheusMetricsReflectRoot() + rv := reflect.ValueOf(root) + if rv.Type().Kind() == reflect.Ptr { + if rv.IsNil() { + return + } + rv = rv.Elem() + } + if rv.Type().Kind() != reflect.Struct { + fmt.Fprintf(w, "# skipping expvar %q; unknown root type\n", name) + return + } + foreachExportedStructField(rv, func(fieldOrJSONName, metricType string, rv reflect.Value) { + mname := name + "_" + fieldOrJSONName + switch rv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Int()) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Uint()) + case reflect.Float32, reflect.Float64: + fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Float()) + case reflect.Struct: + if rv.CanAddr() { + // Slight optimization, not copying big structs if they're addressable: + writePromExpVar(w, name+"_", expvar.KeyValue{Key: fieldOrJSONName, Value: expVarPromStructRoot{rv.Addr().Interface()}}) + } else { + writePromExpVar(w, name+"_", expvar.KeyValue{Key: fieldOrJSONName, Value: expVarPromStructRoot{rv.Interface()}}) + } + } + return + }) + return + } + + if typ == "" { + var funcRet string + if f, ok := kv.Value.(expvar.Func); ok { + v := f() + if ms, ok := v.(runtime.MemStats); ok && name == "memstats" { + writeMemstats(w, &ms) + return + } + if vs, ok := v.(string); ok && strings.HasSuffix(name, "version") { + fmt.Fprintf(w, "%s{version=%q} 1\n", name, vs) + return + } + switch v := v.(type) { + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, uintptr, float32, float64: + fmt.Fprintf(w, "%s %v\n", name, v) + return + } + funcRet = fmt.Sprintf(" returning %T", v) + } + switch kv.Value.(type) { + default: + fmt.Fprintf(w, "# skipping expvar %q (Go type %T%s) with undeclared Prometheus type\n", name, kv.Value, funcRet) + return + case *metrics.LabelMap, *expvar.Map: + // Permit typeless LabelMap and expvar.Map for + // compatibility with old expvar-registered + // metrics.LabelMap. + } + } + + switch v := kv.Value.(type) { + case expvar.Func: + val := v() + switch val.(type) { + case float64, int64, int: + fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, val) + default: + fmt.Fprintf(w, "# skipping expvar func %q returning unknown type %T\n", name, val) + } + + case *metrics.LabelMap: + if typ != "" { + fmt.Fprintf(w, "# TYPE %s %s\n", name, typ) + } + // IntMap uses expvar.Map on the inside, which presorts + // keys. The output ordering is deterministic. + v.Do(func(kv expvar.KeyValue) { + fmt.Fprintf(w, "%s{%s=%q} %v\n", name, v.Label, kv.Key, kv.Value) + }) + case *expvar.Map: + if label != "" && typ != "" { + fmt.Fprintf(w, "# TYPE %s %s\n", name, typ) + v.Do(func(kv expvar.KeyValue) { + fmt.Fprintf(w, "%s{%s=%q} %v\n", name, label, kv.Key, kv.Value) + }) + } else { + v.Do(func(kv expvar.KeyValue) { + fmt.Fprintf(w, "%s_%s %v\n", name, kv.Key, kv.Value) + }) + } + } +} + +var sortedKVsPool = &sync.Pool{New: func() any { return new(sortedKVs) }} + +// sortedKV is a KeyValue with a sort key. +type sortedKV struct { + expvar.KeyValue + sortKey string // KeyValue.Key with type prefix removed +} + +type sortedKVs struct { + kvs []sortedKV +} + +// Handler is an HTTP handler to write expvar values into the +// prometheus export format: +// +// https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md +// +// It makes the following assumptions: +// +// - *expvar.Int are counters (unless marked as a gauge_; see below) +// - a *tailscale/metrics.Set is descended into, joining keys with +// underscores. So use underscores as your metric names. +// - an expvar named starting with "gauge_" or "counter_" is of that +// Prometheus type, and has that prefix stripped. +// - anything else is untyped and thus not exported. +// - expvar.Func can return an int or int64 (for now) and anything else +// is not exported. +// +// This will evolve over time, or perhaps be replaced. +func Handler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; version=0.0.4") + + s := sortedKVsPool.Get().(*sortedKVs) + defer sortedKVsPool.Put(s) + s.kvs = s.kvs[:0] + expvarDo(func(kv expvar.KeyValue) { + s.kvs = append(s.kvs, sortedKV{kv, removeTypePrefixes(kv.Key)}) + }) + sort.Slice(s.kvs, func(i, j int) bool { + return s.kvs[i].sortKey < s.kvs[j].sortKey + }) + for _, e := range s.kvs { + writePromExpVar(w, "", e.KeyValue) + } +} + +// PrometheusMetricsReflectRooter is an optional interface that expvar.Var implementations +// can implement to indicate that they should be walked recursively with reflect to find +// sets of fields to export. +type PrometheusMetricsReflectRooter interface { + expvar.Var + + // PrometheusMetricsReflectRoot returns the struct or struct pointer to walk. + PrometheusMetricsReflectRoot() any +} + +var expvarDo = expvar.Do // pulled out for tests + +func writeMemstats(w io.Writer, ms *runtime.MemStats) { + out := func(name, typ string, v uint64, help string) { + if help != "" { + fmt.Fprintf(w, "# HELP memstats_%s %s\n", name, help) + } + fmt.Fprintf(w, "# TYPE memstats_%s %s\nmemstats_%s %v\n", name, typ, name, v) + } + g := func(name string, v uint64, help string) { out(name, "gauge", v, help) } + c := func(name string, v uint64, help string) { out(name, "counter", v, help) } + g("heap_alloc", ms.HeapAlloc, "current bytes of allocated heap objects (up/down smoothly)") + c("total_alloc", ms.TotalAlloc, "cumulative bytes allocated for heap objects") + g("sys", ms.Sys, "total bytes of memory obtained from the OS") + c("mallocs", ms.Mallocs, "cumulative count of heap objects allocated") + c("frees", ms.Frees, "cumulative count of heap objects freed") + c("num_gc", uint64(ms.NumGC), "number of completed GC cycles") +} + +// sortedStructField is metadata about a struct field used both for sorting once +// (by structTypeSortedFields) and at serving time (by +// foreachExportedStructField). +type sortedStructField struct { + Index int // index of struct field in struct + Name string // struct field name, or "json" name + SortName string // Name with "foo_" type prefixes removed + MetricType string // the "metrictype" struct tag + StructFieldType *reflect.StructField +} + +var structSortedFieldsCache sync.Map // reflect.Type => []sortedStructField + +// structTypeSortedFields returns the sorted fields of t, caching as needed. +func structTypeSortedFields(t reflect.Type) []sortedStructField { + if v, ok := structSortedFieldsCache.Load(t); ok { + return v.([]sortedStructField) + } + fields := make([]sortedStructField, 0, t.NumField()) + for i, n := 0, t.NumField(); i < n; i++ { + sf := t.Field(i) + name := sf.Name + if v := sf.Tag.Get("json"); v != "" { + v, _, _ = strings.Cut(v, ",") + if v == "-" { + // Skip it, regardless of its metrictype. + continue + } + if v != "" { + name = v + } + } + fields = append(fields, sortedStructField{ + Index: i, + Name: name, + SortName: removeTypePrefixes(name), + MetricType: sf.Tag.Get("metrictype"), + StructFieldType: &sf, + }) + } + sort.Slice(fields, func(i, j int) bool { + return fields[i].SortName < fields[j].SortName + }) + structSortedFieldsCache.Store(t, fields) + return fields +} + +// removeTypePrefixes returns s with the first "foo_" prefix in prefixesToTrim +// removed. +func removeTypePrefixes(s string) string { + for _, prefix := range prefixesToTrim { + if trimmed, ok := strings.CutPrefix(s, prefix); ok { + return trimmed + } + } + return s +} + +// foreachExportedStructField iterates over the fields in sorted order of +// their name, after removing metric prefixes. This is not necessarily the +// order they were declared in the struct +func foreachExportedStructField(rv reflect.Value, f func(fieldOrJSONName, metricType string, rv reflect.Value)) { + t := rv.Type() + for _, ssf := range structTypeSortedFields(t) { + sf := ssf.StructFieldType + if ssf.MetricType != "" || sf.Type.Kind() == reflect.Struct { + f(ssf.Name, ssf.MetricType, rv.Field(ssf.Index)) + } else if sf.Type.Kind() == reflect.Ptr && sf.Type.Elem().Kind() == reflect.Struct { + fv := rv.Field(ssf.Index) + if !fv.IsNil() { + f(ssf.Name, ssf.MetricType, fv.Elem()) + } + } + } +} + +type expVarPromStructRoot struct{ v any } + +func (r expVarPromStructRoot) PrometheusMetricsReflectRoot() any { return r.v } +func (r expVarPromStructRoot) String() string { panic("unused") } + +var ( + _ PrometheusMetricsReflectRooter = expVarPromStructRoot{} + _ expvar.Var = expVarPromStructRoot{} +) diff --git a/tsweb/varz/varz_test.go b/tsweb/varz/varz_test.go new file mode 100644 index 000000000..e46caeccf --- /dev/null +++ b/tsweb/varz/varz_test.go @@ -0,0 +1,375 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package varz + +import ( + "expvar" + "net/http/httptest" + "reflect" + "strings" + "testing" + + "tailscale.com/metrics" + "tailscale.com/tstest" + "tailscale.com/version" +) + +func TestVarzHandler(t *testing.T) { + t.Run("globals_log", func(t *testing.T) { + rec := httptest.NewRecorder() + Handler(rec, httptest.NewRequest("GET", "/", nil)) + t.Logf("Got: %s", rec.Body.Bytes()) + }) + + half := new(expvar.Float) + half.Set(0.5) + + tests := []struct { + name string + k string // key name + v expvar.Var + want string + }{ + { + "int", + "foo", + new(expvar.Int), + "# TYPE foo counter\nfoo 0\n", + }, + { + "dash_in_metric_name", + "counter_foo-bar", + new(expvar.Int), + "# TYPE foo_bar counter\nfoo_bar 0\n", + }, + { + "int_with_type_counter", + "counter_foo", + new(expvar.Int), + "# TYPE foo counter\nfoo 0\n", + }, + { + "int_with_type_gauge", + "gauge_foo", + new(expvar.Int), + "# TYPE foo gauge\nfoo 0\n", + }, + { + // For a float = 0.0, Prometheus client_golang outputs "0" + "float_zero", + "foo", + new(expvar.Float), + "# TYPE foo gauge\nfoo 0\n", + }, + { + "float_point_5", + "foo", + half, + "# TYPE foo gauge\nfoo 0.5\n", + }, + { + "float_with_type_counter", + "counter_foo", + half, + "# TYPE foo counter\nfoo 0.5\n", + }, + { + "float_with_type_gauge", + "gauge_foo", + half, + "# TYPE foo gauge\nfoo 0.5\n", + }, + { + "metrics_set", + "s", + &metrics.Set{ + Map: *(func() *expvar.Map { + m := new(expvar.Map) + m.Init() + m.Add("foo", 1) + m.Add("bar", 2) + return m + })(), + }, + "# TYPE s_bar counter\ns_bar 2\n# TYPE s_foo counter\ns_foo 1\n", + }, + { + "metrics_set_TODO_gauge_type", + "gauge_s", // TODO(bradfitz): arguably a bug; should pass down type + &metrics.Set{ + Map: *(func() *expvar.Map { + m := new(expvar.Map) + m.Init() + m.Add("foo", 1) + m.Add("bar", 2) + return m + })(), + }, + "# TYPE s_bar counter\ns_bar 2\n# TYPE s_foo counter\ns_foo 1\n", + }, + { + "expvar_map_untyped", + "api_status_code", + func() *expvar.Map { + m := new(expvar.Map) + m.Init() + m.Add("2xx", 100) + m.Add("5xx", 2) + return m + }(), + "api_status_code_2xx 100\napi_status_code_5xx 2\n", + }, + { + "func_float64", + "counter_x", + expvar.Func(func() any { return float64(1.2) }), + "# TYPE x counter\nx 1.2\n", + }, + { + "func_float64_gauge", + "gauge_y", + expvar.Func(func() any { return float64(1.2) }), + "# TYPE y gauge\ny 1.2\n", + }, + { + "func_float64_untyped", + "z", + expvar.Func(func() any { return float64(1.2) }), + "z 1.2\n", + }, + { + "metrics_label_map", + "counter_m", + &metrics.LabelMap{ + Label: "label", + Map: *(func() *expvar.Map { + m := new(expvar.Map) + m.Init() + m.Add("foo", 1) + m.Add("bar", 2) + return m + })(), + }, + "# TYPE m counter\nm{label=\"bar\"} 2\nm{label=\"foo\"} 1\n", + }, + { + "metrics_label_map_untyped", + "control_save_config", + (func() *metrics.LabelMap { + m := &metrics.LabelMap{Label: "reason"} + m.Add("new", 1) + m.Add("updated", 1) + m.Add("fun", 1) + return m + })(), + "control_save_config{reason=\"fun\"} 1\ncontrol_save_config{reason=\"new\"} 1\ncontrol_save_config{reason=\"updated\"} 1\n", + }, + { + "expvar_label_map", + "counter_labelmap_keyname_m", + func() *expvar.Map { + m := new(expvar.Map) + m.Init() + m.Add("foo", 1) + m.Add("bar", 2) + return m + }(), + "# TYPE m counter\nm{keyname=\"bar\"} 2\nm{keyname=\"foo\"} 1\n", + }, + { + "struct_reflect", + "foo", + someExpVarWithJSONAndPromTypes(), + strings.TrimSpace(` +# TYPE foo_AUint16 counter +foo_AUint16 65535 +# TYPE foo_AnInt8 counter +foo_AnInt8 127 +# TYPE foo_curTemp gauge +foo_curTemp 20.6 +# TYPE foo_curX gauge +foo_curX 3 +# TYPE foo_nestptr_bar counter +foo_nestptr_bar 20 +# TYPE foo_nestptr_foo gauge +foo_nestptr_foo 10 +# TYPE foo_nestvalue_bar counter +foo_nestvalue_bar 2 +# TYPE foo_nestvalue_foo gauge +foo_nestvalue_foo 1 +# TYPE foo_totalY counter +foo_totalY 4 +`) + "\n", + }, + { + "struct_reflect_nil_root", + "foo", + expvarAdapter{(*SomeStats)(nil)}, + "", + }, + { + "func_returning_int", + "num_goroutines", + expvar.Func(func() any { return 123 }), + "num_goroutines 123\n", + }, + { + "string_version_var", + "foo_version", + expvar.Func(func() any { return "1.2.3-foo15" }), + "foo_version{version=\"1.2.3-foo15\"} 1\n", + }, + { + "field_ordering", + "foo", + someExpVarWithFieldNamesSorting(), + strings.TrimSpace(` +# TYPE foo_bar_a gauge +foo_bar_a 1 +# TYPE foo_bar_b counter +foo_bar_b 1 +# TYPE foo_foo_a gauge +foo_foo_a 1 +# TYPE foo_foo_b counter +foo_foo_b 1 +`) + "\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tstest.Replace(t, &expvarDo, func(f func(expvar.KeyValue)) { + f(expvar.KeyValue{Key: tt.k, Value: tt.v}) + }) + rec := httptest.NewRecorder() + Handler(rec, httptest.NewRequest("GET", "/", nil)) + if got := rec.Body.Bytes(); string(got) != tt.want { + t.Errorf("mismatch\n got: %q\n%s\nwant: %q\n%s\n", got, got, tt.want, tt.want) + } + }) + } +} + +type SomeNested struct { + FooG int64 `json:"foo" metrictype:"gauge"` + BarC int64 `json:"bar" metrictype:"counter"` + Omit int `json:"-" metrictype:"counter"` +} + +type SomeStats struct { + Nested SomeNested `json:"nestvalue"` + NestedPtr *SomeNested `json:"nestptr"` + NestedNilPtr *SomeNested `json:"nestnilptr"` + CurX int `json:"curX" metrictype:"gauge"` + NoMetricType int `json:"noMetric" metrictype:""` + TotalY int64 `json:"totalY,omitempty" metrictype:"counter"` + CurTemp float64 `json:"curTemp" metrictype:"gauge"` + AnInt8 int8 `metrictype:"counter"` + AUint16 uint16 `metrictype:"counter"` +} + +// someExpVarWithJSONAndPromTypes returns an expvar.Var that +// implements PrometheusMetricsReflectRooter for TestVarzHandler. +func someExpVarWithJSONAndPromTypes() expvar.Var { + st := &SomeStats{ + Nested: SomeNested{ + FooG: 1, + BarC: 2, + Omit: 3, + }, + NestedPtr: &SomeNested{ + FooG: 10, + BarC: 20, + }, + CurX: 3, + TotalY: 4, + CurTemp: 20.6, + AnInt8: 127, + AUint16: 65535, + } + return expvarAdapter{st} +} + +type expvarAdapter struct { + st *SomeStats +} + +func (expvarAdapter) String() string { return "{}" } // expvar JSON; unused in test + +func (a expvarAdapter) PrometheusMetricsReflectRoot() any { + return a.st +} + +// SomeTestOfFieldNamesSorting demonstrates field +// names that are not in sorted in declaration order, to verify +// that we sort based on field name +type SomeTestOfFieldNamesSorting struct { + FooAG int64 `json:"foo_a" metrictype:"gauge"` + BarAG int64 `json:"bar_a" metrictype:"gauge"` + FooBC int64 `json:"foo_b" metrictype:"counter"` + BarBC int64 `json:"bar_b" metrictype:"counter"` +} + +// someExpVarWithFieldNamesSorting returns an expvar.Var that +// implements PrometheusMetricsReflectRooter for TestVarzHandler. +func someExpVarWithFieldNamesSorting() expvar.Var { + st := &SomeTestOfFieldNamesSorting{ + FooAG: 1, + BarAG: 1, + FooBC: 1, + BarBC: 1, + } + return expvarAdapter2{st} +} + +type expvarAdapter2 struct { + st *SomeTestOfFieldNamesSorting +} + +func (expvarAdapter2) String() string { return "{}" } // expvar JSON; unused in test + +func (a expvarAdapter2) PrometheusMetricsReflectRoot() any { + return a.st +} + +func TestSortedStructAllocs(t *testing.T) { + f := reflect.ValueOf(struct { + Foo int + Bar int + Baz int + }{}) + n := testing.AllocsPerRun(1000, func() { + foreachExportedStructField(f, func(fieldOrJSONName, metricType string, rv reflect.Value) { + // Nothing. + }) + }) + if n != 0 { + t.Errorf("allocs = %v; want 0", n) + } +} + +func TestVarzHandlerSorting(t *testing.T) { + tstest.Replace(t, &expvarDo, func(f func(expvar.KeyValue)) { + f(expvar.KeyValue{Key: "counter_zz", Value: new(expvar.Int)}) + f(expvar.KeyValue{Key: "gauge_aa", Value: new(expvar.Int)}) + }) + rec := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/", nil) + Handler(rec, req) + got := rec.Body.Bytes() + const want = "# TYPE aa gauge\naa 0\n# TYPE zz counter\nzz 0\n" + if string(got) != want { + t.Errorf("got %q; want %q", got, want) + } + rec = new(httptest.ResponseRecorder) // without a body + + // Lock in the current number of allocs, to prevent it from growing. + if !version.IsRace() { + allocs := int(testing.AllocsPerRun(1000, func() { + Handler(rec, req) + })) + if max := 13; allocs > max { + t.Errorf("allocs = %v; want max %v", allocs, max) + } + } +}