archive
/
tailscale
mirror of https://github.com/tailscale/tailscale/
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

tsweb: make VarzHandler capable of walking structs with reflect

Browse Source 
To be used by control, per linked bug's plan.

Updates #2635

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>
pull/2643/head
Brad Fitzpatrick 3 years ago committed by Brad Fitzpatrick
parent b9bd7dbc5d
commit 5aba620fb9
2 changed files with 278 additions and 80 deletions
Show Stats Download Patch File Download Diff File

257
tsweb/tsweb.go
Unescape Escape View File

@ -19,6 +19,7 @@ import (
_ "net/http/pprof"
"os"
"path/filepath"
"reflect"
"runtime"
"strings"
"time"
@ -342,109 +343,168 @@ func Error(code int, msg string, err error) HTTPError {
return HTTPError{Code: code, Msg: msg, Err: err}
}
// VarzHandler is an HTTP handler to write expvar values into the
// prometheus export format:
//
// https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md
//
// It makes the following assumptions:
//
// * *expvar.Int are counters (unless marked as a gauge_; see below)
// * a *tailscale/metrics.Set is descended into, joining keys with
// underscores. So use underscores as your metric names.
// * an expvar named starting with "gauge_" or "counter_" is of that
// Prometheus type, and has that prefix stripped.
// * anything else is untyped and thus not exported.
// * expvar.Func can return an int or int64 (for now) and anything else
// is not exported.
// WritePrometheusExpvar writes kv to w in Prometheus metrics format.
//
// This will evolve over time, or perhaps be replaced.
func VarzHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain; version=0.0.4")
// See VarzHandler for conventions. This is exported primarily for
// people to test their varz.
func WritePrometheusExpvar(w io.Writer, kv expvar.KeyValue) {
writePromExpVar(w, "", kv)
}
func writePromExpVar(w io.Writer, prefix string, kv expvar.KeyValue) {
key := kv.Key
var typ string
var label string
switch {
case strings.HasPrefix(kv.Key, "gauge_"):
typ = "gauge"
key = strings.TrimPrefix(kv.Key, "gauge_")
case strings.HasPrefix(kv.Key, "counter_"):
typ = "counter"
key = strings.TrimPrefix(kv.Key, "counter_")
}
if strings.HasPrefix(key, "labelmap_") {
key = strings.TrimPrefix(key, "labelmap_")
if i := strings.Index(key, "_"); i != -1 {
label, key = key[:i], key[i+1:]
}
}
name := prefix + key
var dump func(prefix string, kv expvar.KeyValue)
dump = func(prefix string, kv expvar.KeyValue) {
key := kv.Key
var typ string
var label string
switch {
case strings.HasPrefix(kv.Key, "gauge_"):
typ = "gauge"
key = strings.TrimPrefix(kv.Key, "gauge_")
case strings.HasPrefix(kv.Key, "counter_"):
switch v := kv.Value.(type) {
case *expvar.Int:
if typ == "" {
typ = "counter"
key = strings.TrimPrefix(kv.Key, "counter_")
}
if strings.HasPrefix(key, "labelmap_") {
key = strings.TrimPrefix(key, "labelmap_")
if i := strings.Index(key, "_"); i != -1 {
label, key = key[:i], key[i+1:]
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, v.Value())
return
case *metrics.Set:
v.Do(func(kv expvar.KeyValue) {
writePromExpVar(w, name+"_", kv)
})
return
case PrometheusMetricsReflectRooter:
root := v.PrometheusMetricsReflectRoot()
rv := reflect.ValueOf(root)
if rv.Type().Kind() == reflect.Ptr {
if rv.IsNil() {
return
}
rv = rv.Elem()
}
name := prefix + key
switch v := kv.Value.(type) {
case *expvar.Int:
if typ == "" {
typ = "counter"
}
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, v.Value())
return
case *metrics.Set:
v.Do(func(kv expvar.KeyValue) {
dump(name+"_", kv)
})
if rv.Type().Kind() != reflect.Struct {
fmt.Fprintf(w, "# skipping expvar %q; unknown root type\n", name)
return
}
if typ == "" {
var funcRet string
if f, ok := kv.Value.(expvar.Func); ok {
v := f()
if ms, ok := v.(runtime.MemStats); ok && name == "memstats" {
writeMemstats(w, &ms)
return
foreachExportedStructField(rv, func(fieldOrJSONName, metricType string, rv reflect.Value) {
mname := name + "_" + fieldOrJSONName
switch rv.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Int())
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Uint())
case reflect.Float32, reflect.Float64:
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Float())
case reflect.Struct:
if rv.CanAddr() {
// Slight optimization, not copying big structs if they're addressable:
writePromExpVar(w, name+"_", expvar.KeyValue{Key: fieldOrJSONName, Value: expVarPromStructRoot{rv.Addr().Interface()}})
} else {
writePromExpVar(w, name+"_", expvar.KeyValue{Key: fieldOrJSONName, Value: expVarPromStructRoot{rv.Interface()}})
}
funcRet = fmt.Sprintf(" returning %T", v)
}
return
})
return
}
if typ == "" {
var funcRet string
if f, ok := kv.Value.(expvar.Func); ok {
v := f()
if ms, ok := v.(runtime.MemStats); ok && name == "memstats" {
writeMemstats(w, &ms)
return
}
funcRet = fmt.Sprintf(" returning %T", v)
}
switch kv.Value.(type) {
default:
fmt.Fprintf(w, "# skipping expvar %q (Go type %T%s) with undeclared Prometheus type\n", name, kv.Value, funcRet)
return
case *metrics.LabelMap:
// Permit typeless LabelMap for compatibility
// with old expvar-registered
// metrics.LabelMap.
}
}
switch v := kv.Value.(type) {
case expvar.Func:
val := v()
switch val.(type) {
case float64, int64, int:
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, val)
default:
fmt.Fprintf(w, "# skipping expvar func %q returning unknown type %T\n", name, val)
}
switch v := kv.Value.(type) {
case expvar.Func:
val := v()
switch val.(type) {
case float64, int64, int:
fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, val)
default:
fmt.Fprintf(w, "# skipping expvar func %q returning unknown type %T\n", name, val)
}
case *metrics.LabelMap:
case *metrics.LabelMap:
if typ != "" {
fmt.Fprintf(w, "# TYPE %s %s\n", name, typ)
}
// IntMap uses expvar.Map on the inside, which presorts
// keys. The output ordering is deterministic.
v.Do(func(kv expvar.KeyValue) {
fmt.Fprintf(w, "%s{%s=%q} %v\n", name, v.Label, kv.Key, kv.Value)
})
case *expvar.Map:
if label != "" && typ != "" {
fmt.Fprintf(w, "# TYPE %s %s\n", name, typ)
// IntMap uses expvar.Map on the inside, which presorts
// keys. The output ordering is deterministic.
v.Do(func(kv expvar.KeyValue) {
fmt.Fprintf(w, "%s{%s=%q} %v\n", name, v.Label, kv.Key, kv.Value)
fmt.Fprintf(w, "%s{%s=%q} %v\n", name, label, kv.Key, kv.Value)
})
case *expvar.Map:
if label != "" && typ != "" {
fmt.Fprintf(w, "# TYPE %s %s\n", name, typ)
v.Do(func(kv expvar.KeyValue) {
fmt.Fprintf(w, "%s{%s=%q} %v\n", name, label, kv.Key, kv.Value)
})
} else {
fmt.Fprintf(w, "# skipping expvar.Map %q with incomplete metadata: label %q, Prometheus type %q\n", name, label, typ)
}
} else {
fmt.Fprintf(w, "# skipping expvar.Map %q with incomplete metadata: label %q, Prometheus type %q\n", name, label, typ)
}
}
}
// VarzHandler is an HTTP handler to write expvar values into the
// prometheus export format:
//
// https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md
//
// It makes the following assumptions:
//
// * *expvar.Int are counters (unless marked as a gauge_; see below)
// * a *tailscale/metrics.Set is descended into, joining keys with
// underscores. So use underscores as your metric names.
// * an expvar named starting with "gauge_" or "counter_" is of that
// Prometheus type, and has that prefix stripped.
// * anything else is untyped and thus not exported.
// * expvar.Func can return an int or int64 (for now) and anything else
// is not exported.
//
// This will evolve over time, or perhaps be replaced.
func VarzHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain; version=0.0.4")
expvarDo(func(kv expvar.KeyValue) {
dump("", kv)
writePromExpVar(w, "", kv)
})
}
// PrometheusMetricsReflectRooter is an optional interface that expvar.Var implementations
// can implement to indicate that they should be walked recursively with reflect to find
// sets of fields to export.
type PrometheusMetricsReflectRooter interface {
expvar.Var
// PrometheusMetricsReflectRoot returns the struct or struct pointer to walk.
PrometheusMetricsReflectRoot() interface{}
}
var expvarDo = expvar.Do // pulled out for tests
func writeMemstats(w io.Writer, ms *runtime.MemStats) {
@ -463,3 +523,42 @@ func writeMemstats(w io.Writer, ms *runtime.MemStats) {
c("frees", ms.Frees, "cumulative count of heap objects freed")
c("num_gc", uint64(ms.NumGC), "number of completed GC cycles")
}
func foreachExportedStructField(rv reflect.Value, f func(fieldOrJSONName, metricType string, rv reflect.Value)) {
t := rv.Type()
for i, n := 0, t.NumField(); i < n; i++ {
sf := t.Field(i)
name := sf.Name
if v := sf.Tag.Get("json"); v != "" {
if i := strings.Index(v, ","); i != -1 {
v = v[:i]
}
if v == "-" {
// Skip it, regardless of its metrictype.
continue
}
if v != "" {
name = v
}
}
metricType := sf.Tag.Get("metrictype")
if metricType != "" || sf.Type.Kind() == reflect.Struct {
f(name, metricType, rv.Field(i))
} else if sf.Type.Kind() == reflect.Ptr && sf.Type.Elem().Kind() == reflect.Struct {
fv := rv.Field(i)
if !fv.IsNil() {
f(name, metricType, fv.Elem())
}
}
}
}
type expVarPromStructRoot struct{ v interface{} }
func (r expVarPromStructRoot) PrometheusMetricsReflectRoot() interface{} { return r.v }
func (r expVarPromStructRoot) String() string { panic("unused") }
var (
_ PrometheusMetricsReflectRooter = expVarPromStructRoot{}
_ expvar.Var = expVarPromStructRoot{}
)

101
tsweb/tsweb_test.go
Unescape Escape View File

@ -12,6 +12,7 @@ import (
"net"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
@ -304,6 +305,12 @@ func BenchmarkLog(b *testing.B) {
}
func TestVarzHandler(t *testing.T) {
t.Run("globals_log", func(t *testing.T) {
rec := httptest.NewRecorder()
VarzHandler(rec, httptest.NewRequest("GET", "/", nil))
t.Logf("Got: %s", rec.Body.Bytes())
})
tests := []struct {
name string
k string // key name
@ -389,6 +396,18 @@ func TestVarzHandler(t *testing.T) {
},
"# TYPE m counter\nm{label=\"bar\"} 2\nm{label=\"foo\"} 1\n",
},
{
"metrics_label_map_untyped",
"control_save_config",
(func() *metrics.LabelMap {
m := &metrics.LabelMap{Label: "reason"}
m.Add("new", 1)
m.Add("updated", 1)
m.Add("fun", 1)
return m
})(),
"control_save_config{reason=\"fun\"} 1\ncontrol_save_config{reason=\"new\"} 1\ncontrol_save_config{reason=\"updated\"} 1\n",
},
{
"expvar_label_map",
"counter_labelmap_keyname_m",
@ -411,6 +430,37 @@ func TestVarzHandler(t *testing.T) {
}(),
"# skipping expvar.Map \"lackslabel\" with incomplete metadata: label \"\", Prometheus type \"counter\"\n",
},
{
"struct_reflect",
"foo",
someExpVarWithJSONAndPromTypes(),
strings.TrimSpace(`
# TYPE foo_nestvalue_foo gauge
foo_nestvalue_foo 1
# TYPE foo_nestvalue_bar counter
foo_nestvalue_bar 2
# TYPE foo_nestptr_foo gauge
foo_nestptr_foo 10
# TYPE foo_nestptr_bar counter
foo_nestptr_bar 20
# TYPE foo_curX gauge
foo_curX 3
# TYPE foo_totalY counter
foo_totalY 4
# TYPE foo_curTemp gauge
foo_curTemp 20.6
# TYPE foo_AnInt8 counter
foo_AnInt8 127
# TYPE foo_AUint16 counter
foo_AUint16 65535
`) + "\n",
},
{
"struct_reflect_nil_root",
"foo",
expvarAdapter{(*SomeStats)(nil)},
"",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@ -421,9 +471,58 @@ func TestVarzHandler(t *testing.T) {
rec := httptest.NewRecorder()
VarzHandler(rec, httptest.NewRequest("GET", "/", nil))
if got := rec.Body.Bytes(); string(got) != tt.want {
t.Errorf("mismatch\n got: %q\nwant: %q\n", got, tt.want)
t.Errorf("mismatch\n got: %q\n%s\nwant: %q\n%s\n", got, got, tt.want, tt.want)
}
})
}
}
type SomeNested struct {
FooG int64 `json:"foo" metrictype:"gauge"`
BarC int64 `json:"bar" metrictype:"counter"`
Omit int `json:"-" metrictype:"counter"`
}
type SomeStats struct {
Nested SomeNested `json:"nestvalue"`
NestedPtr *SomeNested `json:"nestptr"`
NestedNilPtr *SomeNested `json:"nestnilptr"`
CurX int `json:"curX" metrictype:"gauge"`
NoMetricType int `json:"noMetric" metrictype:""`
TotalY int64 `json:"totalY,omitempty" metrictype:"counter"`
CurTemp float64 `json:"curTemp" metrictype:"gauge"`
AnInt8 int8 `metrictype:"counter"`
AUint16 uint16 `metrictype:"counter"`
}
// someExpVarWithJSONAndPromTypes returns an expvar.Var that
// implements PrometheusMetricsReflectRooter for TestVarzHandler.
func someExpVarWithJSONAndPromTypes() expvar.Var {
st := &SomeStats{
Nested: SomeNested{
FooG: 1,
BarC: 2,
Omit: 3,
},
NestedPtr: &SomeNested{
FooG: 10,
BarC: 20,
},
CurX: 3,
TotalY: 4,
CurTemp: 20.6,
AnInt8: 127,
AUint16: 65535,
}
return expvarAdapter{st}
}
type expvarAdapter struct {
st *SomeStats
}
func (expvarAdapter) String() string { return "{}" } // expvar JSON; unused in test
func (a expvarAdapter) PrometheusMetricsReflectRoot() interface{} {
return a.st
}

Write Preview
Loading…
Cancel
Save