From 380a3a083469f3950bf9bdd0a03ca8199fdb91dd Mon Sep 17 00:00:00 2001 From: Fran Bull Date: Fri, 12 Jul 2024 07:08:47 -0700 Subject: [PATCH] appc: track metrics for route info storing Track how often we're writing state and how many routes we're writing. Updates #11008 Signed-off-by: Fran Bull --- appc/appconnector.go | 39 +++++++++++++++++++++++++++++++++++++++ appc/appconnector_test.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/appc/appconnector.go b/appc/appconnector.go index 9dab4764c..671ced953 100644 --- a/appc/appconnector.go +++ b/appc/appconnector.go @@ -11,6 +11,7 @@ package appc import ( "context" + "fmt" "net/netip" "slices" "strings" @@ -21,6 +22,7 @@ import ( "golang.org/x/net/dns/dnsmessage" "tailscale.com/types/logger" "tailscale.com/types/views" + "tailscale.com/util/clientmetric" "tailscale.com/util/dnsname" "tailscale.com/util/execqueue" "tailscale.com/util/mak" @@ -78,6 +80,42 @@ type RouteAdvertiser interface { UnadvertiseRoute(...netip.Prefix) error } +var ( + metricStoreRoutesRateBuckets = []int64{1, 2, 3, 4, 5, 10, 100, 1000} + metricStoreRoutesNBuckets = []int64{1, 2, 3, 4, 5, 10, 100, 1000, 10000} + metricStoreRoutesRate []*clientmetric.Metric + metricStoreRoutesN []*clientmetric.Metric +) + +func initMetricStoreRoutes() { + for _, n := range metricStoreRoutesRateBuckets { + metricStoreRoutesRate = append(metricStoreRoutesRate, clientmetric.NewCounter(fmt.Sprintf("appc_store_routes_rate_%d", n))) + } + metricStoreRoutesRate = append(metricStoreRoutesRate, clientmetric.NewCounter("appc_store_routes_rate_over")) + for _, n := range metricStoreRoutesNBuckets { + metricStoreRoutesN = append(metricStoreRoutesN, clientmetric.NewCounter(fmt.Sprintf("appc_store_routes_n_routes_%d", n))) + } + metricStoreRoutesN = append(metricStoreRoutesN, clientmetric.NewCounter("appc_store_routes_n_routes_over")) +} + +func recordMetric(val int64, buckets []int64, metrics []*clientmetric.Metric) { + if len(buckets) < 1 { + return + } + // finds the first bucket where val <=, or len(buckets) if none match + // for bucket values of 1, 10, 100; 0-1 goes to [0], 2-10 goes to [1], 11-100 goes to [2], 101+ goes to [3] + bucket, _ := slices.BinarySearch(buckets, val) + metrics[bucket].Add(1) +} + +func metricStoreRoutes(rate, nRoutes int64) { + if len(metricStoreRoutesRate) == 0 { + initMetricStoreRoutes() + } + recordMetric(rate, metricStoreRoutesRateBuckets, metricStoreRoutesRate) + recordMetric(nRoutes, metricStoreRoutesNBuckets, metricStoreRoutesN) +} + // RouteInfo is a data structure used to persist the in memory state of an AppConnector // so that we can know, even after a restart, which routes came from ACLs and which were // learned from domains. @@ -141,6 +179,7 @@ func NewAppConnector(logf logger.Logf, routeAdvertiser RouteAdvertiser, routeInf } ac.writeRateMinute = newRateLogger(time.Now, time.Minute, func(c int64, s time.Time, l int64) { ac.logf("routeInfo write rate: %d in minute starting at %v (%d routes)", c, s, l) + metricStoreRoutes(c, l) }) ac.writeRateDay = newRateLogger(time.Now, 24*time.Hour, func(c int64, s time.Time, l int64) { ac.logf("routeInfo write rate: %d in 24 hours starting at %v (%d routes)", c, s, l) diff --git a/appc/appconnector_test.go b/appc/appconnector_test.go index cf5efeff5..7dba8cebd 100644 --- a/appc/appconnector_test.go +++ b/appc/appconnector_test.go @@ -15,6 +15,7 @@ import ( "golang.org/x/net/dns/dnsmessage" "tailscale.com/appc/appctest" "tailscale.com/tstest" + "tailscale.com/util/clientmetric" "tailscale.com/util/mak" "tailscale.com/util/must" ) @@ -569,3 +570,35 @@ func TestRateLogger(t *testing.T) { t.Fatalf("wasCalled: got false, want true") } } + +func TestRouteStoreMetrics(t *testing.T) { + metricStoreRoutes(1, 1) + metricStoreRoutes(1, 1) // the 1 buckets value should be 2 + metricStoreRoutes(5, 5) // the 5 buckets value should be 1 + metricStoreRoutes(6, 6) // the 10 buckets value should be 1 + metricStoreRoutes(10001, 10001) // the over buckets value should be 1 + wanted := map[string]int64{ + "appc_store_routes_n_routes_1": 2, + "appc_store_routes_rate_1": 2, + "appc_store_routes_n_routes_5": 1, + "appc_store_routes_rate_5": 1, + "appc_store_routes_n_routes_10": 1, + "appc_store_routes_rate_10": 1, + "appc_store_routes_n_routes_over": 1, + "appc_store_routes_rate_over": 1, + } + for _, x := range clientmetric.Metrics() { + if x.Value() != wanted[x.Name()] { + t.Errorf("%s: want: %d, got: %d", x.Name(), wanted[x.Name()], x.Value()) + } + } +} + +func TestMetricBucketsAreSorted(t *testing.T) { + if !slices.IsSorted(metricStoreRoutesRateBuckets) { + t.Errorf("metricStoreRoutesRateBuckets must be in order") + } + if !slices.IsSorted(metricStoreRoutesNBuckets) { + t.Errorf("metricStoreRoutesNBuckets must be in order") + } +}