k8s-operator,kube: add metric for Tailnet resources (#18427)

This commit modifies the k8s reconciler for tailnet resources to keep
a running gauge metric of the number of `Tailnet` resources in use.

Updates: https://github.com/tailscale/corp/issues/34561

Signed-off-by: David Bond <davidsbond93@gmail.com>

cmd/k8s-operator/tailnet.go

@@ -23,7 +23,7 @@ import (
 func clientForTailnet(ctx context.Context, cl client.Client, namespace, name string) (tsClient, error) {
 	var tn tsapi.Tailnet
 	if err := cl.Get(ctx, client.ObjectKey{Name: name}, &tn); err != nil {
-		return nil, fmt.Errorf("failed to get Tailnet %q: %w", name, err)
+		return nil, fmt.Errorf("failed to get tailnet %q: %w", name, err)
 	}
 
 	if !operatorutils.TailnetIsReady(&tn) {

k8s-operator/reconciler/tailnet/tailnet.go

@@ -12,6 +12,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"sync"
 	"time"
 
 	"go.uber.org/zap"
@@ -31,7 +32,10 @@ import (
 	operatorutils "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/k8s-operator/reconciler"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstime"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/set"
 )
 
 type (
@@ -44,6 +48,10 @@ type (
 		clock              tstime.Clock
 		logger             *zap.SugaredLogger
 		clientFunc         func(*tsapi.Tailnet, *corev1.Secret) TailscaleClient
+
+		// Metrics related fields
+		mu       sync.Mutex
+		tailnets set.Slice[types.UID]
 	}
 
 	// The ReconcilerOptions type contains configuration values for the Reconciler.
@@ -94,6 +102,11 @@ func (r *Reconciler) Register(mgr manager.Manager) error {
 		Complete(r)
 }
 
+var (
+	// gaugeTailnetResources tracks the overall number of Tailnet resources currently managed by this operator instance.
+	gaugeTailnetResources = clientmetric.NewGauge(kubetypes.MetricTailnetCount)
+)
+
 // Reconcile is invoked when a change occurs to Tailnet resources within the cluster. On create/update, the Tailnet
 // resource is validated ensuring that the specified Secret exists and contains valid OAuth credentials that have
 // required permissions to perform all necessary functions by the operator.
@@ -120,6 +133,11 @@ func (r *Reconciler) delete(ctx context.Context, tailnet *tsapi.Tailnet) (reconc
 		return reconcile.Result{}, fmt.Errorf("failed to remove finalizer from Tailnet %q: %w", tailnet.Name, err)
 	}
 
+	r.mu.Lock()
+	r.tailnets.Remove(tailnet.UID)
+	r.mu.Unlock()
+	gaugeTailnetResources.Set(int64(r.tailnets.Len()))
+
 	return reconcile.Result{}, nil
 }
 
@@ -131,6 +149,11 @@ const (
 )
 
 func (r *Reconciler) createOrUpdate(ctx context.Context, tailnet *tsapi.Tailnet) (reconcile.Result, error) {
+	r.mu.Lock()
+	r.tailnets.Add(tailnet.UID)
+	r.mu.Unlock()
+	gaugeTailnetResources.Set(int64(r.tailnets.Len()))
+
 	name := types.NamespacedName{Name: tailnet.Spec.Credentials.SecretName, Namespace: r.tailscaleNamespace}
 
 	var secret corev1.Secret
@@ -254,7 +277,7 @@ func (r *Reconciler) ensurePermissions(ctx context.Context, tsClient TailscaleCl
 	}
 
 	if _, err := tsClient.ListVIPServices(ctx); err != nil {
-		errs = errors.Join(errs, fmt.Errorf("failed to list VIP services: %w", err))
+		errs = errors.Join(errs, fmt.Errorf("failed to list tailscale services: %w", err))
 	}
 
 	if errs != nil {

k8s-operator/reconciler/tailnet/tailnet_test.go

@@ -248,7 +248,7 @@ func TestReconciler_Reconcile(t *testing.T) {
 					Type:    string(tsapi.TailnetReady),
 					Status:  metav1.ConditionFalse,
 					Reason:  tailnet.ReasonInvalidOAuth,
-					Message: `failed to list VIP services: EOF`,
+					Message: `failed to list tailscale services: EOF`,
 				},
 			},
 		},

kube/kubetypes/types.go

@@ -33,6 +33,7 @@ const (
 	MetricProxyGroupEgressCount          = "k8s_proxygroup_egress_resources"
 	MetricProxyGroupIngressCount         = "k8s_proxygroup_ingress_resources"
 	MetricProxyGroupAPIServerCount       = "k8s_proxygroup_kube_apiserver_resources"
+	MetricTailnetCount                   = "k8s_tailnet_resources"
 
 	// Keys that containerboot writes to state file that can be used to determine its state.
 	// fields set in Tailscale state Secret. These are mostly used by the Tailscale Kubernetes operator to determine