From 79cc6c14420e4aec123b03e9dff41cb840468bcc Mon Sep 17 00:00:00 2001 From: Ethan <32370007+SadFaceSmith@users.noreply.github.com> Date: Sat, 25 Oct 2025 17:29:24 -0400 Subject: [PATCH 1/3] add port naming --- cmd/k8s-operator/metrics_resources.go | 25 +++++++++++++------ cmd/k8s-operator/sts.go | 9 ++++--- .../apis/v1alpha1/types_proxyclass.go | 8 ++++++ 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/cmd/k8s-operator/metrics_resources.go b/cmd/k8s-operator/metrics_resources.go index 0579e3466..94cd6a793 100644 --- a/cmd/k8s-operator/metrics_resources.go +++ b/cmd/k8s-operator/metrics_resources.go @@ -93,7 +93,7 @@ func reconcileMetricsResources(ctx context.Context, logger *zap.SugaredLogger, o Spec: corev1.ServiceSpec{ Selector: opts.proxyLabels, Type: corev1.ServiceTypeClusterIP, - Ports: []corev1.ServicePort{{Protocol: "TCP", Port: 9002, Name: "metrics"}}, + Ports: []corev1.ServicePort{{Protocol: "TCP", Port: 9002, Name: opts.metricsPortName}}, }, } var err error @@ -118,7 +118,7 @@ func reconcileMetricsResources(ctx context.Context, logger *zap.SugaredLogger, o } logger.Infof("ensuring ServiceMonitor for metrics Service %s/%s", metricsSvc.Namespace, metricsSvc.Name) - svcMonitor, err := newServiceMonitor(metricsSvc, pc.Spec.Metrics.ServiceMonitor) + svcMonitor, err := newServiceMonitor(metricsSvc, pc.Spec.Metrics.ServiceMonitor, opts.metricsPortName) if err != nil { return fmt.Errorf("error creating ServiceMonitor: %w", err) } @@ -174,7 +174,7 @@ func maybeCleanupServiceMonitor(ctx context.Context, cl client.Client, stsName, // newServiceMonitor takes a metrics Service created for a proxy and constructs and returns a ServiceMonitor for that // proxy that can be applied to the kube API server. // The ServiceMonitor is returned as Unstructured type - this allows us to avoid importing prometheus-operator API server client/schema. -func newServiceMonitor(metricsSvc *corev1.Service, spec *tsapi.ServiceMonitor) (*unstructured.Unstructured, error) { +func newServiceMonitor(metricsSvc *corev1.Service, spec *tsapi.ServiceMonitor, portName string) (*unstructured.Unstructured, error) { sm := serviceMonitorTemplate(metricsSvc.Name, metricsSvc.Namespace) sm.ObjectMeta.Labels = metricsSvc.Labels if spec != nil && len(spec.Labels) > 0 { @@ -185,7 +185,7 @@ func newServiceMonitor(metricsSvc *corev1.Service, spec *tsapi.ServiceMonitor) ( sm.Spec = ServiceMonitorSpec{ Selector: metav1.LabelSelector{MatchLabels: metricsSvc.Labels}, Endpoints: []ServiceMonitorEndpoint{{ - Port: "metrics", + Port: portName, }}, NamespaceSelector: ServiceMonitorNamespaceSelector{ MatchNames: []string{metricsSvc.Namespace}, @@ -274,10 +274,11 @@ func serviceMonitorTemplate(name, ns string) *ServiceMonitor { } type metricsOpts struct { - proxyStsName string // name of StatefulSet for proxy - tsNamespace string // namespace in which Tailscale is installed - proxyLabels map[string]string // labels of the proxy StatefulSet - proxyType string + proxyStsName string // name of StatefulSet for proxy + tsNamespace string // namespace in which Tailscale is installed + proxyLabels map[string]string // labels of the proxy StatefulSet + proxyType string + metricsPortName string // name for the metrics port (defaults to "metrics") } func isNamespacedProxyType(typ string) bool { @@ -294,3 +295,11 @@ func mergeMapKeys(a, b map[string]string) map[string]string { } return m } + +// metricsPortName returns the configured metrics port name from ProxyClass, or defaults to "metrics". +func metricsPortName(pc *tsapi.ProxyClass) string { + if pc != nil && pc.Spec.Metrics != nil && pc.Spec.Metrics.PortName != "" { + return pc.Spec.Metrics.PortName + } + return "metrics" +} diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index c52ffce85..21c67ff10 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -222,10 +222,11 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga return nil, fmt.Errorf("failed to reconcile statefulset: %w", err) } mo := &metricsOpts{ - proxyStsName: hsvc.Name, - tsNamespace: hsvc.Namespace, - proxyLabels: hsvc.Labels, - proxyType: sts.proxyType, + proxyStsName: hsvc.Name, + tsNamespace: hsvc.Namespace, + proxyLabels: hsvc.Labels, + proxyType: sts.proxyType, + metricsPortName: metricsPortName(sts.ProxyClass), } if err = reconcileMetricsResources(ctx, logger, mo, sts.ProxyClass, a.Client); err != nil { return nil, fmt.Errorf("failed to ensure metrics resources: %w", err) diff --git a/k8s-operator/apis/v1alpha1/types_proxyclass.go b/k8s-operator/apis/v1alpha1/types_proxyclass.go index 4026f9084..2006a7938 100644 --- a/k8s-operator/apis/v1alpha1/types_proxyclass.go +++ b/k8s-operator/apis/v1alpha1/types_proxyclass.go @@ -329,6 +329,14 @@ type Metrics struct { // // Defaults to false. Enable bool `json:"enable"` + // PortName is the name to use for the metrics port exposed on the Service and + // Pod containers. This allows customization when the default "metrics" name + // conflicts with other ports or monitoring configurations. + // The port name must be a valid DNS label (RFC 1123) and cannot exceed 15 characters. + // +optional + // +kubebuilder:validation:MaxLength=15 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + PortName string `json:"portName,omitempty"` // Enable to create a Prometheus ServiceMonitor for scraping the proxy's Tailscale metrics. // The ServiceMonitor will select the metrics Service that gets created when metrics are enabled. // The ingested metrics for each Service monitor will have labels to identify the proxy: From dfd81634b085f391930347f54fa4d3f741347d26 Mon Sep 17 00:00:00 2001 From: Ethan <32370007+SadFaceSmith@users.noreply.github.com> Date: Sat, 25 Oct 2025 17:33:38 -0400 Subject: [PATCH 2/3] update endpoint and proxygroup --- cmd/k8s-operator/proxygroup.go | 9 +++++---- cmd/k8s-operator/sts.go | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index debeb5c6b..9313280cd 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -405,10 +405,11 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro } mo := &metricsOpts{ - tsNamespace: r.tsNamespace, - proxyStsName: pg.Name, - proxyLabels: pgLabels(pg.Name, nil), - proxyType: "proxygroup", + tsNamespace: r.tsNamespace, + proxyStsName: pg.Name, + proxyLabels: pgLabels(pg.Name, nil), + proxyType: "proxygroup", + metricsPortName: metricsPortName(proxyClass), } if err := reconcileMetricsResources(ctx, logger, mo, proxyClass, r.Client); err != nil { return r.notReadyErrf(pg, logger, "error reconciling metrics resources: %w", err) diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 21c67ff10..3e8ae47de 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -854,7 +854,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, // For egress proxies, currently all cluster traffic is forwarded to the tailnet target. logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.") } else { - enableEndpoints(ss, metricsEnabled, debugEnabled) + enableEndpoints(ss, metricsEnabled, debugEnabled, metricsPortName(pc)) } } @@ -957,7 +957,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, return ss } -func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) { +func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool, metricsPortName string) { for i, c := range ss.Spec.Template.Spec.Containers { if isMainContainer(&c) { if debug { @@ -1002,7 +1002,7 @@ func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) { ) ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports, corev1.ContainerPort{ - Name: "metrics", + Name: metricsPortName, Protocol: "TCP", ContainerPort: 9002, }, From b3819025960e20558ee4f8a576994c30408b13a8 Mon Sep 17 00:00:00 2001 From: Ethan <32370007+SadFaceSmith@users.noreply.github.com> Date: Sat, 25 Oct 2025 17:37:55 -0400 Subject: [PATCH 3/3] test cases --- cmd/k8s-operator/sts_test.go | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/cmd/k8s-operator/sts_test.go b/cmd/k8s-operator/sts_test.go index ea28e77a1..bb7089deb 100644 --- a/cmd/k8s-operator/sts_test.go +++ b/cmd/k8s-operator/sts_test.go @@ -71,11 +71,11 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { SecurityContext: &corev1.PodSecurityContext{ RunAsUser: ptr.To(int64(0)), }, - ImagePullSecrets: []corev1.LocalObjectReference{{Name: "docker-creds"}}, - NodeName: "some-node", - NodeSelector: map[string]string{"beta.kubernetes.io/os": "linux"}, - Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{}}}, - Tolerations: []corev1.Toleration{{Key: "", Operator: "Exists"}}, + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "docker-creds"}}, + NodeName: "some-node", + NodeSelector: map[string]string{"beta.kubernetes.io/os": "linux"}, + Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{}}}, + Tolerations: []corev1.Toleration{{Key: "", Operator: "Exists"}}, PriorityClassName: "high-priority", TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ { @@ -310,6 +310,30 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff) } + // 7b. Enable metrics with custom port name. + customPortNamePC := &tsapi.ProxyClass{ + Spec: tsapi.ProxyClassSpec{ + Metrics: &tsapi.Metrics{Enable: true, PortName: "ts-metrics"}, + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{ + TailscaleContainer: &tsapi.Container{ + Debug: &tsapi.Debug{Enable: false}, + }, + }, + }, + }, + } + wantSS = nonUserspaceProxySS.DeepCopy() + wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, + corev1.EnvVar{Name: "TS_LOCAL_ADDR_PORT", Value: "$(POD_IP):9002"}, + corev1.EnvVar{Name: "TS_ENABLE_METRICS", Value: "true"}, + ) + wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "ts-metrics", Protocol: "TCP", ContainerPort: 9002}} + gotSS = applyProxyClassToStatefulSet(customPortNamePC, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) + if diff := cmp.Diff(gotSS, wantSS); diff != "" { + t.Errorf("Unexpected result applying ProxyClass with custom metrics port name to a StatefulSet (-got +want):\n%s", diff) + } + // 8. A Kubernetes API proxy with letsencrypt staging enabled gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy(), &tailscaleSTSConfig{proxyType: string(tsapi.ProxyGroupTypeKubernetesAPIServer)}, zl.Sugar()) verifyEnvVar(t, gotSS, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint)