diff --git a/cmd/k8s-operator/metrics_resources.go b/cmd/k8s-operator/metrics_resources.go index 0579e3466..94cd6a793 100644 --- a/cmd/k8s-operator/metrics_resources.go +++ b/cmd/k8s-operator/metrics_resources.go @@ -93,7 +93,7 @@ func reconcileMetricsResources(ctx context.Context, logger *zap.SugaredLogger, o Spec: corev1.ServiceSpec{ Selector: opts.proxyLabels, Type: corev1.ServiceTypeClusterIP, - Ports: []corev1.ServicePort{{Protocol: "TCP", Port: 9002, Name: "metrics"}}, + Ports: []corev1.ServicePort{{Protocol: "TCP", Port: 9002, Name: opts.metricsPortName}}, }, } var err error @@ -118,7 +118,7 @@ func reconcileMetricsResources(ctx context.Context, logger *zap.SugaredLogger, o } logger.Infof("ensuring ServiceMonitor for metrics Service %s/%s", metricsSvc.Namespace, metricsSvc.Name) - svcMonitor, err := newServiceMonitor(metricsSvc, pc.Spec.Metrics.ServiceMonitor) + svcMonitor, err := newServiceMonitor(metricsSvc, pc.Spec.Metrics.ServiceMonitor, opts.metricsPortName) if err != nil { return fmt.Errorf("error creating ServiceMonitor: %w", err) } @@ -174,7 +174,7 @@ func maybeCleanupServiceMonitor(ctx context.Context, cl client.Client, stsName, // newServiceMonitor takes a metrics Service created for a proxy and constructs and returns a ServiceMonitor for that // proxy that can be applied to the kube API server. // The ServiceMonitor is returned as Unstructured type - this allows us to avoid importing prometheus-operator API server client/schema. -func newServiceMonitor(metricsSvc *corev1.Service, spec *tsapi.ServiceMonitor) (*unstructured.Unstructured, error) { +func newServiceMonitor(metricsSvc *corev1.Service, spec *tsapi.ServiceMonitor, portName string) (*unstructured.Unstructured, error) { sm := serviceMonitorTemplate(metricsSvc.Name, metricsSvc.Namespace) sm.ObjectMeta.Labels = metricsSvc.Labels if spec != nil && len(spec.Labels) > 0 { @@ -185,7 +185,7 @@ func newServiceMonitor(metricsSvc *corev1.Service, spec *tsapi.ServiceMonitor) ( sm.Spec = ServiceMonitorSpec{ Selector: metav1.LabelSelector{MatchLabels: metricsSvc.Labels}, Endpoints: []ServiceMonitorEndpoint{{ - Port: "metrics", + Port: portName, }}, NamespaceSelector: ServiceMonitorNamespaceSelector{ MatchNames: []string{metricsSvc.Namespace}, @@ -274,10 +274,11 @@ func serviceMonitorTemplate(name, ns string) *ServiceMonitor { } type metricsOpts struct { - proxyStsName string // name of StatefulSet for proxy - tsNamespace string // namespace in which Tailscale is installed - proxyLabels map[string]string // labels of the proxy StatefulSet - proxyType string + proxyStsName string // name of StatefulSet for proxy + tsNamespace string // namespace in which Tailscale is installed + proxyLabels map[string]string // labels of the proxy StatefulSet + proxyType string + metricsPortName string // name for the metrics port (defaults to "metrics") } func isNamespacedProxyType(typ string) bool { @@ -294,3 +295,11 @@ func mergeMapKeys(a, b map[string]string) map[string]string { } return m } + +// metricsPortName returns the configured metrics port name from ProxyClass, or defaults to "metrics". +func metricsPortName(pc *tsapi.ProxyClass) string { + if pc != nil && pc.Spec.Metrics != nil && pc.Spec.Metrics.PortName != "" { + return pc.Spec.Metrics.PortName + } + return "metrics" +} diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 946e017a2..f7f33100c 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -405,10 +405,11 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro } mo := &metricsOpts{ - tsNamespace: r.tsNamespace, - proxyStsName: pg.Name, - proxyLabels: pgLabels(pg.Name, nil), - proxyType: "proxygroup", + tsNamespace: r.tsNamespace, + proxyStsName: pg.Name, + proxyLabels: pgLabels(pg.Name, nil), + proxyType: "proxygroup", + metricsPortName: metricsPortName(proxyClass), } if err := reconcileMetricsResources(ctx, logger, mo, proxyClass, r.Client); err != nil { return r.notReadyErrf(pg, logger, "error reconciling metrics resources: %w", err) diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 3e4e72696..060fdce42 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -223,10 +223,11 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga return nil, fmt.Errorf("failed to reconcile statefulset: %w", err) } mo := &metricsOpts{ - proxyStsName: hsvc.Name, - tsNamespace: hsvc.Namespace, - proxyLabels: hsvc.Labels, - proxyType: sts.proxyType, + proxyStsName: hsvc.Name, + tsNamespace: hsvc.Namespace, + proxyLabels: hsvc.Labels, + proxyType: sts.proxyType, + metricsPortName: metricsPortName(sts.ProxyClass), } if err = reconcileMetricsResources(ctx, logger, mo, sts.ProxyClass, a.Client); err != nil { return nil, fmt.Errorf("failed to ensure metrics resources: %w", err) @@ -854,7 +855,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, // For egress proxies, currently all cluster traffic is forwarded to the tailnet target. logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.") } else { - enableEndpoints(ss, metricsEnabled, debugEnabled) + enableEndpoints(ss, metricsEnabled, debugEnabled, metricsPortName(pc)) } } @@ -957,7 +958,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, return ss } -func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) { +func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool, metricsPortName string) { for i, c := range ss.Spec.Template.Spec.Containers { if isMainContainer(&c) { if debug { @@ -1002,7 +1003,7 @@ func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) { ) ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports, corev1.ContainerPort{ - Name: "metrics", + Name: metricsPortName, Protocol: "TCP", ContainerPort: 9002, }, diff --git a/cmd/k8s-operator/sts_test.go b/cmd/k8s-operator/sts_test.go index afe54ed98..bb7089deb 100644 --- a/cmd/k8s-operator/sts_test.go +++ b/cmd/k8s-operator/sts_test.go @@ -310,6 +310,30 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff) } + // 7b. Enable metrics with custom port name. + customPortNamePC := &tsapi.ProxyClass{ + Spec: tsapi.ProxyClassSpec{ + Metrics: &tsapi.Metrics{Enable: true, PortName: "ts-metrics"}, + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{ + TailscaleContainer: &tsapi.Container{ + Debug: &tsapi.Debug{Enable: false}, + }, + }, + }, + }, + } + wantSS = nonUserspaceProxySS.DeepCopy() + wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, + corev1.EnvVar{Name: "TS_LOCAL_ADDR_PORT", Value: "$(POD_IP):9002"}, + corev1.EnvVar{Name: "TS_ENABLE_METRICS", Value: "true"}, + ) + wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "ts-metrics", Protocol: "TCP", ContainerPort: 9002}} + gotSS = applyProxyClassToStatefulSet(customPortNamePC, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) + if diff := cmp.Diff(gotSS, wantSS); diff != "" { + t.Errorf("Unexpected result applying ProxyClass with custom metrics port name to a StatefulSet (-got +want):\n%s", diff) + } + // 8. A Kubernetes API proxy with letsencrypt staging enabled gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy(), &tailscaleSTSConfig{proxyType: string(tsapi.ProxyGroupTypeKubernetesAPIServer)}, zl.Sugar()) verifyEnvVar(t, gotSS, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint) diff --git a/k8s-operator/apis/v1alpha1/types_proxyclass.go b/k8s-operator/apis/v1alpha1/types_proxyclass.go index 670df3b95..c6f45f548 100644 --- a/k8s-operator/apis/v1alpha1/types_proxyclass.go +++ b/k8s-operator/apis/v1alpha1/types_proxyclass.go @@ -329,6 +329,14 @@ type Metrics struct { // // Defaults to false. Enable bool `json:"enable"` + // PortName is the name to use for the metrics port exposed on the Service and + // Pod containers. This allows customization when the default "metrics" name + // conflicts with other ports or monitoring configurations. + // The port name must be a valid DNS label (RFC 1123) and cannot exceed 15 characters. + // +optional + // +kubebuilder:validation:MaxLength=15 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + PortName string `json:"portName,omitempty"` // Enable to create a Prometheus ServiceMonitor for scraping the proxy's Tailscale metrics. // The ServiceMonitor will select the metrics Service that gets created when metrics are enabled. // The ingested metrics for each Service monitor will have labels to identify the proxy: