From bca02252dcae94be7dd9d4bb49c75d09cba2b510 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Fri, 12 Apr 2024 20:18:27 +0100 Subject: [PATCH] cmd/k8s-operator,k8s-operator: optionally serve tailscaled metrics on Pod IP Adds a new .spec.metrics field to ProxyClass to allow users to optionally serve client metrics (tailscaled --debug) on :9001. Metrics cannot currently be enabled for proxies that egress traffic to tailnet and for Ingress proxies with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation (because they currently forward all cluster traffic to their respective backends). The assumption is that users will want to have these metrics enabled continuously to be able to monitor proxy behaviour (as opposed to enabling them temporarily for debugging). Hence we expose them on Pod IP to make it easier to consume them i.e via Prometheus PodMonitor. Updates tailscale/tailscale#11292 Signed-off-by: Irbe Krumina --- .../crds/tailscale.com_proxyclasses.yaml | 11 ++++- .../deploy/examples/proxyclass.yaml | 6 ++- .../deploy/manifests/operator.yaml | 11 ++++- .../deploy/manifests/userspace-proxy.yaml | 4 ++ cmd/k8s-operator/sts.go | 41 +++++++++++++++++-- cmd/k8s-operator/sts_test.go | 28 +++++++++++-- cmd/k8s-operator/testutils_test.go | 14 ++++++- k8s-operator/api.md | 34 +++++++++++++++ .../apis/v1alpha1/types_proxyclass.go | 15 +++++++ .../apis/v1alpha1/zz_generated.deepcopy.go | 20 +++++++++ 10 files changed, 169 insertions(+), 15 deletions(-) diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml index 083b9a422..a067224b8 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml @@ -37,9 +37,16 @@ spec: spec: description: Specification of the desired state of the ProxyClass resource. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status type: object - required: - - statefulSet properties: + metrics: + description: Configuration for proxy metrics. Metrics are currently not supported for egress proxies and for Ingress proxies that have been configured with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation. + type: object + required: + - enable + properties: + enable: + description: Setting enable to true will make the proxy serve Tailscale metrics at :9001/debug/metrics. Defaults to false. + type: boolean statefulSet: description: Configuration parameters for the proxy's StatefulSet. Tailscale Kubernetes operator deploys a StatefulSet for each of the user configured proxies (Tailscale Ingress, Tailscale Service, Connector). type: object diff --git a/cmd/k8s-operator/deploy/examples/proxyclass.yaml b/cmd/k8s-operator/deploy/examples/proxyclass.yaml index 121465bab..3f0d2afa5 100644 --- a/cmd/k8s-operator/deploy/examples/proxyclass.yaml +++ b/cmd/k8s-operator/deploy/examples/proxyclass.yaml @@ -3,13 +3,15 @@ kind: ProxyClass metadata: name: prod spec: + metrics: + enable: true statefulSet: annotations: - platform-component: infra + platform-component: infra pod: labels: team: eng nodeSelector: - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" imagePullSecrets: - name: "foo" diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 63ff36a5b..2c774dd04 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -193,6 +193,15 @@ spec: spec: description: Specification of the desired state of the ProxyClass resource. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: + metrics: + description: Configuration for proxy metrics. Metrics are currently not supported for egress proxies and for Ingress proxies that have been configured with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation. + properties: + enable: + description: Setting enable to true will make the proxy serve Tailscale metrics at :9001/debug/metrics. Defaults to false. + type: boolean + required: + - enable + type: object statefulSet: description: Configuration parameters for the proxy's StatefulSet. Tailscale Kubernetes operator deploys a StatefulSet for each of the user configured proxies (Tailscale Ingress, Tailscale Service, Connector). properties: @@ -637,8 +646,6 @@ spec: type: array type: object type: object - required: - - statefulSet type: object status: description: Status of the ProxyClass. This is set and managed automatically. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status diff --git a/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml b/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml index 031636312..46b49a57b 100644 --- a/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml +++ b/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml @@ -20,3 +20,7 @@ spec: env: - name: TS_USERSPACE value: "true" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 4c800cdbb..06065b969 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -574,7 +574,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName()) if sts.ProxyClass != "" { logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClass) - ss = applyProxyClassToStatefulSet(proxyClass, ss) + ss = applyProxyClassToStatefulSet(proxyClass, ss, sts, logger) } updateSS := func(s *appsv1.StatefulSet) { s.Spec = ss.Spec @@ -605,8 +605,28 @@ func mergeStatefulSetLabelsOrAnnots(current, custom map[string]string, managed [ return custom } -func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet) *appsv1.StatefulSet { - if pc == nil || ss == nil || pc.Spec.StatefulSet == nil { +func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, stsCfg *tailscaleSTSConfig, logger *zap.SugaredLogger) *appsv1.StatefulSet { + if pc == nil || ss == nil { + return ss + } + if pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable { + if stsCfg.TailnetTargetFQDN == "" && stsCfg.TailnetTargetIP == "" && !stsCfg.ForwardClusterTrafficViaL7IngressProxy { + enableMetrics(ss, pc) + } else if stsCfg.ForwardClusterTrafficViaL7IngressProxy { + // TODO (irbekrm): fix this + // For Ingress proxies that have been configured with + // tailscale.com/experimental-forward-cluster-traffic-via-ingress + // annotation, all cluster traffic is forwarded to the + // Ingress backend(s). + logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.") + } else { + // TODO (irbekrm): fix this + // For egress proxies, currently all cluster traffic is forwarded to the tailnet target. + logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.") + } + } + + if pc.Spec.StatefulSet == nil { return ss } @@ -672,6 +692,21 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet) return ss } +func enableMetrics(ss *appsv1.StatefulSet, pc *tsapi.ProxyClass) { + for i, c := range ss.Spec.Template.Spec.Containers { + if c.Name == "tailscale" { + // Serve metrics on on :9001/debug/metrics. If + // we didn't specify Pod IP here, the proxy would, in + // some cases, also listen to its Tailscale IP- we don't + // want folks to start relying on this side-effect as a + // feature. + ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"}) + ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports, corev1.ContainerPort{Name: "metrics", Protocol: "TCP", HostPort: 9001, ContainerPort: 9001}) + break + } + } +} + // tailscaledConfig takes a proxy config, a newly generated auth key if // generated and a Secret with the previous proxy state and auth key and // produces returns tailscaled configuration and a hash of that configuration. diff --git a/cmd/k8s-operator/sts_test.go b/cmd/k8s-operator/sts_test.go index 29c8a24fb..15b188314 100644 --- a/cmd/k8s-operator/sts_test.go +++ b/cmd/k8s-operator/sts_test.go @@ -14,6 +14,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -51,6 +52,10 @@ func Test_statefulSetNameBase(t *testing.T) { } func Test_applyProxyClassToStatefulSet(t *testing.T) { + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } // Setup proxyClassAllOpts := &tsapi.ProxyClass{ Spec: tsapi.ProxyClassSpec{ @@ -104,6 +109,12 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { }, }, } + proxyClassMetrics := &tsapi.ProxyClass{ + Spec: tsapi.ProxyClassSpec{ + Metrics: &tsapi.Metrics{Enable: true}, + }, + } + var userspaceProxySS, nonUserspaceProxySS appsv1.StatefulSet if err := yaml.Unmarshal(userspaceProxyYaml, &userspaceProxySS); err != nil { t.Fatalf("unmarshaling userspace proxy template: %v", err) @@ -147,7 +158,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { wantSS.Spec.Template.Spec.InitContainers[0].Env = append(wantSS.Spec.Template.Spec.InitContainers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...) wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...) - gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy()) + gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { t.Fatalf("Unexpected result applying ProxyClass with all fields set to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff) } @@ -160,7 +171,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations) wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations - gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy()) + gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff) } @@ -180,7 +191,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...) - gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy()) + gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff) } @@ -192,10 +203,19 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations) wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations - gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy()) + gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff) } + + // 5. Test that a ProxyClass with metrics enabled gets correctly applied to a StatefulSet. + wantSS = nonUserspaceProxySS.DeepCopy() + wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"}) + wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "metrics", Protocol: "TCP", ContainerPort: 9001, HostPort: 9001}} + gotSS = applyProxyClassToStatefulSet(proxyClassMetrics, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) + if diff := cmp.Diff(gotSS, wantSS); diff != "" { + t.Fatalf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff) + } } func mergeMapKeys(a, b map[string]string) map[string]string { diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index a4a6a7ffd..7ef32427d 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -53,6 +54,10 @@ type configOpts struct { func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet { t.Helper() + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } tsContainer := corev1.Container{ Name: "tailscale", Image: "tailscale/tailscale", @@ -198,18 +203,23 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil { t.Fatalf("error getting ProxyClass: %v", err) } - return applyProxyClassToStatefulSet(proxyClass, ss) + return applyProxyClassToStatefulSet(proxyClass, ss, new(tailscaleSTSConfig), zl.Sugar()) } return ss } func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet { t.Helper() + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } tsContainer := corev1.Container{ Name: "tailscale", Image: "tailscale/tailscale", Env: []corev1.EnvVar{ {Name: "TS_USERSPACE", Value: "true"}, + {Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, {Name: "TS_KUBE_SECRET", Value: opts.secretName}, {Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"}, {Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"}, @@ -294,7 +304,7 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil { t.Fatalf("error getting ProxyClass: %v", err) } - return applyProxyClassToStatefulSet(proxyClass, ss) + return applyProxyClassToStatefulSet(proxyClass, ss, new(tailscaleSTSConfig), zl.Sugar()) } return ss } diff --git a/k8s-operator/api.md b/k8s-operator/api.md index 97dd3acdc..ed27f1107 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -330,11 +330,45 @@ Specification of the desired state of the ProxyClass resource. https://git.k8s.i + metrics + object + + Configuration for proxy metrics. Metrics are currently not supported for egress proxies and for Ingress proxies that have been configured with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation.
+ + false + statefulSet object Configuration parameters for the proxy's StatefulSet. Tailscale Kubernetes operator deploys a StatefulSet for each of the user configured proxies (Tailscale Ingress, Tailscale Service, Connector).
+ false + + + + +### ProxyClass.spec.metrics +[↩ Parent](#proxyclassspec) + + + +Configuration for proxy metrics. Metrics are currently not supported for egress proxies and for Ingress proxies that have been configured with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation. + + + + + + + + + + + + + +
NameTypeDescriptionRequired
enableboolean + Setting enable to true will make the proxy serve Tailscale metrics at :9001/debug/metrics. Defaults to false.
+ 		true
diff --git a/k8s-operator/apis/v1alpha1/types_proxyclass.go b/k8s-operator/apis/v1alpha1/types_proxyclass.go index ae93555e4..347ab047c 100644 --- a/k8s-operator/apis/v1alpha1/types_proxyclass.go +++ b/k8s-operator/apis/v1alpha1/types_proxyclass.go @@ -52,7 +52,14 @@ type ProxyClassSpec struct { // Configuration parameters for the proxy's StatefulSet. Tailscale // Kubernetes operator deploys a StatefulSet for each of the user // configured proxies (Tailscale Ingress, Tailscale Service, Connector). + // +optional StatefulSet *StatefulSet `json:"statefulSet"` + // Configuration for proxy metrics. Metrics are currently not supported + // for egress proxies and for Ingress proxies that have been configured + // with tailscale.com/experimental-forward-cluster-traffic-via-ingress + // annotation. + // +optional + Metrics *Metrics `json:"metrics,omitempty"` } type StatefulSet struct { @@ -126,6 +133,14 @@ type Pod struct { // https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling // +optional Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + // +optional +} + +type Metrics struct { + // Setting enable to true will make the proxy serve Tailscale metrics + // at :9001/debug/metrics. + // Defaults to false. + Enable bool `json:"enable"` } type Container struct { diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index fd14fe2ae..5d19e4ce1 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -178,6 +178,21 @@ func (in *Env) DeepCopy() *Env { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Metrics) DeepCopyInto(out *Metrics) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Metrics. +func (in *Metrics) DeepCopy() *Metrics { + if in == nil { + return nil + } + out := new(Metrics) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Pod) DeepCopyInto(out *Pod) { *out = *in @@ -308,6 +323,11 @@ func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) { *out = new(StatefulSet) (*in).DeepCopyInto(*out) } + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = new(Metrics) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassSpec.