From 303125d96d7c31e6c5fe945ad0da9ad095a880a7 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Tue, 27 Feb 2024 15:14:09 +0000 Subject: [PATCH] cmd/k8s-operator: configure all proxies with declarative config (#11238) Containerboot container created for operator's ingress and egress proxies are now always configured by passing a configfile to tailscaled (tailscaled --config . It does not run 'tailscale set' or 'tailscale up'. Upgrading existing setups to this version as well as downgrading existing setups at this version works. Updates tailscale/tailscale#10869 Signed-off-by: Irbe Krumina --- cmd/k8s-operator/connector_test.go | 43 +++--- cmd/k8s-operator/deploy/manifests/proxy.yaml | 2 - .../deploy/manifests/userspace-proxy.yaml | 2 - cmd/k8s-operator/ingress_test.go | 28 ++-- cmd/k8s-operator/operator_test.go | 21 +++ cmd/k8s-operator/sts.go | 87 ++++------- cmd/k8s-operator/sts_test.go | 16 +- cmd/k8s-operator/testutils_test.go | 145 ++++++++++-------- 8 files changed, 175 insertions(+), 169 deletions(-) diff --git a/cmd/k8s-operator/connector_test.go b/cmd/k8s-operator/connector_test.go index 291aaec61..7e8e9599f 100644 --- a/cmd/k8s-operator/connector_test.go +++ b/cmd/k8s-operator/connector_test.go @@ -67,14 +67,13 @@ func TestConnector(t *testing.T) { fullName, shortName := findGenName(t, fc, "", "test", "connector") opts := configOpts{ - stsName: shortName, - secretName: fullName, - parentType: "connector", - hostname: "test-connector", - shouldUseDeclarativeConfig: true, - isExitNode: true, - subnetRoutes: "10.40.0.0/14", - confFileHash: "9321660203effb80983eaecc7b5ac5a8c53934926f46e895b9fe295dcfc5a904", + stsName: shortName, + secretName: fullName, + parentType: "connector", + hostname: "test-connector", + isExitNode: true, + subnetRoutes: "10.40.0.0/14", + confFileHash: "9321660203effb80983eaecc7b5ac5a8c53934926f46e895b9fe295dcfc5a904", } expectEqual(t, fc, expectedSecret(t, opts)) expectEqual(t, fc, expectedSTS(t, fc, opts)) @@ -152,13 +151,12 @@ func TestConnector(t *testing.T) { fullName, shortName = findGenName(t, fc, "", "test", "connector") opts = configOpts{ - stsName: shortName, - secretName: fullName, - parentType: "connector", - shouldUseDeclarativeConfig: true, - subnetRoutes: "10.40.0.0/14", - hostname: "test-connector", - confFileHash: "57d922331890c9b1c8c6ae664394cb254334c551d9cd9db14537b5d9da9fb17e", + stsName: shortName, + secretName: fullName, + parentType: "connector", + subnetRoutes: "10.40.0.0/14", + hostname: "test-connector", + confFileHash: "57d922331890c9b1c8c6ae664394cb254334c551d9cd9db14537b5d9da9fb17e", } expectEqual(t, fc, expectedSecret(t, opts)) expectEqual(t, fc, expectedSTS(t, fc, opts)) @@ -239,14 +237,13 @@ func TestConnectorWithProxyClass(t *testing.T) { fullName, shortName := findGenName(t, fc, "", "test", "connector") opts := configOpts{ - stsName: shortName, - secretName: fullName, - parentType: "connector", - hostname: "test-connector", - shouldUseDeclarativeConfig: true, - isExitNode: true, - subnetRoutes: "10.40.0.0/14", - confFileHash: "9321660203effb80983eaecc7b5ac5a8c53934926f46e895b9fe295dcfc5a904", + stsName: shortName, + secretName: fullName, + parentType: "connector", + hostname: "test-connector", + isExitNode: true, + subnetRoutes: "10.40.0.0/14", + confFileHash: "9321660203effb80983eaecc7b5ac5a8c53934926f46e895b9fe295dcfc5a904", } expectEqual(t, fc, expectedSecret(t, opts)) expectEqual(t, fc, expectedSTS(t, fc, opts)) diff --git a/cmd/k8s-operator/deploy/manifests/proxy.yaml b/cmd/k8s-operator/deploy/manifests/proxy.yaml index daa795b3e..50d5eac0a 100644 --- a/cmd/k8s-operator/deploy/manifests/proxy.yaml +++ b/cmd/k8s-operator/deploy/manifests/proxy.yaml @@ -28,8 +28,6 @@ spec: env: - name: TS_USERSPACE value: "false" - - name: TS_AUTH_ONCE - value: "true" - name: POD_IP valueFrom: fieldRef: diff --git a/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml b/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml index fe9fd443e..031636312 100644 --- a/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml +++ b/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml @@ -20,5 +20,3 @@ spec: env: - name: TS_USERSPACE value: "true" - - name: TS_AUTH_ONCE - value: "true" diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go index bdbe98d81..a7cec1a81 100644 --- a/cmd/k8s-operator/ingress_test.go +++ b/cmd/k8s-operator/ingress_test.go @@ -88,11 +88,12 @@ func TestTailscaleIngress(t *testing.T) { fullName, shortName := findGenName(t, fc, "default", "test", "ingress") opts := configOpts{ - stsName: shortName, - secretName: fullName, - namespace: "default", - parentType: "ingress", - hostname: "default-test", + stsName: shortName, + secretName: fullName, + namespace: "default", + parentType: "ingress", + hostname: "default-test", + confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", } serveConfig := &ipn.ServeConfig{ TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, @@ -125,6 +126,9 @@ func TestTailscaleIngress(t *testing.T) { mak.Set(&ing.ObjectMeta.Annotations, AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy, "true") }) opts.shouldEnableForwardingClusterTrafficViaIngress = true + // configfile hash changed at this point in test env only because we + // lost auth key due to how changes are applied in test client. + opts.confFileHash = "fb9006e30ecda75e88c29dcd0ca2dd28a2ae964d001c66e1be3efe159cc3821d" expectReconciled(t, ingR, "default", "test") expectEqual(t, fc, expectedSTS(t, fc, opts)) @@ -219,11 +223,12 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { fullName, shortName := findGenName(t, fc, "default", "test", "ingress") opts := configOpts{ - stsName: shortName, - secretName: fullName, - namespace: "default", - parentType: "ingress", - hostname: "default-test", + stsName: shortName, + secretName: fullName, + namespace: "default", + parentType: "ingress", + hostname: "default-test", + confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", } serveConfig := &ipn.ServeConfig{ TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, @@ -256,6 +261,9 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { }) expectReconciled(t, ingR, "default", "test") opts.proxyClass = pc.Name + // configfile hash changed at this point in test env only because we + // lost auth key due to how changes are applied in test client. + opts.confFileHash = "fb9006e30ecda75e88c29dcd0ca2dd28a2ae964d001c66e1be3efe159cc3821d" expectEqual(t, fc, expectedSTSUserspace(t, fc, opts)) // 4. tailscale.com/proxy-class label is removed from the Ingress, the diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index 90ed243bb..829c72eaa 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -67,6 +67,7 @@ func TestLoadBalancerClass(t *testing.T) { parentType: "svc", hostname: "default-test", clusterTargetIP: "10.20.30.40", + confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", } expectEqual(t, fc, expectedSecret(t, opts)) @@ -208,6 +209,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { parentType: "svc", tailnetTargetFQDN: tailnetTargetFQDN, hostname: "default-test", + confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", } expectEqual(t, fc, expectedSecret(t, o)) @@ -318,6 +320,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { parentType: "svc", tailnetTargetIP: tailnetTargetIP, hostname: "default-test", + confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", } expectEqual(t, fc, expectedSecret(t, o)) @@ -425,6 +428,7 @@ func TestAnnotations(t *testing.T) { parentType: "svc", hostname: "default-test", clusterTargetIP: "10.20.30.40", + confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", } expectEqual(t, fc, expectedSecret(t, o)) @@ -533,6 +537,7 @@ func TestAnnotationIntoLB(t *testing.T) { parentType: "svc", hostname: "default-test", clusterTargetIP: "10.20.30.40", + confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", } expectEqual(t, fc, expectedSecret(t, o)) @@ -581,6 +586,8 @@ func TestAnnotationIntoLB(t *testing.T) { }) expectReconciled(t, sr, "default", "test") // None of the proxy machinery should have changed... + // (although configfile hash will change in test env only because we lose auth key due to out test not syncing secret.StringData -> secret.Data) + o.confFileHash = "fb9006e30ecda75e88c29dcd0ca2dd28a2ae964d001c66e1be3efe159cc3821d" expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(t, fc, o)) // ... but the service should have a LoadBalancer status. @@ -664,6 +671,7 @@ func TestLBIntoAnnotation(t *testing.T) { parentType: "svc", hostname: "default-test", clusterTargetIP: "10.20.30.40", + confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", } expectEqual(t, fc, expectedSecret(t, o)) @@ -730,6 +738,10 @@ func TestLBIntoAnnotation(t *testing.T) { }) expectReconciled(t, sr, "default", "test") + // configfile hash changes on a re-apply in this case in tests only as + // we lose the auth key due to the test apply not syncing + // secret.StringData -> Data. + o.confFileHash = "fb9006e30ecda75e88c29dcd0ca2dd28a2ae964d001c66e1be3efe159cc3821d" expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(t, fc, o)) @@ -805,6 +817,7 @@ func TestCustomHostname(t *testing.T) { parentType: "svc", hostname: "reindeer-flotilla", clusterTargetIP: "10.20.30.40", + confFileHash: "42376226c7d76ed6d6318315dc6c402f7d993bc0b01a5b0e6c8a833106b7509e", } expectEqual(t, fc, expectedSecret(t, o)) @@ -920,6 +933,7 @@ func TestCustomPriorityClassName(t *testing.T) { hostname: "tailscale-critical", priorityClassName: "custom-priority-class-name", clusterTargetIP: "10.20.30.40", + confFileHash: "13cdef0d5f6f0f2406af028710ea1e0f99f65aba4021e4e70ac75a73cf141fd1", } expectEqual(t, fc, expectedSTS(t, fc, o)) @@ -982,6 +996,7 @@ func TestProxyClassForService(t *testing.T) { parentType: "svc", hostname: "default-test", clusterTargetIP: "10.20.30.40", + confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", } expectEqual(t, fc, expectedSecret(t, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) @@ -1008,6 +1023,10 @@ func TestProxyClassForService(t *testing.T) { }}} }) opts.proxyClass = pc.Name + // configfile hash changes on a second apply in test env only because we + // lose auth key due to out test not syncing secret.StringData -> + // secret.Data + opts.confFileHash = "fb9006e30ecda75e88c29dcd0ca2dd28a2ae964d001c66e1be3efe159cc3821d" expectReconciled(t, sr, "default", "test") expectEqual(t, fc, expectedSTS(t, fc, opts)) @@ -1071,6 +1090,7 @@ func TestDefaultLoadBalancer(t *testing.T) { parentType: "svc", hostname: "default-test", clusterTargetIP: "10.20.30.40", + confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", } expectEqual(t, fc, expectedSTS(t, fc, o)) } @@ -1124,6 +1144,7 @@ func TestProxyFirewallMode(t *testing.T) { hostname: "default-test", firewallMode: "nftables", clusterTargetIP: "10.20.30.40", + confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", } expectEqual(t, fc, expectedSTS(t, fc, o)) diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 87f114b63..1c231b1f2 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -86,7 +86,6 @@ const ( // ensure that it does not get removed when a ProxyClass configuration // is applied. podAnnotationLastSetClusterIP = "tailscale.com/operator-last-set-cluster-ip" - podAnnotationLastSetHostname = "tailscale.com/operator-last-set-hostname" podAnnotationLastSetTailnetTargetIP = "tailscale.com/operator-last-set-ts-tailnet-target-ip" podAnnotationLastSetTailnetTargetFQDN = "tailscale.com/operator-last-set-ts-tailnet-target-fqdn" // podAnnotationLastSetConfigFileHash is sha256 hash of the current tailscaled configuration contents. @@ -101,7 +100,7 @@ var ( // tailscaleManagedLabels are label keys that tailscale operator sets on StatefulSets and Pods. tailscaleManagedLabels = []string{LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"} // tailscaleManagedAnnotations are annotation keys that tailscale operator sets on StatefulSets and Pods. - tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetHostname, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN, podAnnotationLastSetConfigFileHash} + tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN, podAnnotationLastSetConfigFileHash} ) type tailscaleSTSConfig struct { @@ -312,9 +311,9 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * authKey, hash string ) if orig == nil { - // Secret doesn't exist yet, create one. Initially it contains - // only the Tailscale authkey, but once Tailscale starts it'll - // also store the daemon state. + // Initially it contains only tailscaled config, but when the + // proxy starts, it will also store there the state, certs and + // ACME account key. sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, stsC.ChildResourceLabels) if err != nil { return "", "", err @@ -337,17 +336,13 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * return "", "", err } } - if !shouldDoTailscaledDeclarativeConfig(stsC) && authKey != "" { - mak.Set(&secret.StringData, "authkey", authKey) - } - if shouldDoTailscaledDeclarativeConfig(stsC) { - confFileBytes, h, err := tailscaledConfig(stsC, authKey, orig) - if err != nil { - return "", "", fmt.Errorf("error creating tailscaled config: %w", err) - } - hash = h - mak.Set(&secret.StringData, tailscaledConfigKey, string(confFileBytes)) + confFileBytes, h, err := tailscaledConfig(stsC, authKey, orig) + if err != nil { + return "", "", fmt.Errorf("error creating tailscaled config: %w", err) } + hash = h + mak.Set(&secret.StringData, tailscaledConfigKey, string(confFileBytes)) + if stsC.ServeConfig != nil { j, err := json.Marshal(stsC.ServeConfig) if err != nil { @@ -477,6 +472,10 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S Name: "TS_KUBE_SECRET", Value: proxySecret, }, + corev1.EnvVar{ + Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", + Value: "/etc/tsconfig/tailscaled", + }, ) if sts.ForwardClusterTrafficViaL7IngressProxy { container.Env = append(container.Env, corev1.EnvVar{ @@ -484,42 +483,25 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S Value: "true", }) } - if !shouldDoTailscaledDeclarativeConfig(sts) { - container.Env = append(container.Env, corev1.EnvVar{ - Name: "TS_HOSTNAME", - Value: sts.Hostname, - }) - // containerboot currently doesn't have a way to re-read the hostname/ip as - // it is passed via an environment variable. So we need to restart the - // container when the value changes. We do this by adding an annotation to - // the pod template that contains the last value we set. - mak.Set(&pod.Annotations, podAnnotationLastSetHostname, sts.Hostname) - } // Configure containeboot to run tailscaled with a configfile read from the state Secret. - if shouldDoTailscaledDeclarativeConfig(sts) { - mak.Set(&ss.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, tsConfigHash) - pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ - Name: "tailscaledconfig", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: proxySecret, - Items: []corev1.KeyToPath{{ - Key: tailscaledConfigKey, - Path: tailscaledConfigKey, - }}, - }, + mak.Set(&ss.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, tsConfigHash) + pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: "tailscaledconfig", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: proxySecret, + Items: []corev1.KeyToPath{{ + Key: tailscaledConfigKey, + Path: tailscaledConfigKey, + }}, }, - }) - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: "tailscaledconfig", - ReadOnly: true, - MountPath: "/etc/tsconfig", - }) - container.Env = append(container.Env, corev1.EnvVar{ - Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", - Value: "/etc/tsconfig/tailscaled", - }) - } + }, + }) + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: "tailscaledconfig", + ReadOnly: true, + MountPath: "/etc/tsconfig", + }) if a.tsFirewallMode != "" { container.Env = append(container.Env, corev1.EnvVar{ @@ -828,10 +810,3 @@ func nameForService(svc *corev1.Service) (string, error) { func isValidFirewallMode(m string) bool { return m == "auto" || m == "nftables" || m == "iptables" } - -// shouldDoTailscaledDeclarativeConfig determines whether the proxy instance -// should be configured to run tailscaled only with a all config opts passed to -// tailscaled. -func shouldDoTailscaledDeclarativeConfig(stsC *tailscaleSTSConfig) bool { - return stsC.Connector != nil -} diff --git a/cmd/k8s-operator/sts_test.go b/cmd/k8s-operator/sts_test.go index b1acdfd91..94c42e8f8 100644 --- a/cmd/k8s-operator/sts_test.go +++ b/cmd/k8s-operator/sts_test.go @@ -247,28 +247,28 @@ func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) { }, { name: "no custom annots specified and none present in current annots, return current annots", - current: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"}, - want: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"}, + current: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4"}, + want: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4"}, managed: tailscaleManagedAnnotations, }, { name: "no custom annots specified, but some present in current annots, return tailscale managed annots only from the current annots", - current: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"}, - want: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"}, + current: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4"}, + want: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4"}, managed: tailscaleManagedAnnotations, }, { name: "custom annots specified, current annots only contain tailscale managed annots, return a union of both", - current: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"}, + current: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4"}, custom: map[string]string{"foo": "bar", "something.io/foo": "bar"}, - want: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"}, + want: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4"}, managed: tailscaleManagedAnnotations, }, { name: "custom annots specified, current annots contain tailscale managed annots and custom annots, some of which are not present in the new custom annots, return a union of managed annots and the desired custom annots", - current: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"}, + current: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4"}, custom: map[string]string{"something.io/foo": "bar"}, - want: map[string]string{"something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"}, + want: map[string]string{"something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4"}, managed: tailscaleManagedAnnotations, }, { diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index 8e4834873..58c150e5d 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -44,7 +44,6 @@ type configOpts struct { clusterTargetIP string subnetRoutes string isExitNode bool - shouldUseDeclarativeConfig bool // tailscaled in proxy should be configured using config file confFileHash string serveConfig *ipn.ServeConfig shouldEnableForwardingClusterTrafficViaIngress bool @@ -58,9 +57,9 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef Image: "tailscale/tailscale", Env: []corev1.EnvVar{ {Name: "TS_USERSPACE", Value: "false"}, - {Name: "TS_AUTH_ONCE", Value: "true"}, {Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, {Name: "TS_KUBE_SECRET", Value: opts.secretName}, + {Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"}, }, SecurityContext: &corev1.SecurityContext{ Capabilities: &corev1.Capabilities{ @@ -77,37 +76,28 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef } annots := make(map[string]string) var volumes []corev1.Volume - if opts.shouldUseDeclarativeConfig { - volumes = []corev1.Volume{ - { - Name: "tailscaledconfig", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: opts.secretName, - Items: []corev1.KeyToPath{ - { - Key: "tailscaled", - Path: "tailscaled", - }, + volumes = []corev1.Volume{ + { + Name: "tailscaledconfig", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: opts.secretName, + Items: []corev1.KeyToPath{ + { + Key: "tailscaled", + Path: "tailscaled", }, }, }, }, - } - tsContainer.VolumeMounts = []corev1.VolumeMount{{ - Name: "tailscaledconfig", - ReadOnly: true, - MountPath: "/etc/tsconfig", - }} - tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{ - Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", - Value: "/etc/tsconfig/tailscaled", - }) - annots["tailscale.com/operator-last-set-config-file-hash"] = opts.confFileHash - } else { - tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{Name: "TS_HOSTNAME", Value: opts.hostname}) - annots["tailscale.com/operator-last-set-hostname"] = opts.hostname + }, } + tsContainer.VolumeMounts = []corev1.VolumeMount{{ + Name: "tailscaledconfig", + ReadOnly: true, + MountPath: "/etc/tsconfig", + }} + annots["tailscale.com/operator-last-set-config-file-hash"] = opts.confFileHash if opts.firewallMode != "" { tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{ Name: "TS_DEBUG_FIREWALL_MODE", @@ -211,22 +201,43 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef } func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet { + t.Helper() tsContainer := corev1.Container{ Name: "tailscale", Image: "tailscale/tailscale", Env: []corev1.EnvVar{ {Name: "TS_USERSPACE", Value: "true"}, - {Name: "TS_AUTH_ONCE", Value: "true"}, {Name: "TS_KUBE_SECRET", Value: opts.secretName}, - {Name: "TS_HOSTNAME", Value: opts.hostname}, + {Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"}, {Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"}, }, ImagePullPolicy: "Always", - VolumeMounts: []corev1.VolumeMount{{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"}}, + VolumeMounts: []corev1.VolumeMount{ + {Name: "tailscaledconfig", ReadOnly: true, MountPath: "/etc/tsconfig"}, + {Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"}, + }, + } + volumes := []corev1.Volume{ + { + Name: "tailscaledconfig", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: opts.secretName, + Items: []corev1.KeyToPath{ + { + Key: "tailscaled", + Path: "tailscaled", + }, + }, + }, + }, + }, + {Name: "serve-config", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, + Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}, + }, } - annots := make(map[string]string) - volumes := []corev1.Volume{{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}}} - annots["tailscale.com/operator-last-set-hostname"] = opts.hostname ss := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ Kind: "StatefulSet", @@ -250,7 +261,6 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps ServiceName: opts.stsName, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Annotations: annots, DeletionGracePeriodSeconds: ptr.To[int64](10), Labels: map[string]string{ "tailscale.com/managed": "true", @@ -259,6 +269,7 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps "tailscale.com/parent-resource-type": opts.parentType, "app": "1234-UID", }, + Annotations: map[string]string{"tailscale.com/operator-last-set-config-file-hash": opts.confFileHash}, }, Spec: corev1.PodSpec{ ServiceAccountName: "proxies", @@ -310,11 +321,6 @@ func expectedHeadlessService(name string, parentType string) *corev1.Service { func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret { t.Helper() - labels := map[string]string{ - "tailscale.com/managed": "true", - "tailscale.com/parent-resource": "test", - "tailscale.com/parent-resource-type": opts.parentType, - } s := &corev1.Secret{ TypeMeta: metav1.TypeMeta{ Kind: "Secret", @@ -332,37 +338,40 @@ func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret { } mak.Set(&s.StringData, "serve-config", string(serveConfigBs)) } - if !opts.shouldUseDeclarativeConfig { - mak.Set(&s.StringData, "authkey", "secret-authkey") - labels["tailscale.com/parent-resource-ns"] = opts.namespace - } else { - conf := &ipn.ConfigVAlpha{ - Version: "alpha0", - AcceptDNS: "false", - Hostname: &opts.hostname, - Locked: "false", - AuthKey: ptr.To("secret-authkey"), + conf := &ipn.ConfigVAlpha{ + Version: "alpha0", + AcceptDNS: "false", + Hostname: &opts.hostname, + Locked: "false", + AuthKey: ptr.To("secret-authkey"), + } + var routes []netip.Prefix + if opts.subnetRoutes != "" || opts.isExitNode { + r := opts.subnetRoutes + if opts.isExitNode { + r = "0.0.0.0/0,::/0," + r } - var routes []netip.Prefix - if opts.subnetRoutes != "" || opts.isExitNode { - r := opts.subnetRoutes - if opts.isExitNode { - r = "0.0.0.0/0,::/0," + r - } - for _, rr := range strings.Split(r, ",") { - prefix, err := netip.ParsePrefix(rr) - if err != nil { - t.Fatal(err) - } - routes = append(routes, prefix) + for _, rr := range strings.Split(r, ",") { + prefix, err := netip.ParsePrefix(rr) + if err != nil { + t.Fatal(err) } + routes = append(routes, prefix) } - conf.AdvertiseRoutes = routes - b, err := json.Marshal(conf) - if err != nil { - t.Fatalf("error marshalling tailscaled config") - } - mak.Set(&s.StringData, "tailscaled", string(b)) + } + conf.AdvertiseRoutes = routes + b, err := json.Marshal(conf) + if err != nil { + t.Fatalf("error marshalling tailscaled config") + } + mak.Set(&s.StringData, "tailscaled", string(b)) + labels := map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource": "test", + "tailscale.com/parent-resource-ns": "default", + "tailscale.com/parent-resource-type": opts.parentType, + } + if opts.parentType == "connector" { labels["tailscale.com/parent-resource-ns"] = "" // Connector is cluster scoped } s.Labels = labels