cmd/k8s-operator: configure all proxies with declarative config (#11238)

Containerboot container created for operator's ingress and egress proxies
are now always configured by passing a configfile to tailscaled
(tailscaled --config <configfile-path>.
It does not run 'tailscale set' or 'tailscale up'.
Upgrading existing setups to this version as well as
downgrading existing setups at this version works.

Updates tailscale/tailscale#10869

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/connector_test.go

@@ -67,14 +67,13 @@ func TestConnector(t *testing.T) {
 	fullName, shortName := findGenName(t, fc, "", "test", "connector")
 
 	opts := configOpts{
-		stsName:                    shortName,
+		stsName:      shortName,
-		secretName:                 fullName,
+		secretName:   fullName,
-		parentType:                 "connector",
+		parentType:   "connector",
-		hostname:                   "test-connector",
+		hostname:     "test-connector",
-		shouldUseDeclarativeConfig: true,
+		isExitNode:   true,
-		isExitNode:                 true,
+		subnetRoutes: "10.40.0.0/14",
-		subnetRoutes:               "10.40.0.0/14",
+		confFileHash: "9321660203effb80983eaecc7b5ac5a8c53934926f46e895b9fe295dcfc5a904",
-		confFileHash:               "9321660203effb80983eaecc7b5ac5a8c53934926f46e895b9fe295dcfc5a904",
 	}
 	expectEqual(t, fc, expectedSecret(t, opts))
 	expectEqual(t, fc, expectedSTS(t, fc, opts))
@@ -152,13 +151,12 @@ func TestConnector(t *testing.T) {
 	fullName, shortName = findGenName(t, fc, "", "test", "connector")
 
 	opts = configOpts{
-		stsName:                    shortName,
+		stsName:      shortName,
-		secretName:                 fullName,
+		secretName:   fullName,
-		parentType:                 "connector",
+		parentType:   "connector",
-		shouldUseDeclarativeConfig: true,
+		subnetRoutes: "10.40.0.0/14",
-		subnetRoutes:               "10.40.0.0/14",
+		hostname:     "test-connector",
-		hostname:                   "test-connector",
+		confFileHash: "57d922331890c9b1c8c6ae664394cb254334c551d9cd9db14537b5d9da9fb17e",
-		confFileHash:               "57d922331890c9b1c8c6ae664394cb254334c551d9cd9db14537b5d9da9fb17e",
 	}
 	expectEqual(t, fc, expectedSecret(t, opts))
 	expectEqual(t, fc, expectedSTS(t, fc, opts))
@@ -239,14 +237,13 @@ func TestConnectorWithProxyClass(t *testing.T) {
 	fullName, shortName := findGenName(t, fc, "", "test", "connector")
 
 	opts := configOpts{
-		stsName:                    shortName,
+		stsName:      shortName,
-		secretName:                 fullName,
+		secretName:   fullName,
-		parentType:                 "connector",
+		parentType:   "connector",
-		hostname:                   "test-connector",
+		hostname:     "test-connector",
-		shouldUseDeclarativeConfig: true,
+		isExitNode:   true,
-		isExitNode:                 true,
+		subnetRoutes: "10.40.0.0/14",
-		subnetRoutes:               "10.40.0.0/14",
+		confFileHash: "9321660203effb80983eaecc7b5ac5a8c53934926f46e895b9fe295dcfc5a904",
-		confFileHash:               "9321660203effb80983eaecc7b5ac5a8c53934926f46e895b9fe295dcfc5a904",
 	}
 	expectEqual(t, fc, expectedSecret(t, opts))
 	expectEqual(t, fc, expectedSTS(t, fc, opts))

cmd/k8s-operator/deploy/manifests/proxy.yaml

@@ -28,8 +28,6 @@ spec:
           env:
             - name: TS_USERSPACE
               value: "false"
-            - name: TS_AUTH_ONCE
-              value: "true"
             - name: POD_IP
               valueFrom:
                 fieldRef:

cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml

@@ -20,5 +20,3 @@ spec:
           env:
             - name: TS_USERSPACE
               value: "true"
-            - name: TS_AUTH_ONCE
-              value: "true"

cmd/k8s-operator/ingress_test.go

@@ -88,11 +88,12 @@ func TestTailscaleIngress(t *testing.T) {
 
 	fullName, shortName := findGenName(t, fc, "default", "test", "ingress")
 	opts := configOpts{
-		stsName:    shortName,
+		stsName:      shortName,
-		secretName: fullName,
+		secretName:   fullName,
-		namespace:  "default",
+		namespace:    "default",
-		parentType: "ingress",
+		parentType:   "ingress",
-		hostname:   "default-test",
+		hostname:     "default-test",
+		confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549",
 	}
 	serveConfig := &ipn.ServeConfig{
 		TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
@@ -125,6 +126,9 @@ func TestTailscaleIngress(t *testing.T) {
 		mak.Set(&ing.ObjectMeta.Annotations, AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy, "true")
 	})
 	opts.shouldEnableForwardingClusterTrafficViaIngress = true
+	// configfile hash changed at this point in test env only because we
+	// lost auth key due to how changes are applied in test client.
+	opts.confFileHash = "fb9006e30ecda75e88c29dcd0ca2dd28a2ae964d001c66e1be3efe159cc3821d"
 	expectReconciled(t, ingR, "default", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, opts))
 
@@ -219,11 +223,12 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 
 	fullName, shortName := findGenName(t, fc, "default", "test", "ingress")
 	opts := configOpts{
-		stsName:    shortName,
+		stsName:      shortName,
-		secretName: fullName,
+		secretName:   fullName,
-		namespace:  "default",
+		namespace:    "default",
-		parentType: "ingress",
+		parentType:   "ingress",
-		hostname:   "default-test",
+		hostname:     "default-test",
+		confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549",
 	}
 	serveConfig := &ipn.ServeConfig{
 		TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
@@ -256,6 +261,9 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 	})
 	expectReconciled(t, ingR, "default", "test")
 	opts.proxyClass = pc.Name
+	// configfile hash changed at this point in test env only because we
+	// lost auth key due to how changes are applied in test client.
+	opts.confFileHash = "fb9006e30ecda75e88c29dcd0ca2dd28a2ae964d001c66e1be3efe159cc3821d"
 	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts))
 
 	// 4. tailscale.com/proxy-class label is removed from the Ingress, the

cmd/k8s-operator/operator_test.go

@@ -67,6 +67,7 @@ func TestLoadBalancerClass(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		confFileHash:    "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549",
 	}
 
 	expectEqual(t, fc, expectedSecret(t, opts))
@@ -208,6 +209,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
 		parentType:        "svc",
 		tailnetTargetFQDN: tailnetTargetFQDN,
 		hostname:          "default-test",
+		confFileHash:      "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549",
 	}
 
 	expectEqual(t, fc, expectedSecret(t, o))
@@ -318,6 +320,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 		parentType:      "svc",
 		tailnetTargetIP: tailnetTargetIP,
 		hostname:        "default-test",
+		confFileHash:    "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549",
 	}
 
 	expectEqual(t, fc, expectedSecret(t, o))
@@ -425,6 +428,7 @@ func TestAnnotations(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		confFileHash:    "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549",
 	}
 
 	expectEqual(t, fc, expectedSecret(t, o))
@@ -533,6 +537,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		confFileHash:    "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549",
 	}
 
 	expectEqual(t, fc, expectedSecret(t, o))
@@ -581,6 +586,8 @@ func TestAnnotationIntoLB(t *testing.T) {
 	})
 	expectReconciled(t, sr, "default", "test")
 	// None of the proxy machinery should have changed...
+	// (although configfile hash will change in test env only because we lose auth key due to out test not syncing secret.StringData -> secret.Data)
+	o.confFileHash = "fb9006e30ecda75e88c29dcd0ca2dd28a2ae964d001c66e1be3efe159cc3821d"
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
 	expectEqual(t, fc, expectedSTS(t, fc, o))
 	// ... but the service should have a LoadBalancer status.
@@ -664,6 +671,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		confFileHash:    "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549",
 	}
 
 	expectEqual(t, fc, expectedSecret(t, o))
@@ -730,6 +738,10 @@ func TestLBIntoAnnotation(t *testing.T) {
 	})
 	expectReconciled(t, sr, "default", "test")
 
+	// configfile hash changes on a re-apply in this case in tests only as
+	// we lose the auth key due to the test apply not syncing
+	// secret.StringData -> Data.
+	o.confFileHash = "fb9006e30ecda75e88c29dcd0ca2dd28a2ae964d001c66e1be3efe159cc3821d"
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
 	expectEqual(t, fc, expectedSTS(t, fc, o))
 
@@ -805,6 +817,7 @@ func TestCustomHostname(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "reindeer-flotilla",
 		clusterTargetIP: "10.20.30.40",
+		confFileHash:    "42376226c7d76ed6d6318315dc6c402f7d993bc0b01a5b0e6c8a833106b7509e",
 	}
 
 	expectEqual(t, fc, expectedSecret(t, o))
@@ -920,6 +933,7 @@ func TestCustomPriorityClassName(t *testing.T) {
 		hostname:          "tailscale-critical",
 		priorityClassName: "custom-priority-class-name",
 		clusterTargetIP:   "10.20.30.40",
+		confFileHash:      "13cdef0d5f6f0f2406af028710ea1e0f99f65aba4021e4e70ac75a73cf141fd1",
 	}
 
 	expectEqual(t, fc, expectedSTS(t, fc, o))
@@ -982,6 +996,7 @@ func TestProxyClassForService(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		confFileHash:    "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549",
 	}
 	expectEqual(t, fc, expectedSecret(t, opts))
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
@@ -1008,6 +1023,10 @@ func TestProxyClassForService(t *testing.T) {
 			}}}
 	})
 	opts.proxyClass = pc.Name
+	// configfile hash changes on a second apply in test env only because we
+	// lose auth key due to out test not syncing secret.StringData ->
+	// secret.Data
+	opts.confFileHash = "fb9006e30ecda75e88c29dcd0ca2dd28a2ae964d001c66e1be3efe159cc3821d"
 	expectReconciled(t, sr, "default", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, opts))
 
@@ -1071,6 +1090,7 @@ func TestDefaultLoadBalancer(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
+		confFileHash:    "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549",
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o))
 }
@@ -1124,6 +1144,7 @@ func TestProxyFirewallMode(t *testing.T) {
 		hostname:        "default-test",
 		firewallMode:    "nftables",
 		clusterTargetIP: "10.20.30.40",
+		confFileHash:    "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549",
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o))
 

cmd/k8s-operator/sts.go

@@ -86,7 +86,6 @@ const (
 	// ensure that it does not get removed when a ProxyClass configuration
 	// is applied.
 	podAnnotationLastSetClusterIP         = "tailscale.com/operator-last-set-cluster-ip"
-	podAnnotationLastSetHostname          = "tailscale.com/operator-last-set-hostname"
 	podAnnotationLastSetTailnetTargetIP   = "tailscale.com/operator-last-set-ts-tailnet-target-ip"
 	podAnnotationLastSetTailnetTargetFQDN = "tailscale.com/operator-last-set-ts-tailnet-target-fqdn"
 	// podAnnotationLastSetConfigFileHash is sha256 hash of the current tailscaled configuration contents.
@@ -101,7 +100,7 @@ var (
 	// tailscaleManagedLabels are label keys that tailscale operator sets on StatefulSets and Pods.
 	tailscaleManagedLabels = []string{LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"}
 	// tailscaleManagedAnnotations are annotation keys that tailscale operator sets on StatefulSets and Pods.
-	tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetHostname, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN, podAnnotationLastSetConfigFileHash}
+	tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN, podAnnotationLastSetConfigFileHash}
 )
 
 type tailscaleSTSConfig struct {
@@ -312,9 +311,9 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 		authKey, hash string
 	)
 	if orig == nil {
-		// Secret doesn't exist yet, create one. Initially it contains
+		// Initially it contains only tailscaled config, but when the
-		// only the Tailscale authkey, but once Tailscale starts it'll
+		// proxy starts, it will also store there the state, certs and
-		// also store the daemon state.
+		// ACME account key.
 		sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, stsC.ChildResourceLabels)
 		if err != nil {
 			return "", "", err
@@ -337,17 +336,13 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 			return "", "", err
 		}
 	}
-	if !shouldDoTailscaledDeclarativeConfig(stsC) && authKey != "" {
+	confFileBytes, h, err := tailscaledConfig(stsC, authKey, orig)
-		mak.Set(&secret.StringData, "authkey", authKey)
+	if err != nil {
-	}
+		return "", "", fmt.Errorf("error creating tailscaled config: %w", err)
-	if shouldDoTailscaledDeclarativeConfig(stsC) {
-		confFileBytes, h, err := tailscaledConfig(stsC, authKey, orig)
-		if err != nil {
-			return "", "", fmt.Errorf("error creating tailscaled config: %w", err)
-		}
-		hash = h
-		mak.Set(&secret.StringData, tailscaledConfigKey, string(confFileBytes))
 	}
+	hash = h
+	mak.Set(&secret.StringData, tailscaledConfigKey, string(confFileBytes))
+
 	if stsC.ServeConfig != nil {
 		j, err := json.Marshal(stsC.ServeConfig)
 		if err != nil {
@@ -477,6 +472,10 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			Name:  "TS_KUBE_SECRET",
 			Value: proxySecret,
 		},
+		corev1.EnvVar{
+			Name:  "EXPERIMENTAL_TS_CONFIGFILE_PATH",
+			Value: "/etc/tsconfig/tailscaled",
+		},
 	)
 	if sts.ForwardClusterTrafficViaL7IngressProxy {
 		container.Env = append(container.Env, corev1.EnvVar{
@@ -484,42 +483,25 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			Value: "true",
 		})
 	}
-	if !shouldDoTailscaledDeclarativeConfig(sts) {
-		container.Env = append(container.Env, corev1.EnvVar{
-			Name:  "TS_HOSTNAME",
-			Value: sts.Hostname,
-		})
-		// containerboot currently doesn't have a way to re-read the hostname/ip as
-		// it is passed via an environment variable. So we need to restart the
-		// container when the value changes. We do this by adding an annotation to
-		// the pod template that contains the last value we set.
-		mak.Set(&pod.Annotations, podAnnotationLastSetHostname, sts.Hostname)
-	}
 	// Configure containeboot to run tailscaled with a configfile read from the state Secret.
-	if shouldDoTailscaledDeclarativeConfig(sts) {
+	mak.Set(&ss.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, tsConfigHash)
-		mak.Set(&ss.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, tsConfigHash)
+	pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
-		pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
+		Name: "tailscaledconfig",
-			Name: "tailscaledconfig",
+		VolumeSource: corev1.VolumeSource{
-			VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
-				Secret: &corev1.SecretVolumeSource{
+				SecretName: proxySecret,
-					SecretName: proxySecret,
+				Items: []corev1.KeyToPath{{
-					Items: []corev1.KeyToPath{{
+					Key:  tailscaledConfigKey,
-						Key:  tailscaledConfigKey,
+					Path: tailscaledConfigKey,
-						Path: tailscaledConfigKey,
+				}},
-					}},
-				},
 			},
-		})
+		},
-		container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
+	})
-			Name:      "tailscaledconfig",
+	container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
-			ReadOnly:  true,
+		Name:      "tailscaledconfig",
-			MountPath: "/etc/tsconfig",
+		ReadOnly:  true,
-		})
+		MountPath: "/etc/tsconfig",
-		container.Env = append(container.Env, corev1.EnvVar{
+	})
-			Name:  "EXPERIMENTAL_TS_CONFIGFILE_PATH",
-			Value: "/etc/tsconfig/tailscaled",
-		})
-	}
 
 	if a.tsFirewallMode != "" {
 		container.Env = append(container.Env, corev1.EnvVar{
@@ -828,10 +810,3 @@ func nameForService(svc *corev1.Service) (string, error) {
 func isValidFirewallMode(m string) bool {
 	return m == "auto" || m == "nftables" || m == "iptables"
 }
-
-// shouldDoTailscaledDeclarativeConfig determines whether the proxy instance
-// should be configured to run tailscaled only with a all config opts passed to
-// tailscaled.
-func shouldDoTailscaledDeclarativeConfig(stsC *tailscaleSTSConfig) bool {
-	return stsC.Connector != nil
-}

cmd/k8s-operator/sts_test.go

@@ -247,28 +247,28 @@ func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) {
 		},
 		{
 			name:    "no custom annots specified and none present in current annots, return current annots",
-			current: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"},
+			current: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4"},
-			want:    map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"},
+			want:    map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4"},
 			managed: tailscaleManagedAnnotations,
 		},
 		{
 			name:    "no custom annots specified, but some present in current annots, return tailscale managed annots only from the current annots",
-			current: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"},
+			current: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4"},
-			want:    map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"},
+			want:    map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4"},
 			managed: tailscaleManagedAnnotations,
 		},
 		{
 			name:    "custom annots specified, current annots only contain tailscale managed annots, return a union of both",
-			current: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"},
+			current: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4"},
 			custom:  map[string]string{"foo": "bar", "something.io/foo": "bar"},
-			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"},
+			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4"},
 			managed: tailscaleManagedAnnotations,
 		},
 		{
 			name:    "custom annots specified, current annots contain tailscale managed annots and custom annots, some of which are not present in the new custom annots, return a union of managed annots and the desired custom annots",
-			current: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"},
+			current: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4"},
 			custom:  map[string]string{"something.io/foo": "bar"},
-			want:    map[string]string{"something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"},
+			want:    map[string]string{"something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4"},
 			managed: tailscaleManagedAnnotations,
 		},
 		{

cmd/k8s-operator/testutils_test.go

@@ -44,7 +44,6 @@ type configOpts struct {
 	clusterTargetIP                                string
 	subnetRoutes                                   string
 	isExitNode                                     bool
-	shouldUseDeclarativeConfig                     bool // tailscaled in proxy should be configured using config file
 	confFileHash                                   string
 	serveConfig                                    *ipn.ServeConfig
 	shouldEnableForwardingClusterTrafficViaIngress bool
@@ -58,9 +57,9 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 		Image: "tailscale/tailscale",
 		Env: []corev1.EnvVar{
 			{Name: "TS_USERSPACE", Value: "false"},
-			{Name: "TS_AUTH_ONCE", Value: "true"},
 			{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
+			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
 		},
 		SecurityContext: &corev1.SecurityContext{
 			Capabilities: &corev1.Capabilities{
@@ -77,37 +76,28 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 	}
 	annots := make(map[string]string)
 	var volumes []corev1.Volume
-	if opts.shouldUseDeclarativeConfig {
+	volumes = []corev1.Volume{
-		volumes = []corev1.Volume{
+		{
-			{
+			Name: "tailscaledconfig",
-				Name: "tailscaledconfig",
+			VolumeSource: corev1.VolumeSource{
-				VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
-					Secret: &corev1.SecretVolumeSource{
+					SecretName: opts.secretName,
-						SecretName: opts.secretName,
+					Items: []corev1.KeyToPath{
-						Items: []corev1.KeyToPath{
+						{
-							{
+							Key:  "tailscaled",
-								Key:  "tailscaled",
+							Path: "tailscaled",
-								Path: "tailscaled",
-							},
 						},
 					},
 				},
 			},
-		}
+		},
-		tsContainer.VolumeMounts = []corev1.VolumeMount{{
-			Name:      "tailscaledconfig",
-			ReadOnly:  true,
-			MountPath: "/etc/tsconfig",
-		}}
-		tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
-			Name:  "EXPERIMENTAL_TS_CONFIGFILE_PATH",
-			Value: "/etc/tsconfig/tailscaled",
-		})
-		annots["tailscale.com/operator-last-set-config-file-hash"] = opts.confFileHash
-	} else {
-		tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{Name: "TS_HOSTNAME", Value: opts.hostname})
-		annots["tailscale.com/operator-last-set-hostname"] = opts.hostname
 	}
+	tsContainer.VolumeMounts = []corev1.VolumeMount{{
+		Name:      "tailscaledconfig",
+		ReadOnly:  true,
+		MountPath: "/etc/tsconfig",
+	}}
+	annots["tailscale.com/operator-last-set-config-file-hash"] = opts.confFileHash
 	if opts.firewallMode != "" {
 		tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
 			Name:  "TS_DEBUG_FIREWALL_MODE",
@@ -211,22 +201,43 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 }
 
 func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
+	t.Helper()
 	tsContainer := corev1.Container{
 		Name:  "tailscale",
 		Image: "tailscale/tailscale",
 		Env: []corev1.EnvVar{
 			{Name: "TS_USERSPACE", Value: "true"},
-			{Name: "TS_AUTH_ONCE", Value: "true"},
 			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
-			{Name: "TS_HOSTNAME", Value: opts.hostname},
+			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
 			{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"},
 		},
 		ImagePullPolicy: "Always",
-		VolumeMounts:    []corev1.VolumeMount{{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"}},
+		VolumeMounts: []corev1.VolumeMount{
+			{Name: "tailscaledconfig", ReadOnly: true, MountPath: "/etc/tsconfig"},
+			{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"},
+		},
+	}
+	volumes := []corev1.Volume{
+		{
+			Name: "tailscaledconfig",
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: opts.secretName,
+					Items: []corev1.KeyToPath{
+						{
+							Key:  "tailscaled",
+							Path: "tailscaled",
+						},
+					},
+				},
+			},
+		},
+		{Name: "serve-config",
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName,
+					Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}},
+		},
 	}
-	annots := make(map[string]string)
-	volumes := []corev1.Volume{{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}}}
-	annots["tailscale.com/operator-last-set-hostname"] = opts.hostname
 	ss := &appsv1.StatefulSet{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "StatefulSet",
@@ -250,7 +261,6 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 			ServiceName: opts.stsName,
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
-					Annotations:                annots,
 					DeletionGracePeriodSeconds: ptr.To[int64](10),
 					Labels: map[string]string{
 						"tailscale.com/managed":              "true",
@@ -259,6 +269,7 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 						"tailscale.com/parent-resource-type": opts.parentType,
 						"app":                                "1234-UID",
 					},
+					Annotations: map[string]string{"tailscale.com/operator-last-set-config-file-hash": opts.confFileHash},
 				},
 				Spec: corev1.PodSpec{
 					ServiceAccountName: "proxies",
@@ -310,11 +321,6 @@ func expectedHeadlessService(name string, parentType string) *corev1.Service {
 
 func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret {
 	t.Helper()
-	labels := map[string]string{
-		"tailscale.com/managed":              "true",
-		"tailscale.com/parent-resource":      "test",
-		"tailscale.com/parent-resource-type": opts.parentType,
-	}
 	s := &corev1.Secret{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Secret",
@@ -332,37 +338,40 @@ func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret {
 		}
 		mak.Set(&s.StringData, "serve-config", string(serveConfigBs))
 	}
-	if !opts.shouldUseDeclarativeConfig {
+	conf := &ipn.ConfigVAlpha{
-		mak.Set(&s.StringData, "authkey", "secret-authkey")
+		Version:   "alpha0",
-		labels["tailscale.com/parent-resource-ns"] = opts.namespace
+		AcceptDNS: "false",
-	} else {
+		Hostname:  &opts.hostname,
-		conf := &ipn.ConfigVAlpha{
+		Locked:    "false",
-			Version:   "alpha0",
+		AuthKey:   ptr.To("secret-authkey"),
-			AcceptDNS: "false",
+	}
-			Hostname:  &opts.hostname,
+	var routes []netip.Prefix
-			Locked:    "false",
+	if opts.subnetRoutes != "" || opts.isExitNode {
-			AuthKey:   ptr.To("secret-authkey"),
+		r := opts.subnetRoutes
+		if opts.isExitNode {
+			r = "0.0.0.0/0,::/0," + r
 		}
-		var routes []netip.Prefix
+		for _, rr := range strings.Split(r, ",") {
-		if opts.subnetRoutes != "" || opts.isExitNode {
+			prefix, err := netip.ParsePrefix(rr)
-			r := opts.subnetRoutes
+			if err != nil {
-			if opts.isExitNode {
+				t.Fatal(err)
-				r = "0.0.0.0/0,::/0," + r
-			}
-			for _, rr := range strings.Split(r, ",") {
-				prefix, err := netip.ParsePrefix(rr)
-				if err != nil {
-					t.Fatal(err)
-				}
-				routes = append(routes, prefix)
 			}
+			routes = append(routes, prefix)
 		}
-		conf.AdvertiseRoutes = routes
+	}
-		b, err := json.Marshal(conf)
+	conf.AdvertiseRoutes = routes
-		if err != nil {
+	b, err := json.Marshal(conf)
-			t.Fatalf("error marshalling tailscaled config")
+	if err != nil {
-		}
+		t.Fatalf("error marshalling tailscaled config")
-		mak.Set(&s.StringData, "tailscaled", string(b))
+	}
+	mak.Set(&s.StringData, "tailscaled", string(b))
+	labels := map[string]string{
+		"tailscale.com/managed":              "true",
+		"tailscale.com/parent-resource":      "test",
+		"tailscale.com/parent-resource-ns":   "default",
+		"tailscale.com/parent-resource-type": opts.parentType,
+	}
+	if opts.parentType == "connector" {
 		labels["tailscale.com/parent-resource-ns"] = "" // Connector is cluster scoped
 	}
 	s.Labels = labels