cmd/k8s-operator: users can configure firewall mode for kube operator proxies (#9769)

* cmd/k8s-operator: users can configure operator to set firewall mode for proxies

Users can now pass PROXY_FIREWALL_MODE={nftables,auto,iptables} to operator to make it create ingress/egress proxies with that firewall mode

Also makes sure that if an invalid firewall mode gets configured, the operator will not start provisioning proxy resources, but will instead log an error and write an error event to the related Service.

Updates tailscale/tailscale#9310

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/manifests/operator.yaml

@@ -153,6 +153,8 @@ spec:
               value: tag:k8s
             - name: APISERVER_PROXY
               value: "false"
+            - name: PROXY_FIREWALL_MODE
+              value: auto
           volumeMounts:
           - name: oauth
             mountPath: /oauth

cmd/k8s-operator/operator.go

@@ -52,6 +52,7 @@ func main() {
 		image             = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest")
 		priorityClassName = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
 		tags              = defaultEnv("PROXY_TAGS", "tag:k8s")
+		tsFirewallMode    = defaultEnv("PROXY_FIREWALL_MODE", "")
 	)
 
 	var opts []kzap.Opts
@@ -70,7 +71,7 @@ func main() {
 	defer s.Close()
 	restConfig := config.GetConfigOrDie()
 	maybeLaunchAPIServerProxy(zlog, restConfig, s)
-	runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags)
+	runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags, tsFirewallMode)
 }
 
 // initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the
@@ -179,7 +180,7 @@ waitOnline:
 
 // runReconcilers starts the controller-runtime manager and registers the
 // ServiceReconciler. It blocks forever.
-func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string, restConfig *rest.Config, tsClient *tailscale.Client, image, priorityClassName, tags string) {
+func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string, restConfig *rest.Config, tsClient *tailscale.Client, image, priorityClassName, tags, tsFirewallMode string) {
 	var (
 		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
 	)
@@ -216,6 +217,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 		operatorNamespace:      tsNamespace,
 		proxyImage:             image,
 		proxyPriorityClassName: priorityClassName,
+		tsFirewallMode:         tsFirewallMode,
 	}
 	err = builder.
 		ControllerManagedBy(mgr).
@@ -228,6 +230,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 			Client:                mgr.GetClient(),
 			logger:                zlog.Named("service-reconciler"),
 			isDefaultLoadBalancer: isDefaultLoadBalancer,
+			recorder:              eventRecorder,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create controller: %v", err)

cmd/k8s-operator/operator_test.go

@@ -70,7 +70,12 @@ func TestLoadBalancerClass(t *testing.T) {
 
 	expectEqual(t, fc, expectedSecret(fullName))
 	expectEqual(t, fc, expectedHeadlessService(shortName))
-	expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
+	o := stsOpts{
+		name:       shortName,
+		secretName: fullName,
+		hostname:   "default-test",
+	}
+	expectEqual(t, fc, expectedSTS(o))
 
 	// Normally the Tailscale proxy pod would come up here and write its info
 	// into the secret. Simulate that, then verify reconcile again and verify
@@ -202,7 +207,13 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 
 	expectEqual(t, fc, expectedSecret(fullName))
 	expectEqual(t, fc, expectedHeadlessService(shortName))
-	expectEqual(t, fc, expectedEgressSTS(shortName, fullName, tailnetTargetIP, "default-test", ""))
+	o := stsOpts{
+		name:            shortName,
+		secretName:      fullName,
+		tailnetTargetIP: tailnetTargetIP,
+		hostname:        "default-test",
+	}
+	expectEqual(t, fc, expectedSTS(o))
 	want := &corev1.Service{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Service",
@@ -226,7 +237,13 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 	expectEqual(t, fc, want)
 	expectEqual(t, fc, expectedSecret(fullName))
 	expectEqual(t, fc, expectedHeadlessService(shortName))
-	expectEqual(t, fc, expectedEgressSTS(shortName, fullName, tailnetTargetIP, "default-test", ""))
+	o = stsOpts{
+		name:            shortName,
+		secretName:      fullName,
+		tailnetTargetIP: tailnetTargetIP,
+		hostname:        "default-test",
+	}
+	expectEqual(t, fc, expectedSTS(o))
 
 	// Change the tailscale-target-ip annotation which should update the
 	// StatefulSet
@@ -305,7 +322,12 @@ func TestAnnotations(t *testing.T) {
 
 	expectEqual(t, fc, expectedSecret(fullName))
 	expectEqual(t, fc, expectedHeadlessService(shortName))
-	expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
+	o := stsOpts{
+		name:       shortName,
+		secretName: fullName,
+		hostname:   "default-test",
+	}
+	expectEqual(t, fc, expectedSTS(o))
 	want := &corev1.Service{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Service",
@@ -405,7 +427,12 @@ func TestAnnotationIntoLB(t *testing.T) {
 
 	expectEqual(t, fc, expectedSecret(fullName))
 	expectEqual(t, fc, expectedHeadlessService(shortName))
-	expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
+	o := stsOpts{
+		name:       shortName,
+		secretName: fullName,
+		hostname:   "default-test",
+	}
+	expectEqual(t, fc, expectedSTS(o))
 
 	// Normally the Tailscale proxy pod would come up here and write its info
 	// into the secret. Simulate that, since it would have normally happened at
@@ -450,7 +477,12 @@ func TestAnnotationIntoLB(t *testing.T) {
 	expectReconciled(t, sr, "default", "test")
 	// None of the proxy machinery should have changed...
 	expectEqual(t, fc, expectedHeadlessService(shortName))
-	expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
+	o = stsOpts{
+		name:       shortName,
+		secretName: fullName,
+		hostname:   "default-test",
+	}
+	expectEqual(t, fc, expectedSTS(o))
 	// ... but the service should have a LoadBalancer status.
 
 	want = &corev1.Service{
@@ -528,7 +560,12 @@ func TestLBIntoAnnotation(t *testing.T) {
 
 	expectEqual(t, fc, expectedSecret(fullName))
 	expectEqual(t, fc, expectedHeadlessService(shortName))
-	expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
+	o := stsOpts{
+		name:       shortName,
+		secretName: fullName,
+		hostname:   "default-test",
+	}
+	expectEqual(t, fc, expectedSTS(o))
 
 	// Normally the Tailscale proxy pod would come up here and write its info
 	// into the secret. Simulate that, then verify reconcile again and verify
@@ -591,7 +628,12 @@ func TestLBIntoAnnotation(t *testing.T) {
 	expectReconciled(t, sr, "default", "test")
 
 	expectEqual(t, fc, expectedHeadlessService(shortName))
-	expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
+	o = stsOpts{
+		name:       shortName,
+		secretName: fullName,
+		hostname:   "default-test",
+	}
+	expectEqual(t, fc, expectedSTS(o))
 
 	want = &corev1.Service{
 		TypeMeta: metav1.TypeMeta{
@@ -661,7 +703,12 @@ func TestCustomHostname(t *testing.T) {
 
 	expectEqual(t, fc, expectedSecret(fullName))
 	expectEqual(t, fc, expectedHeadlessService(shortName))
-	expectEqual(t, fc, expectedSTS(shortName, fullName, "reindeer-flotilla", ""))
+	o := stsOpts{
+		name:       shortName,
+		secretName: fullName,
+		hostname:   "reindeer-flotilla",
+	}
+	expectEqual(t, fc, expectedSTS(o))
 	want := &corev1.Service{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Service",
@@ -735,7 +782,7 @@ func TestCustomPriorityClassName(t *testing.T) {
 			defaultTags:            []string{"tag:k8s"},
 			operatorNamespace:      "operator-ns",
 			proxyImage:             "tailscale/tailscale",
-			proxyPriorityClassName: "tailscale-critical",
+			proxyPriorityClassName: "custom-priority-class-name",
 		},
 		logger: zl.Sugar(),
 	}
@@ -752,7 +799,7 @@ func TestCustomPriorityClassName(t *testing.T) {
 			UID: types.UID("1234-UID"),
 			Annotations: map[string]string{
 				"tailscale.com/expose":   "true",
-				"tailscale.com/hostname": "custom-priority-class-name",
+				"tailscale.com/hostname": "tailscale-critical",
 			},
 		},
 		Spec: corev1.ServiceSpec{
@@ -764,8 +811,14 @@ func TestCustomPriorityClassName(t *testing.T) {
 	expectReconciled(t, sr, "default", "test")
 
 	fullName, shortName := findGenName(t, fc, "default", "test")
+	o := stsOpts{
+		name:              shortName,
+		secretName:        fullName,
+		hostname:          "tailscale-critical",
+		priorityClassName: "custom-priority-class-name",
+	}
 
-	expectEqual(t, fc, expectedSTS(shortName, fullName, "custom-priority-class-name", "tailscale-critical"))
+	expectEqual(t, fc, expectedSTS(o))
 }
 
 func TestDefaultLoadBalancer(t *testing.T) {
@@ -811,7 +864,63 @@ func TestDefaultLoadBalancer(t *testing.T) {
 
 	expectEqual(t, fc, expectedSecret(fullName))
 	expectEqual(t, fc, expectedHeadlessService(shortName))
-	expectEqual(t, fc, expectedSTS(shortName, fullName, "default-test", ""))
+	o := stsOpts{
+		name:       shortName,
+		secretName: fullName,
+		hostname:   "default-test",
+	}
+	expectEqual(t, fc, expectedSTS(o))
+}
+
+func TestProxyFirewallMode(t *testing.T) {
+	fc := fake.NewFakeClient()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	sr := &ServiceReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+			tsFirewallMode:    "nftables",
+		},
+		logger:                zl.Sugar(),
+		isDefaultLoadBalancer: true,
+	}
+
+	// Create a service that we should manage, and check that the initial round
+	// of objects looks right.
+	mustCreate(t, fc, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			// The apiserver is supposed to set the UID, but the fake client
+			// doesn't. So, set it explicitly because other code later depends
+			// on it being set.
+			UID: types.UID("1234-UID"),
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP: "10.20.30.40",
+			Type:      corev1.ServiceTypeLoadBalancer,
+		},
+	})
+
+	expectReconciled(t, sr, "default", "test")
+
+	fullName, shortName := findGenName(t, fc, "default", "test")
+	o := stsOpts{
+		name:         shortName,
+		secretName:   fullName,
+		hostname:     "default-test",
+		firewallMode: "nftables",
+	}
+	expectEqual(t, fc, expectedSTS(o))
+
 }
 
 func expectedSecret(name string) *corev1.Secret {
@@ -862,83 +971,44 @@ func expectedHeadlessService(name string) *corev1.Service {
 	}
 }
 
-func expectedSTS(stsName, secretName, hostname, priorityClassName string) *appsv1.StatefulSet {
+func expectedSTS(opts stsOpts) *appsv1.StatefulSet {
-	return &appsv1.StatefulSet{
+	containerEnv := []corev1.EnvVar{
-		TypeMeta: metav1.TypeMeta{
+		{Name: "TS_USERSPACE", Value: "false"},
-			Kind:       "StatefulSet",
+		{Name: "TS_AUTH_ONCE", Value: "true"},
-			APIVersion: "apps/v1",
+		{Name: "TS_KUBE_SECRET", Value: opts.secretName},
-		},
+		{Name: "TS_HOSTNAME", Value: opts.hostname},
-		ObjectMeta: metav1.ObjectMeta{
+	}
-			Name:      stsName,
+	annots := map[string]string{
-			Namespace: "operator-ns",
+		"tailscale.com/operator-last-set-hostname": opts.hostname,
-			Labels: map[string]string{
+	}
-				"tailscale.com/managed":              "true",
+	if opts.tailnetTargetIP != "" {
-				"tailscale.com/parent-resource":      "test",
+		annots["tailscale.com/operator-last-set-ts-tailnet-target-ip"] = opts.tailnetTargetIP
-				"tailscale.com/parent-resource-ns":   "default",
+		containerEnv = append(containerEnv, corev1.EnvVar{
-				"tailscale.com/parent-resource-type": "svc",
+			Name:  "TS_TAILNET_TARGET_IP",
-			},
+			Value: opts.tailnetTargetIP,
-		},
+		})
-		Spec: appsv1.StatefulSetSpec{
+	} else {
-			Replicas: ptr.To[int32](1),
+		containerEnv = append(containerEnv, corev1.EnvVar{
-			Selector: &metav1.LabelSelector{
+			Name:  "TS_DEST_IP",
-				MatchLabels: map[string]string{"app": "1234-UID"},
+			Value: "10.20.30.40",
-			},
+		})
-			ServiceName: stsName,
+
-			Template: corev1.PodTemplateSpec{
+		annots["tailscale.com/operator-last-set-cluster-ip"] = "10.20.30.40"
-				ObjectMeta: metav1.ObjectMeta{
+
-					Annotations: map[string]string{
+	}
-						"tailscale.com/operator-last-set-hostname":   hostname,
+	if opts.firewallMode != "" {
-						"tailscale.com/operator-last-set-cluster-ip": "10.20.30.40",
+		containerEnv = append(containerEnv, corev1.EnvVar{
-					},
+			Name:  "TS_DEBUG_FIREWALL_MODE",
-					DeletionGracePeriodSeconds: ptr.To[int64](10),
+			Value: opts.firewallMode,
-					Labels:                     map[string]string{"app": "1234-UID"},
+		})
-				},
-				Spec: corev1.PodSpec{
-					ServiceAccountName: "proxies",
-					PriorityClassName:  priorityClassName,
-					InitContainers: []corev1.Container{
-						{
-							Name:    "sysctler",
-							Image:   "tailscale/tailscale",
-							Command: []string{"/bin/sh"},
-							Args:    []string{"-c", "sysctl -w net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1"},
-							SecurityContext: &corev1.SecurityContext{
-								Privileged: ptr.To(true),
-							},
-						},
-					},
-					Containers: []corev1.Container{
-						{
-							Name:  "tailscale",
-							Image: "tailscale/tailscale",
-							Env: []corev1.EnvVar{
-								{Name: "TS_USERSPACE", Value: "false"},
-								{Name: "TS_AUTH_ONCE", Value: "true"},
-								{Name: "TS_KUBE_SECRET", Value: secretName},
-								{Name: "TS_HOSTNAME", Value: hostname},
-								{Name: "TS_DEST_IP", Value: "10.20.30.40"},
-							},
-							SecurityContext: &corev1.SecurityContext{
-								Capabilities: &corev1.Capabilities{
-									Add: []corev1.Capability{"NET_ADMIN"},
-								},
-							},
-							ImagePullPolicy: "Always",
-						},
-					},
-				},
-			},
-		},
 	}
-}
-func expectedEgressSTS(stsName, secretName, tailnetTargetIP, hostname, priorityClassName string) *appsv1.StatefulSet {
 	return &appsv1.StatefulSet{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "StatefulSet",
 			APIVersion: "apps/v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      stsName,
+			Name:      opts.name,
 			Namespace: "operator-ns",
 			Labels: map[string]string{
 				"tailscale.com/managed":              "true",
@@ -952,19 +1022,16 @@ func expectedEgressSTS(stsName, secretName, tailnetTargetIP, hostname, priorityC
 			Selector: &metav1.LabelSelector{
 				MatchLabels: map[string]string{"app": "1234-UID"},
 			},
-			ServiceName: stsName,
+			ServiceName: opts.name,
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
-					Annotations: map[string]string{
+					Annotations:                annots,
-						"tailscale.com/operator-last-set-hostname":             hostname,
-						"tailscale.com/operator-last-set-ts-tailnet-target-ip": tailnetTargetIP,
-					},
 					DeletionGracePeriodSeconds: ptr.To[int64](10),
 					Labels:                     map[string]string{"app": "1234-UID"},
 				},
 				Spec: corev1.PodSpec{
 					ServiceAccountName: "proxies",
-					PriorityClassName:  priorityClassName,
+					PriorityClassName:  opts.priorityClassName,
 					InitContainers: []corev1.Container{
 						{
 							Name:    "sysctler",
@@ -980,13 +1047,7 @@ func expectedEgressSTS(stsName, secretName, tailnetTargetIP, hostname, priorityC
 						{
 							Name:  "tailscale",
 							Image: "tailscale/tailscale",
-							Env: []corev1.EnvVar{
+							Env:   containerEnv,
-								{Name: "TS_USERSPACE", Value: "false"},
-								{Name: "TS_AUTH_ONCE", Value: "true"},
-								{Name: "TS_KUBE_SECRET", Value: secretName},
-								{Name: "TS_HOSTNAME", Value: hostname},
-								{Name: "TS_TAILNET_TARGET_IP", Value: tailnetTargetIP},
-							},
 							SecurityContext: &corev1.SecurityContext{
 								Capabilities: &corev1.Capabilities{
 									Add: []corev1.Capability{"NET_ADMIN"},
@@ -1126,6 +1187,15 @@ func expectRequeue(t *testing.T, sr *ServiceReconciler, ns, name string) {
 	}
 }
 
+type stsOpts struct {
+	name              string
+	secretName        string
+	hostname          string
+	priorityClassName string
+	firewallMode      string
+	tailnetTargetIP   string
+}
+
 type fakeTSClient struct {
 	sync.Mutex
 	keyRequests []tailscale.KeyCapabilities

cmd/k8s-operator/sts.go

@@ -79,6 +79,14 @@ type tailscaleSTSReconciler struct {
 	operatorNamespace      string
 	proxyImage             string
 	proxyPriorityClassName string
+	tsFirewallMode         string
+}
+
+func (sts tailscaleSTSReconciler) validate() error {
+	if sts.tsFirewallMode != "" && !isValidFirewallMode(sts.tsFirewallMode) {
+		return fmt.Errorf("invalid proxy firewall mode %s, valid modes are iptables, nftables or unset", sts.tsFirewallMode)
+	}
+	return nil
 }
 
 // IsHTTPSEnabledOnTailnet reports whether HTTPS is enabled on the tailnet.
@@ -360,6 +368,13 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			},
 		})
 	}
+	if a.tsFirewallMode != "" {
+		container.Env = append(container.Env, corev1.EnvVar{
+			Name:  "TS_DEBUG_FIREWALL_MODE",
+			Value: a.tsFirewallMode,
+		},
+		)
+	}
 	ss.ObjectMeta = metav1.ObjectMeta{
 		Name:      headlessSvc.Name,
 		Namespace: a.operatorNamespace,
@@ -499,3 +514,7 @@ func nameForService(svc *corev1.Service) (string, error) {
 	}
 	return svc.Namespace + "-" + svc.Name, nil
 }
+
+func isValidFirewallMode(m string) bool {
+	return m == "auto" || m == "nftables" || m == "iptables"
+}

cmd/k8s-operator/svc.go

@@ -17,6 +17,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"tailscale.com/util/clientmetric"
@@ -37,6 +38,8 @@ type ServiceReconciler struct {
 	// managedEgressProxies is a set of all egress proxies that we're currently
 	// managing. This is only used for metrics.
 	managedEgressProxies set.Slice[types.UID]
+
+	recorder record.EventRecorder
 }
 
 var (
@@ -136,6 +139,15 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 // This function adds a finalizer to svc, ensuring that we can handle orderly
 // deprovisioning later.
 func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) error {
+	// run for proxy config related validations here as opposed to running
+	// them earlier. This is to prevent cleanup etc being blocked on a
+	// misconfigured proxy param
+	if err := a.ssr.validate(); err != nil {
+		msg := fmt.Sprintf("unable to provision proxy resources: invalid config: %v", err)
+		a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDCONFIG", msg)
+		a.logger.Error(msg)
+		return nil
+	}
 	hostname, err := nameForService(svc)
 	if err != nil {
 		return err