cmd/k8s-operator: Allow specifying cluster ips for nameservers (#16477)

This commit modifies the kubernetes operator's `DNSConfig` resource
with the addition of a new field at `nameserver.service.clusterIP`.

This field allows users to specify a static in-cluster IP address of
the nameserver when deployed.

Fixes #14305

Signed-off-by: David Bond <davidsbond93@gmail.com>

cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml

@@ -101,6 +101,13 @@ spec:
                         tag:
                           description: Tag defaults to unstable.
                           type: string
+                    service:
+                      description: Service configuration.
+                      type: object
+                      properties:
+                        clusterIP:
+                          description: ClusterIP sets the static IP of the service used by the nameserver.
+                          type: string
             status:
               description: |-
                 Status describes the status of the DNSConfig. This is set
@@ -172,7 +179,7 @@ spec:
                     ip:
                       description: |-
                         IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.
-                        Currently you must manually update your cluster DNS config to add
+                        Currently, you must manually update your cluster DNS config to add
                         this address as a stub nameserver for ts.net for cluster workloads to be
                         able to resolve MagicDNS names associated with egress or Ingress
                         proxies.

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -389,6 +389,13 @@ spec:
                                                 description: Tag defaults to unstable.
                                                 type: string
                                         type: object
+                                    service:
+                                        description: Service configuration.
+                                        properties:
+                                            clusterIP:
+                                                description: ClusterIP sets the static IP of the service used by the nameserver.
+                                                type: string
+                                        type: object
                                 type: object
                         required:
                             - nameserver
@@ -462,7 +469,7 @@ spec:
                                     ip:
                                         description: |-
                                             IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.
-                                            Currently you must manually update your cluster DNS config to add
+                                            Currently, you must manually update your cluster DNS config to add
                                             this address as a stub nameserver for ts.net for cluster workloads to be
                                             able to resolve MagicDNS names associated with egress or Ingress
                                             proxies.

cmd/k8s-operator/nameserver.go

@@ -7,14 +7,13 @@ package main
 
 import (
 	"context"
+	_ "embed"
 	"errors"
 	"fmt"
 	"slices"
 	"strings"
 	"sync"
 
-	_ "embed"
-
 	"go.uber.org/zap"
 	xslices "golang.org/x/exp/slices"
 	appsv1 "k8s.io/api/apps/v1"
@@ -183,6 +182,10 @@ func (a *NameserverReconciler) maybeProvision(ctx context.Context, tsDNSCfg *tsa
 	if tsDNSCfg.Spec.Nameserver.Image != nil && tsDNSCfg.Spec.Nameserver.Image.Tag != "" {
 		dCfg.imageTag = tsDNSCfg.Spec.Nameserver.Image.Tag
 	}
+	if tsDNSCfg.Spec.Nameserver.Service != nil {
+		dCfg.clusterIP = tsDNSCfg.Spec.Nameserver.Service.ClusterIP
+	}
+
 	for _, deployable := range []deployable{saDeployable, deployDeployable, svcDeployable, cmDeployable} {
 		if err := deployable.updateObj(ctx, dCfg, a.Client); err != nil {
 			return fmt.Errorf("error reconciling %s: %w", deployable.kind, err)
@@ -213,6 +216,7 @@ type deployConfig struct {
 	labels    map[string]string
 	ownerRefs []metav1.OwnerReference
 	namespace string
+	clusterIP string
 }
 
 var (
@@ -267,6 +271,7 @@ var (
 			svc.ObjectMeta.Labels = cfg.labels
 			svc.ObjectMeta.OwnerReferences = cfg.ownerRefs
 			svc.ObjectMeta.Namespace = cfg.namespace
+			svc.Spec.ClusterIP = cfg.clusterIP
 			_, err := createOrUpdate[corev1.Service](ctx, kubeClient, cfg.namespace, svc, func(*corev1.Service) {})
 			return err
 		},

cmd/k8s-operator/nameserver_test.go

@@ -26,7 +26,7 @@ import (
 )
 
 func TestNameserverReconciler(t *testing.T) {
-	dnsCfg := &tsapi.DNSConfig{
+	dnsConfig := &tsapi.DNSConfig{
 		TypeMeta: metav1.TypeMeta{Kind: "DNSConfig", APIVersion: "tailscale.com/v1alpha1"},
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "test",
@@ -37,67 +37,94 @@ func TestNameserverReconciler(t *testing.T) {
 					Repo: "test",
 					Tag:  "v0.0.1",
 				},
+				Service: &tsapi.NameserverService{
+					ClusterIP: "5.4.3.2",
+				},
 			},
 		},
 	}
 
 	fc := fake.NewClientBuilder().
 		WithScheme(tsapi.GlobalScheme).
-		WithObjects(dnsCfg).
-		WithStatusSubresource(dnsCfg).
+		WithObjects(dnsConfig).
+		WithStatusSubresource(dnsConfig).
 		Build()
-	zl, err := zap.NewDevelopment()
+
+	logger, err := zap.NewDevelopment()
 	if err != nil {
 		t.Fatal(err)
 	}
-	cl := tstest.NewClock(tstest.ClockOpts{})
-	nr := &NameserverReconciler{
+
+	clock := tstest.NewClock(tstest.ClockOpts{})
+	reconciler := &NameserverReconciler{
 		Client:      fc,
-		clock:       cl,
-		logger:      zl.Sugar(),
-		tsNamespace: "tailscale",
+		clock:       clock,
+		logger:      logger.Sugar(),
+		tsNamespace: tsNamespace,
 	}
-	expectReconciled(t, nr, "", "test")
-	// Verify that nameserver Deployment has been created and has the expected fields.
-	wantsDeploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: "tailscale"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: appsv1.SchemeGroupVersion.Identifier()}}
-	if err := yaml.Unmarshal(deployYaml, wantsDeploy); err != nil {
+	expectReconciled(t, reconciler, "", "test")
+
+	ownerReference := metav1.NewControllerRef(dnsConfig, tsapi.SchemeGroupVersion.WithKind("DNSConfig"))
+	nameserverLabels := nameserverResourceLabels(dnsConfig.Name, tsNamespace)
+
+	wantsDeploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: tsNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: appsv1.SchemeGroupVersion.Identifier()}}
+	t.Run("deployment has expected fields", func(t *testing.T) {
+		if err = yaml.Unmarshal(deployYaml, wantsDeploy); err != nil {
 			t.Fatalf("unmarshalling yaml: %v", err)
 		}
-	dnsCfgOwnerRef := metav1.NewControllerRef(dnsCfg, tsapi.SchemeGroupVersion.WithKind("DNSConfig"))
-	wantsDeploy.OwnerReferences = []metav1.OwnerReference{*dnsCfgOwnerRef}
+		wantsDeploy.OwnerReferences = []metav1.OwnerReference{*ownerReference}
 		wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.1"
-	wantsDeploy.Namespace = "tailscale"
-	labels := nameserverResourceLabels("test", "tailscale")
-	wantsDeploy.ObjectMeta.Labels = labels
+		wantsDeploy.Namespace = tsNamespace
+		wantsDeploy.ObjectMeta.Labels = nameserverLabels
 		expectEqual(t, fc, wantsDeploy)
+	})
 
+	wantsSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: tsNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: corev1.SchemeGroupVersion.Identifier()}}
+	t.Run("service has expected fields", func(t *testing.T) {
+		if err = yaml.Unmarshal(svcYaml, wantsSvc); err != nil {
+			t.Fatalf("unmarshalling yaml: %v", err)
+		}
+		wantsSvc.Spec.ClusterIP = dnsConfig.Spec.Nameserver.Service.ClusterIP
+		wantsSvc.OwnerReferences = []metav1.OwnerReference{*ownerReference}
+		wantsSvc.Namespace = tsNamespace
+		wantsSvc.ObjectMeta.Labels = nameserverLabels
+		expectEqual(t, fc, wantsSvc)
+	})
+
+	t.Run("dns config status is set", func(t *testing.T) {
 		// Verify that DNSConfig advertizes the nameserver's Service IP address,
 		// has the ready status condition and tailscale finalizer.
 		mustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) {
 			svc.Spec.ClusterIP = "1.2.3.4"
 		})
-	expectReconciled(t, nr, "", "test")
-	dnsCfg.Status.Nameserver = &tsapi.NameserverStatus{
+		expectReconciled(t, reconciler, "", "test")
+
+		dnsConfig.Finalizers = []string{FinalizerName}
+		dnsConfig.Status.Nameserver = &tsapi.NameserverStatus{
 			IP: "1.2.3.4",
 		}
-	dnsCfg.Finalizers = []string{FinalizerName}
-	dnsCfg.Status.Conditions = append(dnsCfg.Status.Conditions, metav1.Condition{
+		dnsConfig.Status.Conditions = append(dnsConfig.Status.Conditions, metav1.Condition{
 			Type:               string(tsapi.NameserverReady),
 			Status:             metav1.ConditionTrue,
 			Reason:             reasonNameserverCreated,
 			Message:            reasonNameserverCreated,
-		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
+			LastTransitionTime: metav1.Time{Time: clock.Now().Truncate(time.Second)},
 		})
-	expectEqual(t, fc, dnsCfg)
 
-	// // Verify that nameserver image gets updated to match DNSConfig spec.
+		expectEqual(t, fc, dnsConfig)
+	})
+
+	t.Run("nameserver image can be updated", func(t *testing.T) {
+		// Verify that nameserver image gets updated to match DNSConfig spec.
 		mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) {
 			dnsCfg.Spec.Nameserver.Image.Tag = "v0.0.2"
 		})
-	expectReconciled(t, nr, "", "test")
+		expectReconciled(t, reconciler, "", "test")
 		wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2"
 		expectEqual(t, fc, wantsDeploy)
+	})
 
+	t.Run("reconciler does not overwrite custom configuration", func(t *testing.T) {
 		// Verify that when another actor sets ConfigMap data, it does not get
 		// overwritten by nameserver reconciler.
 		dnsRecords := &operatorutils.Records{Version: "v1alpha1", IP4: map[string][]string{"foo.ts.net": {"1.2.3.4"}}}
@@ -105,23 +132,35 @@ func TestNameserverReconciler(t *testing.T) {
 		if err != nil {
 			t.Fatalf("error marshalling ConfigMap contents: %v", err)
 		}
+
 		mustUpdate(t, fc, "tailscale", "dnsrecords", func(cm *corev1.ConfigMap) {
 			mak.Set(&cm.Data, "records.json", string(bs))
 		})
-	expectReconciled(t, nr, "", "test")
-	wantCm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dnsrecords",
-		Namespace: "tailscale", Labels: labels, OwnerReferences: []metav1.OwnerReference{*dnsCfgOwnerRef}},
+
+		expectReconciled(t, reconciler, "", "test")
+
+		wantCm := &corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:            "dnsrecords",
+				Namespace:       "tailscale",
+				Labels:          nameserverLabels,
+				OwnerReferences: []metav1.OwnerReference{*ownerReference},
+			},
 			TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"},
 			Data:     map[string]string{"records.json": string(bs)},
 		}
+
 		expectEqual(t, fc, wantCm)
+	})
 
+	t.Run("uses default nameserver image", func(t *testing.T) {
 		// Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset,
 		// the nameserver image defaults to tailscale/k8s-nameserver:unstable.
 		mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) {
 			dnsCfg.Spec.Nameserver.Image = nil
 		})
-	expectReconciled(t, nr, "", "test")
+		expectReconciled(t, reconciler, "", "test")
 		wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:unstable"
 		expectEqual(t, fc, wantsDeploy)
+	})
 }

k8s-operator/api.md

@@ -422,6 +422,7 @@ _Appears in:_
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
 | `image` _[NameserverImage](#nameserverimage)_ | Nameserver image. Defaults to tailscale/k8s-nameserver:unstable. |  |  |
+| `service` _[NameserverService](#nameserverservice)_ | Service configuration. |  |  |
 
 
 #### NameserverImage
@@ -441,6 +442,22 @@ _Appears in:_
 | `tag` _string_ | Tag defaults to unstable. |  |  |
 
 
+#### NameserverService
+
+
+
+
+
+
+
+_Appears in:_
+- [Nameserver](#nameserver)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `clusterIP` _string_ | ClusterIP sets the static IP of the service used by the nameserver. |  |  |
+
+
 #### NameserverStatus
 
 
@@ -454,7 +471,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `ip` _string_ | IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.<br />Currently you must manually update your cluster DNS config to add<br />this address as a stub nameserver for ts.net for cluster workloads to be<br />able to resolve MagicDNS names associated with egress or Ingress<br />proxies.<br />The IP address will change if you delete and recreate the DNSConfig. |  |  |
+| `ip` _string_ | IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.<br />Currently, you must manually update your cluster DNS config to add<br />this address as a stub nameserver for ts.net for cluster workloads to be<br />able to resolve MagicDNS names associated with egress or Ingress<br />proxies.<br />The IP address will change if you delete and recreate the DNSConfig. |  |  |
 
 
 #### NodePortConfig

k8s-operator/apis/v1alpha1/types_tsdnsconfig.go

@@ -82,6 +82,9 @@ type Nameserver struct {
 	// Nameserver image. Defaults to tailscale/k8s-nameserver:unstable.
 	// +optional
 	Image *NameserverImage `json:"image,omitempty"`
+	// Service configuration.
+	// +optional
+	Service *NameserverService `json:"service,omitempty"`
 }
 
 type NameserverImage struct {
@@ -93,6 +96,12 @@ type NameserverImage struct {
 	Tag string `json:"tag,omitempty"`
 }
 
+type NameserverService struct {
+	// ClusterIP sets the static IP of the service used by the nameserver.
+	// +optional
+	ClusterIP string `json:"clusterIP,omitempty"`
+}
+
 type DNSConfigStatus struct {
 	// +listType=map
 	// +listMapKey=type
@@ -105,7 +114,7 @@ type DNSConfigStatus struct {
 
 type NameserverStatus struct {
 	// IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.
-	// Currently you must manually update your cluster DNS config to add
+	// Currently, you must manually update your cluster DNS config to add
 	// this address as a stub nameserver for ts.net for cluster workloads to be
 	// able to resolve MagicDNS names associated with egress or Ingress
 	// proxies.

k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go

@@ -385,6 +385,11 @@ func (in *Nameserver) DeepCopyInto(out *Nameserver) {
 		*out = new(NameserverImage)
 		**out = **in
 	}
+	if in.Service != nil {
+		in, out := &in.Service, &out.Service
+		*out = new(NameserverService)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Nameserver.
@@ -412,6 +417,21 @@ func (in *NameserverImage) DeepCopy() *NameserverImage {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *NameserverService) DeepCopyInto(out *NameserverService) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NameserverService.
+func (in *NameserverService) DeepCopy() *NameserverService {
+	if in == nil {
+		return nil
+	}
+	out := new(NameserverService)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NameserverStatus) DeepCopyInto(out *NameserverStatus) {
 	*out = *in