cmd/k8s-operator: allow pod tolerations on nameservers (#17260)

This commit modifies the `DNSConfig` custom resource to allow specifying
[tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/)
on the nameserver pods.

This will allow users to dictate where their nameserver pods are located
within their clusters.

Fixes: https://github.com/tailscale/tailscale/issues/17092

Signed-off-by: David Bond <davidsbond93@gmail.com>

cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml

@@ -100,6 +100,49 @@ spec:
                         tag:
                           description: Tag defaults to unstable.
                           type: string
+                    pod:
+                      description: Pod configuration.
+                      type: object
+                      properties:
+                        tolerations:
+                          description: If specified, applies tolerations to the pods deployed by the DNSConfig resource.
+                          type: array
+                          items:
+                            description: |-
+                              The pod this Toleration is attached to tolerates any taint that matches
+                              the triple <key,value,effect> using the matching operator <operator>.
+                            type: object
+                            properties:
+                              effect:
+                                description: |-
+                                  Effect indicates the taint effect to match. Empty means match all taint effects.
+                                  When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+                                type: string
+                              key:
+                                description: |-
+                                  Key is the taint key that the toleration applies to. Empty means match all taint keys.
+                                  If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+                                type: string
+                              operator:
+                                description: |-
+                                  Operator represents a key's relationship to the value.
+                                  Valid operators are Exists and Equal. Defaults to Equal.
+                                  Exists is equivalent to wildcard for value, so that a pod can
+                                  tolerate all taints of a particular category.
+                                type: string
+                              tolerationSeconds:
+                                description: |-
+                                  TolerationSeconds represents the period of time the toleration (which must be
+                                  of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
+                                  it is not set, which means tolerate the taint forever (do not evict). Zero and
+                                  negative values will be treated as 0 (evict immediately) by the system.
+                                type: integer
+                                format: int64
+                              value:
+                                description: |-
+                                  Value is the taint value the toleration matches to.
+                                  If the operator is Exists, the value should be empty, otherwise just a regular string.
+                                type: string
                     replicas:
                       description: Replicas specifies how many Pods to create. Defaults to 1.
                       type: integer

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -431,6 +431,49 @@ spec:
                                                 description: Tag defaults to unstable.
                                                 type: string
                                         type: object
+                                    pod:
+                                        description: Pod configuration.
+                                        properties:
+                                            tolerations:
+                                                description: If specified, applies tolerations to the pods deployed by the DNSConfig resource.
+                                                items:
+                                                    description: |-
+                                                        The pod this Toleration is attached to tolerates any taint that matches
+                                                        the triple <key,value,effect> using the matching operator <operator>.
+                                                    properties:
+                                                        effect:
+                                                            description: |-
+                                                                Effect indicates the taint effect to match. Empty means match all taint effects.
+                                                                When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+                                                            type: string
+                                                        key:
+                                                            description: |-
+                                                                Key is the taint key that the toleration applies to. Empty means match all taint keys.
+                                                                If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+                                                            type: string
+                                                        operator:
+                                                            description: |-
+                                                                Operator represents a key's relationship to the value.
+                                                                Valid operators are Exists and Equal. Defaults to Equal.
+                                                                Exists is equivalent to wildcard for value, so that a pod can
+                                                                tolerate all taints of a particular category.
+                                                            type: string
+                                                        tolerationSeconds:
+                                                            description: |-
+                                                                TolerationSeconds represents the period of time the toleration (which must be
+                                                                of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
+                                                                it is not set, which means tolerate the taint forever (do not evict). Zero and
+                                                                negative values will be treated as 0 (evict immediately) by the system.
+                                                            format: int64
+                                                            type: integer
+                                                        value:
+                                                            description: |-
+                                                                Value is the taint value the toleration matches to.
+                                                                If the operator is Exists, the value should be empty, otherwise just a regular string.
+                                                            type: string
+                                                    type: object
+                                                type: array
+                                        type: object
                                     replicas:
                                         description: Replicas specifies how many Pods to create. Defaults to 1.
                                         format: int32

cmd/k8s-operator/nameserver.go

@@ -191,6 +191,9 @@ func (a *NameserverReconciler) maybeProvision(ctx context.Context, tsDNSCfg *tsa
 	if tsDNSCfg.Spec.Nameserver.Service != nil {
 		dCfg.clusterIP = tsDNSCfg.Spec.Nameserver.Service.ClusterIP
 	}
+	if tsDNSCfg.Spec.Nameserver.Pod != nil {
+		dCfg.tolerations = tsDNSCfg.Spec.Nameserver.Pod.Tolerations
+	}
 
 	for _, deployable := range []deployable{saDeployable, deployDeployable, svcDeployable, cmDeployable} {
 		if err := deployable.updateObj(ctx, dCfg, a.Client); err != nil {
@@ -217,13 +220,14 @@ type deployable struct {
 }
 
 type deployConfig struct {
-	replicas  int32
+	replicas    int32
-	imageRepo string
+	imageRepo   string
-	imageTag  string
+	imageTag    string
-	labels    map[string]string
+	labels      map[string]string
-	ownerRefs []metav1.OwnerReference
+	ownerRefs   []metav1.OwnerReference
-	namespace string
+	namespace   string
-	clusterIP string
+	clusterIP   string
+	tolerations []corev1.Toleration
 }
 
 var (
@@ -248,6 +252,7 @@ var (
 			d.ObjectMeta.Namespace = cfg.namespace
 			d.ObjectMeta.Labels = cfg.labels
 			d.ObjectMeta.OwnerReferences = cfg.ownerRefs
+			d.Spec.Template.Spec.Tolerations = cfg.tolerations
 			updateF := func(oldD *appsv1.Deployment) {
 				oldD.Spec = d.Spec
 			}

cmd/k8s-operator/nameserver_test.go

@@ -42,6 +42,16 @@ func TestNameserverReconciler(t *testing.T) {
 				Service: &tsapi.NameserverService{
 					ClusterIP: "5.4.3.2",
 				},
+				Pod: &tsapi.NameserverPod{
+					Tolerations: []corev1.Toleration{
+						{
+							Key:      "some-key",
+							Operator: corev1.TolerationOpEqual,
+							Value:    "some-value",
+							Effect:   corev1.TaintEffectNoSchedule,
+						},
+					},
+				},
 			},
 		},
 	}
@@ -79,6 +89,15 @@ func TestNameserverReconciler(t *testing.T) {
 		wantsDeploy.Spec.Replicas = ptr.To[int32](3)
 		wantsDeploy.Namespace = tsNamespace
 		wantsDeploy.ObjectMeta.Labels = nameserverLabels
+		wantsDeploy.Spec.Template.Spec.Tolerations = []corev1.Toleration{
+			{
+				Key:      "some-key",
+				Operator: corev1.TolerationOpEqual,
+				Value:    "some-value",
+				Effect:   corev1.TaintEffectNoSchedule,
+			},
+		}
+
 		expectEqual(t, fc, wantsDeploy)
 	})
 

k8s-operator/api.md

@@ -443,6 +443,7 @@ _Appears in:_
 | --- | --- | --- | --- |
 | `image` _[NameserverImage](#nameserverimage)_ | Nameserver image. Defaults to tailscale/k8s-nameserver:unstable. |  |  |
 | `service` _[NameserverService](#nameserverservice)_ | Service configuration. |  |  |
+| `pod` _[NameserverPod](#nameserverpod)_ | Pod configuration. |  |  |
 | `replicas` _integer_ | Replicas specifies how many Pods to create. Defaults to 1. |  | Minimum: 0 <br /> |
 
 
@@ -463,6 +464,22 @@ _Appears in:_
 | `tag` _string_ | Tag defaults to unstable. |  |  |
 
 
+#### NameserverPod
+
+
+
+
+
+
+
+_Appears in:_
+- [Nameserver](#nameserver)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#toleration-v1-core) array_ | If specified, applies tolerations to the pods deployed by the DNSConfig resource. |  |  |
+
+
 #### NameserverService
 
 

k8s-operator/apis/v1alpha1/types_tsdnsconfig.go

@@ -6,6 +6,7 @@
 package v1alpha1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -84,6 +85,9 @@ type Nameserver struct {
 	// Service configuration.
 	// +optional
 	Service *NameserverService `json:"service,omitempty"`
+	// Pod configuration.
+	// +optional
+	Pod *NameserverPod `json:"pod,omitempty"`
 	// Replicas specifies how many Pods to create. Defaults to 1.
 	// +optional
 	// +kubebuilder:validation:Minimum=0
@@ -105,6 +109,12 @@ type NameserverService struct {
 	ClusterIP string `json:"clusterIP,omitempty"`
 }
 
+type NameserverPod struct {
+	// If specified, applies tolerations to the pods deployed by the DNSConfig resource.
+	// +optional
+	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+}
+
 type DNSConfigStatus struct {
 	// +listType=map
 	// +listMapKey=type

k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go

@@ -422,6 +422,11 @@ func (in *Nameserver) DeepCopyInto(out *Nameserver) {
 		*out = new(NameserverService)
 		**out = **in
 	}
+	if in.Pod != nil {
+		in, out := &in.Pod, &out.Pod
+		*out = new(NameserverPod)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.Replicas != nil {
 		in, out := &in.Replicas, &out.Replicas
 		*out = new(int32)
@@ -454,6 +459,28 @@ func (in *NameserverImage) DeepCopy() *NameserverImage {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *NameserverPod) DeepCopyInto(out *NameserverPod) {
+	*out = *in
+	if in.Tolerations != nil {
+		in, out := &in.Tolerations, &out.Tolerations
+		*out = make([]corev1.Toleration, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NameserverPod.
+func (in *NameserverPod) DeepCopy() *NameserverPod {
+	if in == nil {
+		return nil
+	}
+	out := new(NameserverPod)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NameserverService) DeepCopyInto(out *NameserverService) {
 	*out = *in