cmd/k8s-operator: base ProxyGroup StatefulSet on common proxy.yaml definition (#13714)

As discussed in #13684, base the ProxyGroup's proxy definitions on the same
scaffolding as the existing proxies, as defined in proxy.yaml

Updates #13406

Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>

cmd/k8s-operator/proxygroup.go

@@ -239,7 +239,10 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 			return fmt.Errorf("error provisioning ConfigMap: %w", err)
 		}
 	}
-	ss := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, cfgHash)
+	ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, cfgHash)
+	if err != nil {
+		return fmt.Errorf("error generating StatefulSet spec: %w", err)
+	}
 	ss = applyProxyClassToStatefulSet(proxyClass, ss, nil, logger)
 	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) {
 		s.ObjectMeta.Labels = ss.ObjectMeta.Labels

cmd/k8s-operator/proxygroup_specs.go

@@ -12,6 +12,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/yaml"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/egressservices"
 	"tailscale.com/types/ptr"
@@ -19,21 +20,34 @@ import (
 
 // Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be
 // applied over the top after.
-func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHash string) *appsv1.StatefulSet {
+func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHash string) (*appsv1.StatefulSet, error) {
-	return &appsv1.StatefulSet{
+	ss := new(appsv1.StatefulSet)
-		ObjectMeta: metav1.ObjectMeta{
+	if err := yaml.Unmarshal(proxyYaml, &ss); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err)
+	}
+	// Validate some base assumptions.
+	if len(ss.Spec.Template.Spec.InitContainers) != 1 {
+		return nil, fmt.Errorf("[unexpected] base proxy config had %d init containers instead of 1", len(ss.Spec.Template.Spec.InitContainers))
+	}
+	if len(ss.Spec.Template.Spec.Containers) != 1 {
+		return nil, fmt.Errorf("[unexpected] base proxy config had %d containers instead of 1", len(ss.Spec.Template.Spec.Containers))
+	}
+
+	// StatefulSet config.
+	ss.ObjectMeta = metav1.ObjectMeta{
 		Name:            pg.Name,
 		Namespace:       namespace,
 		Labels:          pgLabels(pg.Name, nil),
 		OwnerReferences: pgOwnerReference(pg),
-		},
+	}
-		Spec: appsv1.StatefulSetSpec{
+	ss.Spec.Replicas = ptr.To(pgReplicas(pg))
-			Replicas: ptr.To(pgReplicas(pg)),
+	ss.Spec.Selector = &metav1.LabelSelector{
-			Selector: &metav1.LabelSelector{
 		MatchLabels: pgLabels(pg.Name, nil),
-			},
+	}
-			Template: corev1.PodTemplateSpec{
+
-				ObjectMeta: metav1.ObjectMeta{
+	// Template config.
+	tmpl := &ss.Spec.Template
+	tmpl.ObjectMeta = metav1.ObjectMeta{
 		Name:                       pg.Name,
 		Namespace:                  namespace,
 		Labels:                     pgLabels(pg.Name, nil),
@@ -41,37 +55,42 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 		Annotations: map[string]string{
 			podAnnotationLastSetConfigFileHash: cfgHash,
 		},
-				},
+	}
-				Spec: corev1.PodSpec{
+	tmpl.Spec.ServiceAccountName = pg.Name
-					ServiceAccountName: pg.Name,
+	tmpl.Spec.InitContainers[0].Image = image
-					InitContainers: []corev1.Container{
+	tmpl.Spec.Volumes = func() []corev1.Volume {
-						{
+		var volumes []corev1.Volume
-							Name:  "sysctler",
+		for i := range pgReplicas(pg) {
-							Image: image,
+			volumes = append(volumes, corev1.Volume{
-							SecurityContext: &corev1.SecurityContext{
+				Name: fmt.Sprintf("tailscaledconfig-%d", i),
-								Privileged: ptr.To(true),
+				VolumeSource: corev1.VolumeSource{
-							},
+					Secret: &corev1.SecretVolumeSource{
-							Command: []string{
+						SecretName: fmt.Sprintf("%s-%d-config", pg.Name, i),
-								"/bin/sh",
-								"-c",
-							},
-							Args: []string{
-								"sysctl -w net.ipv4.ip_forward=1 && if sysctl net.ipv6.conf.all.forwarding; then sysctl -w net.ipv6.conf.all.forwarding=1; fi",
-							},
 					},
 				},
-					Containers: []corev1.Container{
+			})
-						{
+		}
-							Name:  "tailscale",
+
-							Image: image,
+		if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
-							SecurityContext: &corev1.SecurityContext{
+			volumes = append(volumes, corev1.Volume{
-								Capabilities: &corev1.Capabilities{
+				Name: pgEgressCMName(pg.Name),
-									Add: []corev1.Capability{
+				VolumeSource: corev1.VolumeSource{
-										"NET_ADMIN",
+					ConfigMap: &corev1.ConfigMapVolumeSource{
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: pgEgressCMName(pg.Name),
 						},
 					},
 				},
-							VolumeMounts: func() []corev1.VolumeMount {
+			})
+		}
+
+		return volumes
+	}()
+
+	// Main container config.
+	c := &ss.Spec.Template.Spec.Containers[0]
+	c.Image = image
+	c.VolumeMounts = func() []corev1.VolumeMount {
 		var mounts []corev1.VolumeMount
 		for i := range pgReplicas(pg) {
 			mounts = append(mounts, corev1.VolumeMount{
@@ -88,9 +107,10 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 				ReadOnly:  true,
 			})
 		}
+
 		return mounts
-							}(),
+	}()
-							Env: func() []corev1.EnvVar {
+	c.Env = func() []corev1.EnvVar {
 		envs := []corev1.EnvVar{
 			{
 				// TODO(irbekrm): verify that .status.podIPs are always set, else read in .status.podIP as well.
@@ -127,12 +147,6 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 				Value: "false",
 			},
 		}
-								if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
-									envs = append(envs, corev1.EnvVar{
-										Name:  "TS_EGRESS_SERVICES_CONFIG_PATH",
-										Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices),
-									})
-								}
 
 		if tsFirewallMode != "" {
 			envs = append(envs, corev1.EnvVar{
@@ -141,41 +155,17 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 			})
 		}
 
-								return envs
-							}(),
-						},
-					},
-					Volumes: func() []corev1.Volume {
-						var volumes []corev1.Volume
-						for i := range pgReplicas(pg) {
-							volumes = append(volumes, corev1.Volume{
-								Name: fmt.Sprintf("tailscaledconfig-%d", i),
-								VolumeSource: corev1.VolumeSource{
-									Secret: &corev1.SecretVolumeSource{
-										SecretName: fmt.Sprintf("%s-%d-config", pg.Name, i),
-									},
-								},
-							})
-						}
 		if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
-							volumes = append(volumes, corev1.Volume{
+			envs = append(envs, corev1.EnvVar{
-								Name: pgEgressCMName(pg.Name),
+				Name:  "TS_EGRESS_SERVICES_CONFIG_PATH",
-								VolumeSource: corev1.VolumeSource{
+				Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices),
-									ConfigMap: &corev1.ConfigMapVolumeSource{
-										LocalObjectReference: corev1.LocalObjectReference{
-											Name: pgEgressCMName(pg.Name),
-										},
-									},
-								},
 			})
 		}
 
-						return volumes
+		return envs
-					}(),
+	}()
-				},
+
-			},
+	return ss, nil
-		},
-	}
 }
 
 func pgServiceAccount(pg *tsapi.ProxyGroup, namespace string) *corev1.ServiceAccount {

cmd/k8s-operator/proxygroup_test.go

@@ -197,7 +197,10 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox
 	role := pgRole(pg, tsNamespace)
 	roleBinding := pgRoleBinding(pg, tsNamespace)
 	serviceAccount := pgServiceAccount(pg, tsNamespace)
-	statefulSet := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", "")
+	statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", "")
+	if err != nil {
+		t.Fatal(err)
+	}
 	statefulSet.Annotations = defaultProxyClassAnnotations
 
 	if shouldExist {