cmd/{k8s-operator,k8s-proxy},kube/k8s-proxy: add static endpoints for kube-apiserver type ProxyGroups (#16523)

Updates #13358

Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>

cmd/k8s-operator/proxygroup.go

@@ -824,6 +824,10 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 				cfg.AcceptRoutes = &proxyClass.Spec.TailscaleConfig.AcceptRoutes
 			}
 
+			if len(endpoints[nodePortSvcName]) > 0 {
+				cfg.StaticEndpoints = endpoints[nodePortSvcName]
+			}
+
 			cfgB, err := json.Marshal(cfg)
 			if err != nil {
 				return nil, fmt.Errorf("error marshalling k8s-proxy config: %w", err)

cmd/k8s-operator/proxygroup_specs.go

@@ -66,7 +66,7 @@ func pgNodePortService(pg *tsapi.ProxyGroup, name string, namespace string) *cor
 // applied over the top after.
 func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) {
 	if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer {
-		return kubeAPIServerStatefulSet(pg, namespace, image)
+		return kubeAPIServerStatefulSet(pg, namespace, image, port)
 	}
 	ss := new(appsv1.StatefulSet)
 	if err := yaml.Unmarshal(proxyYaml, &ss); err != nil {
@@ -276,7 +276,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
 	return ss, nil
 }
 
-func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string) (*appsv1.StatefulSet, error) {
+func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, port *uint16) (*appsv1.StatefulSet, error) {
 	sts := &appsv1.StatefulSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:            pg.Name,
@@ -302,48 +302,59 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string) (*a
 						{
 							Name:  mainContainerName,
 							Image: image,
-							Env: []corev1.EnvVar{
+							Env: func() []corev1.EnvVar {
-								{
+								envs := []corev1.EnvVar{
-									// Used as default hostname and in Secret names.
+									{
-									Name: "POD_NAME",
+										// Used as default hostname and in Secret names.
-									ValueFrom: &corev1.EnvVarSource{
+										Name: "POD_NAME",
-										FieldRef: &corev1.ObjectFieldSelector{
+										ValueFrom: &corev1.EnvVarSource{
-											FieldPath: "metadata.name",
+											FieldRef: &corev1.ObjectFieldSelector{
+												FieldPath: "metadata.name",
+											},
 										},
 									},
-								},
+									{
-								{
+										// Used by kubeclient to post Events about the Pod's lifecycle.
-									// Used by kubeclient to post Events about the Pod's lifecycle.
+										Name: "POD_UID",
-									Name: "POD_UID",
+										ValueFrom: &corev1.EnvVarSource{
-									ValueFrom: &corev1.EnvVarSource{
+											FieldRef: &corev1.ObjectFieldSelector{
-										FieldRef: &corev1.ObjectFieldSelector{
+												FieldPath: "metadata.uid",
-											FieldPath: "metadata.uid",
+											},
 										},
 									},
-								},
+									{
-								{
+										// Used in an interpolated env var if metrics enabled.
-									// Used in an interpolated env var if metrics enabled.
+										Name: "POD_IP",
-									Name: "POD_IP",
+										ValueFrom: &corev1.EnvVarSource{
-									ValueFrom: &corev1.EnvVarSource{
+											FieldRef: &corev1.ObjectFieldSelector{
-										FieldRef: &corev1.ObjectFieldSelector{
+												FieldPath: "status.podIP",
-											FieldPath: "status.podIP",
+											},
 										},
 									},
-								},
+									{
-								{
+										// Included for completeness with POD_IP and easier backwards compatibility in future.
-									// Included for completeness with POD_IP and easier backwards compatibility in future.
+										Name: "POD_IPS",
-									Name: "POD_IPS",
+										ValueFrom: &corev1.EnvVarSource{
-									ValueFrom: &corev1.EnvVarSource{
+											FieldRef: &corev1.ObjectFieldSelector{
-										FieldRef: &corev1.ObjectFieldSelector{
+												FieldPath: "status.podIPs",
-											FieldPath: "status.podIPs",
+											},
 										},
 									},
-								},
+									{
-								{
+										Name:  "TS_K8S_PROXY_CONFIG",
-									Name:  "TS_K8S_PROXY_CONFIG",
+										Value: filepath.Join("/etc/tsconfig/$(POD_NAME)/", kubeAPIServerConfigFile),
-									Value: filepath.Join("/etc/tsconfig/$(POD_NAME)/", kubeAPIServerConfigFile),
+									},
-								},
+								}
-							},
+
+								if port != nil {
+									envs = append(envs, corev1.EnvVar{
+										Name:  "PORT",
+										Value: strconv.Itoa(int(*port)),
+									})
+								}
+
+								return envs
+							}(),
 							VolumeMounts: func() []corev1.VolumeMount {
 								var mounts []corev1.VolumeMount
 

cmd/k8s-proxy/k8s-proxy.go

@@ -14,6 +14,7 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 	"time"
 
@@ -63,6 +64,20 @@ func run(logger *zap.SugaredLogger) error {
 		logger = logger.WithOptions(zap.IncreaseLevel(level))
 	}
 
+	// TODO:(ChaosInTheCRD) This is a temporary workaround until we can set static endpoints using prefs
+	if se := cfg.Parsed.StaticEndpoints; len(se) > 0 {
+		logger.Debugf("setting static endpoints '%v' via TS_DEBUG_PRETENDPOINT environment variable", cfg.Parsed.StaticEndpoints)
+		ses := make([]string, len(se))
+		for i, e := range se {
+			ses[i] = e.String()
+		}
+
+		err := os.Setenv("TS_DEBUG_PRETENDPOINT", strings.Join(ses, ","))
+		if err != nil {
+			return err
+		}
+	}
+
 	if cfg.Parsed.App != nil {
 		hostinfo.SetApp(*cfg.Parsed.App)
 	}

kube/k8s-proxy/conf/conf.go

@@ -10,6 +10,7 @@ package conf
 import (
 	"encoding/json"
 	"fmt"
+	"net/netip"
 	"os"
 
 	"github.com/tailscale/hujson"
@@ -55,6 +56,9 @@ type ConfigV1Alpha1 struct {
 	KubeAPIServer *KubeAPIServer `json:",omitempty"` // Config specific to the API Server proxy.
 	ServerURL     *string        `json:",omitempty"` // URL of the Tailscale coordination server.
 	AcceptRoutes  *bool          `json:",omitempty"` // Accepts routes advertised by other Tailscale nodes.
+	// StaticEndpoints are additional, user-defined endpoints that this node
+	// should advertise amongst its wireguard endpoints.
+	StaticEndpoints []netip.AddrPort `json:",omitempty"`
 }
 
 type KubeAPIServer struct {