From fe709c81e50e22f579c20023aba24c15db3f52e3 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Wed, 30 Aug 2023 08:31:37 +0100 Subject: [PATCH] cmd/k8s-operator,cmd/containerboot: add kube egress proxy (#9031) First part of work for the functionality that allows users to create an egress proxy to access Tailnet services from within Kubernetes cluster workloads. This PR allows creating an egress proxy that can access Tailscale services over HTTP only. Updates tailscale/tailscale#8184 Signed-off-by: irbekrm Co-authored-by: Maisem Ali Co-authored-by: Rhea Ghosh --- cmd/containerboot/main.go | 113 ++++++++++++++++--- cmd/k8s-operator/ingress.go | 2 +- cmd/k8s-operator/operator_test.go | 179 +++++++++++++++++++++++++++++- cmd/k8s-operator/sts.go | 47 +++++--- cmd/k8s-operator/svc.go | 47 +++++--- 5 files changed, 338 insertions(+), 50 deletions(-) diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 1a2bb70d5..3676db02c 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -16,6 +16,8 @@ // - TS_ROUTES: subnet routes to advertise. // - TS_DEST_IP: proxy all incoming Tailscale traffic to the given // destination. +// - TS_TAILNET_TARGET_IP: proxy all incoming non-Tailscale traffic to the given +// destination. // - TS_TAILSCALED_EXTRA_ARGS: extra arguments to 'tailscaled'. // - TS_EXTRA_ARGS: extra arguments to 'tailscale login', these are not // reset on restart. @@ -88,8 +90,9 @@ func main() { AuthKey: defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""), Hostname: defaultEnv("TS_HOSTNAME", ""), Routes: defaultEnv("TS_ROUTES", ""), - ProxyTo: defaultEnv("TS_DEST_IP", ""), ServeConfigPath: defaultEnv("TS_SERVE_CONFIG", ""), + ProxyTo: defaultEnv("TS_DEST_IP", ""), + TailnetTargetIP: defaultEnv("TS_TAILNET_TARGET_IP", ""), DaemonExtraArgs: defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""), ExtraArgs: defaultEnv("TS_EXTRA_ARGS", ""), InKubernetes: os.Getenv("KUBERNETES_SERVICE_HOST") != "", @@ -107,16 +110,17 @@ func main() { if cfg.ProxyTo != "" && cfg.UserspaceMode { log.Fatal("TS_DEST_IP is not supported with TS_USERSPACE") } - if cfg.ProxyTo != "" && cfg.ServeConfigPath != "" { - log.Fatal("TS_DEST_IP is not supported with TS_SERVE_CONFIG") + + if cfg.TailnetTargetIP != "" && cfg.UserspaceMode { + log.Fatal("TS_TAILNET_TARGET_IP is not supported with TS_USERSPACE") } if !cfg.UserspaceMode { if err := ensureTunFile(cfg.Root); err != nil { log.Fatalf("Unable to create tuntap device file: %v", err) } - if cfg.ProxyTo != "" || cfg.Routes != "" { - if err := ensureIPForwarding(cfg.Root, cfg.ProxyTo, cfg.Routes); err != nil { + if cfg.ProxyTo != "" || cfg.Routes != "" || cfg.TailnetTargetIP != "" { + if err := ensureIPForwarding(cfg.Root, cfg.ProxyTo, cfg.TailnetTargetIP, cfg.Routes); err != nil { log.Printf("Failed to enable IP forwarding: %v", err) log.Printf("To run tailscale as a proxy or router container, IP forwarding must be enabled.") if cfg.InKubernetes { @@ -270,7 +274,7 @@ authLoop: } var ( - wantProxy = cfg.ProxyTo != "" + wantProxy = cfg.ProxyTo != "" || cfg.TailnetTargetIP != "" wantDeviceInfo = cfg.InKubernetes && cfg.KubeSecret != "" && cfg.KubernetesCanPatch startupTasksDone = false currentIPs deephash.Sum // tailscale IPs assigned to device @@ -298,10 +302,12 @@ authLoop: } if n.NetMap != nil { addrs := n.NetMap.SelfNode.Addresses().AsSlice() - if cfg.ProxyTo != "" && len(addrs) > 0 && deephash.Update(¤tIPs, &addrs) { + newCurrentIPs := deephash.Hash(&addrs) + ipsHaveChanged := newCurrentIPs != currentIPs + if cfg.ProxyTo != "" && len(addrs) > 0 && ipsHaveChanged { log.Printf("Installing proxy rules") - if err := installIPTablesRule(ctx, cfg.ProxyTo, addrs); err != nil { - log.Fatalf("installing proxy rules: %v", err) + if err := installIngressForwardingRule(ctx, cfg.ProxyTo, addrs); err != nil { + log.Fatalf("installing ingress proxy rules: %v", err) } } if cfg.ServeConfigPath != "" && len(n.NetMap.DNS.CertDomains) > 0 { @@ -314,6 +320,13 @@ authLoop: } } } + if cfg.TailnetTargetIP != "" && ipsHaveChanged && len(addrs) > 0 { + if err := installEgressForwardingRule(ctx, cfg.TailnetTargetIP, addrs); err != nil { + log.Fatalf("installing egress proxy rules: %v", err) + } + } + currentIPs = newCurrentIPs + deviceInfo := []any{n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name()} if cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != "" && deephash.Update(¤tDeviceInfo, &deviceInfo) { if err := storeDeviceInfo(ctx, cfg.KubeSecret, n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses().AsSlice()); err != nil { @@ -572,14 +585,25 @@ func ensureTunFile(root string) error { } // ensureIPForwarding enables IPv4/IPv6 forwarding for the container. -func ensureIPForwarding(root, proxyTo, routes string) error { +func ensureIPForwarding(root, clusterProxyTarget, tailnetTargetiP, routes string) error { var ( v4Forwarding, v6Forwarding bool ) - if proxyTo != "" { - proxyIP, err := netip.ParseAddr(proxyTo) + if clusterProxyTarget != "" { + proxyIP, err := netip.ParseAddr(clusterProxyTarget) if err != nil { - return fmt.Errorf("invalid proxy destination IP: %v", err) + return fmt.Errorf("invalid cluster destination IP: %v", err) + } + if proxyIP.Is4() { + v4Forwarding = true + } else { + v6Forwarding = true + } + } + if tailnetTargetiP != "" { + proxyIP, err := netip.ParseAddr(tailnetTargetiP) + if err != nil { + return fmt.Errorf("invalid tailnet destination IP: %v", err) } if proxyIP.Is4() { v4Forwarding = true @@ -629,7 +653,53 @@ func ensureIPForwarding(root, proxyTo, routes string) error { return nil } -func installIPTablesRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix) error { +func installEgressForwardingRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix) error { + dst, err := netip.ParseAddr(dstStr) + if err != nil { + return err + } + argv0 := "iptables" + if dst.Is6() { + argv0 = "ip6tables" + } + var local string + for _, pfx := range tsIPs { + if !pfx.IsSingleIP() { + continue + } + if pfx.Addr().Is4() != dst.Is4() { + continue + } + local = pfx.Addr().String() + break + } + if local == "" { + return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstStr, tsIPs) + } + // Technically, if the control server ever changes the IPs assigned to this + // node, we'll slowly accumulate iptables rules. This shouldn't happen, so + // for now we'll live with it. + // Set up a rule that ensures that all packets + // except for those received on tailscale0 interface is forwarded to + // destination address + cmdDNAT := exec.CommandContext(ctx, argv0, "-t", "nat", "-I", "PREROUTING", "1", "!", "-i", "tailscale0", "-j", "DNAT", "--to-destination", dstStr) + cmdDNAT.Stdout = os.Stdout + cmdDNAT.Stderr = os.Stderr + if err := cmdDNAT.Run(); err != nil { + return fmt.Errorf("executing iptables failed: %w", err) + } + // Set up a rule that ensures that all packets sent to the destination + // address will have the proxy's IP set as source IP + cmdSNAT := exec.CommandContext(ctx, argv0, "-t", "nat", "-I", "POSTROUTING", "1", "--destination", dstStr, "-j", "SNAT", "--to-source", local) + cmdSNAT.Stdout = os.Stdout + cmdSNAT.Stderr = os.Stderr + if err := cmdSNAT.Run(); err != nil { + return fmt.Errorf("setting up SNAT via iptables failed: %w", err) + } + return nil +} + +func installIngressForwardingRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix) error { dst, err := netip.ParseAddr(dstStr) if err != nil { return err @@ -666,10 +736,17 @@ func installIPTablesRule(ctx context.Context, dstStr string, tsIPs []netip.Prefi // settings is all the configuration for containerboot. type settings struct { - AuthKey string - Hostname string - Routes string - ProxyTo string + AuthKey string + Hostname string + Routes string + // ProxyTo is the destination IP to which all incoming + // Tailscale traffic should be proxied. If empty, no proxying + // is done. This is typically a locally reachable IP. + ProxyTo string + // TailnetTargetIP is the destination IP to which all incoming + // non-Tailscale traffic should be proxied. If empty, no + // proxying is done. This is typically a Tailscale IP. + TailnetTargetIP string ServeConfigPath string DaemonExtraArgs string ExtraArgs string diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index dd52299e0..ff1440cd8 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -190,7 +190,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga ChildResourceLabels: crl, } - if err := a.ssr.Provision(ctx, logger, sts); err != nil { + if _, err := a.ssr.Provision(ctx, logger, sts); err != nil { return fmt.Errorf("failed to provision: %w", err) } diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index 00cea9e92..48bedeac6 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -7,6 +7,7 @@ package main import ( "context" + "fmt" "strings" "sync" "testing" @@ -153,6 +154,111 @@ func TestLoadBalancerClass(t *testing.T) { } expectEqual(t, fc, want) } +func TestTailnetTargetIPAnnotation(t *testing.T) { + fc := fake.NewFakeClient() + ft := &fakeTSClient{} + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + tailnetTargetIP := "100.66.66.66" + sr := &ServiceReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: ft, + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + logger: zl.Sugar(), + } + + // Create a service that we should manage, and check that the initial round + // of objects looks right. + mustCreate(t, fc, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + // The apiserver is supposed to set the UID, but the fake client + // doesn't. So, set it explicitly because other code later depends + // on it being set. + UID: types.UID("1234-UID"), + Annotations: map[string]string{ + AnnotationTailnetTargetIP: tailnetTargetIP, + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: map[string]string{ + "foo": "bar", + }, + }, + }) + + expectReconciled(t, sr, "default", "test") + + fullName, shortName := findGenName(t, fc, "default", "test") + + expectEqual(t, fc, expectedSecret(fullName)) + expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedEgressSTS(shortName, fullName, tailnetTargetIP, "default-test", "")) + want := &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + Finalizers: []string{"tailscale.com/finalizer"}, + UID: types.UID("1234-UID"), + Annotations: map[string]string{ + AnnotationTailnetTargetIP: tailnetTargetIP, + }, + }, + Spec: corev1.ServiceSpec{ + ExternalName: fmt.Sprintf("%s.operator-ns.svc", shortName), + Type: corev1.ServiceTypeExternalName, + Selector: nil, + }, + } + expectEqual(t, fc, want) + expectEqual(t, fc, expectedSecret(fullName)) + expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedEgressSTS(shortName, fullName, tailnetTargetIP, "default-test", "")) + + // Change the tailscale-target-ip annotation which should update the + // StatefulSet + tailnetTargetIP = "100.77.77.77" + mustUpdate(t, fc, "default", "test", func(s *corev1.Service) { + s.ObjectMeta.Annotations = map[string]string{ + AnnotationTailnetTargetIP: tailnetTargetIP, + } + }) + + // Remove the tailscale-target-ip annotation which should make the + // operator clean up + mustUpdate(t, fc, "default", "test", func(s *corev1.Service) { + s.ObjectMeta.Annotations = map[string]string{} + }) + expectReconciled(t, sr, "default", "test") + + // // synchronous StatefulSet deletion triggers a requeue. But, the StatefulSet + // // didn't create any child resources since this is all faked, so the + // // deletion goes through immediately. + expectReconciled(t, sr, "default", "test") + expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName) + // // The deletion triggers another reconcile, to finish the cleanup. + expectReconciled(t, sr, "default", "test") + expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName) + expectMissing[corev1.Service](t, fc, "operator-ns", shortName) + expectMissing[corev1.Secret](t, fc, "operator-ns", fullName) + + // At the moment we don't revert changes to the user created Service - + // we don't have a reliable way how to tell what it was before and also + // we don't really expect it to be re-used +} func TestAnnotations(t *testing.T) { fc := fake.NewFakeClient() @@ -781,8 +887,8 @@ func expectedSTS(stsName, secretName, hostname, priorityClassName string) *appsv Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "tailscale.com/operator-last-set-hostname": hostname, - "tailscale.com/operator-last-set-ip": "10.20.30.40", + "tailscale.com/operator-last-set-hostname": hostname, + "tailscale.com/operator-last-set-cluster-ip": "10.20.30.40", }, DeletionGracePeriodSeconds: ptr.To[int64](10), Labels: map[string]string{"app": "1234-UID"}, @@ -825,6 +931,75 @@ func expectedSTS(stsName, secretName, hostname, priorityClassName string) *appsv }, } } +func expectedEgressSTS(stsName, secretName, tailnetTargetIP, hostname, priorityClassName string) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: stsName, + Namespace: "operator-ns", + Labels: map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource": "test", + "tailscale.com/parent-resource-ns": "default", + "tailscale.com/parent-resource-type": "svc", + }, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To[int32](1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "1234-UID"}, + }, + ServiceName: stsName, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "tailscale.com/operator-last-set-hostname": hostname, + "tailscale.com/operator-last-set-ts-tailnet-target-ip": tailnetTargetIP, + }, + DeletionGracePeriodSeconds: ptr.To[int64](10), + Labels: map[string]string{"app": "1234-UID"}, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "proxies", + PriorityClassName: priorityClassName, + InitContainers: []corev1.Container{ + { + Name: "sysctler", + Image: "busybox", + Command: []string{"/bin/sh"}, + Args: []string{"-c", "sysctl -w net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1"}, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "tailscale", + Image: "tailscale/tailscale", + Env: []corev1.EnvVar{ + {Name: "TS_USERSPACE", Value: "false"}, + {Name: "TS_AUTH_ONCE", Value: "true"}, + {Name: "TS_KUBE_SECRET", Value: secretName}, + {Name: "TS_HOSTNAME", Value: hostname}, + {Name: "TS_TAILNET_TARGET_IP", Value: tailnetTargetIP}, + }, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN"}, + }, + }, + ImagePullPolicy: "Always", + }, + }, + }, + }, + }, + } +} func findGenName(t *testing.T, client client.Client, ns, name string) (full, noSuffix string) { t.Helper() diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 9994f01ca..a706835a2 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -38,17 +38,19 @@ const ( FinalizerName = "tailscale.com/finalizer" // Annotations settable by users on services. - AnnotationExpose = "tailscale.com/expose" - AnnotationTags = "tailscale.com/tags" - AnnotationHostname = "tailscale.com/hostname" + AnnotationExpose = "tailscale.com/expose" + AnnotationTags = "tailscale.com/tags" + AnnotationHostname = "tailscale.com/hostname" + AnnotationTailnetTargetIP = "tailscale.com/ts-tailnet-target-ip" // Annotations settable by users on ingresses. AnnotationFunnel = "tailscale.com/funnel" // Annotations set by the operator on pods to trigger restarts when the // hostname or IP changes. - podAnnotationLastSetIP = "tailscale.com/operator-last-set-ip" - podAnnotationLastSetHostname = "tailscale.com/operator-last-set-hostname" + podAnnotationLastSetClusterIP = "tailscale.com/operator-last-set-cluster-ip" + podAnnotationLastSetHostname = "tailscale.com/operator-last-set-hostname" + podAnnotationLastSetTailnetTargetIP = "tailscale.com/operator-last-set-ts-tailnet-target-ip" ) type tailscaleSTSConfig struct { @@ -57,7 +59,11 @@ type tailscaleSTSConfig struct { ChildResourceLabels map[string]string ServeConfig *ipn.ServeConfig - TargetIP string + // Tailscale target in cluster we are setting up ingress for + ClusterTargetIP string + + // Tailscale IP of a Tailscale service we are setting up egress for + TailnetTargetIP string Hostname string Tags []string // if empty, use defaultTags @@ -74,23 +80,23 @@ type tailscaleSTSReconciler struct { // Provision ensures that the StatefulSet for the given service is running and // up to date. -func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig) error { +func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig) (*corev1.Service, error) { // Do full reconcile. hsvc, err := a.reconcileHeadlessService(ctx, logger, sts) if err != nil { - return fmt.Errorf("failed to reconcile headless service: %w", err) + return nil, fmt.Errorf("failed to reconcile headless service: %w", err) } secretName, err := a.createOrGetSecret(ctx, logger, sts, hsvc) if err != nil { - return fmt.Errorf("failed to create or get API key secret: %w", err) + return nil, fmt.Errorf("failed to create or get API key secret: %w", err) } _, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName) if err != nil { - return fmt.Errorf("failed to reconcile statefulset: %w", err) + return nil, fmt.Errorf("failed to reconcile statefulset: %w", err) } - return nil + return hsvc, nil } // Cleanup removes all resources associated that were created by Provision with @@ -305,11 +311,17 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S Name: "TS_HOSTNAME", Value: sts.Hostname, }) - if sts.TargetIP != "" { + if sts.ClusterTargetIP != "" { container.Env = append(container.Env, corev1.EnvVar{ Name: "TS_DEST_IP", - Value: sts.TargetIP, + Value: sts.ClusterTargetIP, + }) + } else if sts.TailnetTargetIP != "" { + container.Env = append(container.Env, corev1.EnvVar{ + Name: "TS_TAILNET_TARGET_IP", + Value: sts.TailnetTargetIP, }) + } else if sts.ServeConfig != nil { container.Env = append(container.Env, corev1.EnvVar{ Name: "TS_SERVE_CONFIG", @@ -350,10 +362,13 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S // container when the value changes. We do this by adding an annotation to // the pod template that contains the last value we set. ss.Spec.Template.Annotations = map[string]string{ - "tailscale.com/operator-last-set-hostname": sts.Hostname, + podAnnotationLastSetHostname: sts.Hostname, + } + if sts.ClusterTargetIP != "" { + ss.Spec.Template.Annotations[podAnnotationLastSetClusterIP] = sts.ClusterTargetIP } - if sts.TargetIP != "" { - ss.Spec.Template.Annotations["tailscale.com/operator-last-set-ip"] = sts.TargetIP + if sts.TailnetTargetIP != "" { + ss.Spec.Template.Annotations[podAnnotationLastSetTailnetTargetIP] = sts.TailnetTargetIP } ss.Spec.Template.Labels = map[string]string{ "app": sts.ParentResourceUID, diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index a14a61c61..3bab3532a 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -55,8 +55,8 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request } else if err != nil { return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err) } - if !svc.DeletionTimestamp.IsZero() || !a.shouldExpose(svc) { - logger.Debugf("service is being deleted or should not be exposed, cleaning up") + if !svc.DeletionTimestamp.IsZero() || !a.shouldExpose(svc) && !a.hasTailnetTargetAnnotation(svc) { + logger.Debugf("service is being deleted or is (no longer) referring to Tailscale ingress/egress, ensuring any created resources are cleaned up") return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc) } @@ -122,24 +122,34 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga tags = strings.Split(tstr, ",") } - clusterIPAddr, err := netip.ParseAddr(svc.Spec.ClusterIP) - if err != nil { - return fmt.Errorf("failed to parse cluster IP: %w", err) - } - sts := &tailscaleSTSConfig{ ParentResourceName: svc.Name, ParentResourceUID: string(svc.UID), - TargetIP: svc.Spec.ClusterIP, + ClusterTargetIP: svc.Spec.ClusterIP, Hostname: hostname, Tags: tags, ChildResourceLabels: crl, + TailnetTargetIP: svc.Annotations[AnnotationTailnetTargetIP], } - if err := a.ssr.Provision(ctx, logger, sts); err != nil { + var hsvc *corev1.Service + if hsvc, err = a.ssr.Provision(ctx, logger, sts); err != nil { return fmt.Errorf("failed to provision: %w", err) } + if a.hasTailnetTargetAnnotation(svc) { + headlessSvcName := hsvc.Name + "." + hsvc.Namespace + ".svc" + if svc.Spec.ExternalName != headlessSvcName || svc.Spec.Type != corev1.ServiceTypeExternalName { + svc.Spec.ExternalName = headlessSvcName + svc.Spec.Selector = nil + svc.Spec.Type = corev1.ServiceTypeExternalName + if err := a.Update(ctx, svc); err != nil { + return fmt.Errorf("failed to update service: %w", err) + } + } + return nil + } + if !a.hasLoadBalancerClass(svc) { logger.Debugf("service is not a LoadBalancer, so not updating ingress") return nil @@ -163,6 +173,10 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga ingress := []corev1.LoadBalancerIngress{ {Hostname: tsHost}, } + clusterIPAddr, err := netip.ParseAddr(svc.Spec.ClusterIP) + if err != nil { + return fmt.Errorf("failed to parse cluster IP: %w", err) + } for _, ip := range tsIPs { addr, err := netip.ParseAddr(ip) if err != nil { @@ -186,7 +200,7 @@ func (a *ServiceReconciler) shouldExpose(svc *corev1.Service) bool { return false } - return a.hasLoadBalancerClass(svc) || a.hasAnnotation(svc) + return a.hasLoadBalancerClass(svc) || a.hasExposeAnnotation(svc) } func (a *ServiceReconciler) hasLoadBalancerClass(svc *corev1.Service) bool { @@ -196,7 +210,14 @@ func (a *ServiceReconciler) hasLoadBalancerClass(svc *corev1.Service) bool { svc.Spec.LoadBalancerClass == nil && a.isDefaultLoadBalancer) } -func (a *ServiceReconciler) hasAnnotation(svc *corev1.Service) bool { - return svc != nil && - svc.Annotations[AnnotationExpose] == "true" +// hasExposeAnnotation reports whether Service has the tailscale.com/expose +// annotation set +func (a *ServiceReconciler) hasExposeAnnotation(svc *corev1.Service) bool { + return svc != nil && svc.Annotations[AnnotationExpose] == "true" +} + +// hasTailnetTargetAnnotation reports whether Service has a +// tailscale.com/ts-tailnet-target-ip annotation set +func (a *ServiceReconciler) hasTailnetTargetAnnotation(svc *corev1.Service) bool { + return svc != nil && svc.Annotations[AnnotationTailnetTargetIP] != "" }