From a6cc2fdc3e2645a6931c128f8e8f716680e968b0 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Thu, 8 Feb 2024 06:45:42 +0000 Subject: [PATCH] cmd/{containerboot,k8s-operator/deploy/manifests}: optionally allow proxying cluster traffic to a cluster target via ingress proxy (#11036) * cmd/containerboot,cmd/k8s-operator/deploy/manifests: optionally forward cluster traffic via ingress proxy. If a tailscale Ingress has tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation, configure the associated ingress proxy to have its tailscale serve proxy to listen on Pod's IP address. This ensures that cluster traffic too can be forwarded via this proxy to the ingress backend(s). In containerboot, if EXPERIMENTAL_PROXY_CLUSTER_TRAFFIC_VIA_INGRESS is set to true and the node is Kubernetes operator ingress proxy configured via Ingress, make sure that traffic from within the cluster can be proxied to the ingress target. Updates tailscale/tailscale#10499 Signed-off-by: Irbe Krumina --- cmd/containerboot/main.go | 114 ++++++++++++--- cmd/k8s-operator/deploy/manifests/proxy.yaml | 4 + cmd/k8s-operator/ingress.go | 4 + cmd/k8s-operator/ingress_test.go | 139 +++++++++++++++++++ cmd/k8s-operator/operator_test.go | 24 ++-- cmd/k8s-operator/sts.go | 34 ++++- cmd/k8s-operator/testutils_test.go | 126 ++++++++++++++--- 7 files changed, 390 insertions(+), 55 deletions(-) create mode 100644 cmd/k8s-operator/ingress_test.go diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 2dffcfb4b..5d14826d8 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -55,6 +55,15 @@ // and not `tailscale up` or `tailscale set`. // The config file contents are currently read once on container start. // NB: This env var is currently experimental and the logic will likely change! +// - EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS: if set to true +// and if this containerboot instance is an L7 ingress proxy (created by +// the Kubernetes operator), set up rules to allow proxying cluster traffic, +// received on the Pod IP of this node, to the ingress target in the cluster. +// This, in conjunction with MagicDNS name resolution in cluster, can be +// useful for cases where a cluster workload needs to access a target in +// cluster using the same hostname (in this case, the MagicDNS name of the ingress proxy) +// as a non-cluster workload on tailnet. +// This is only meant to be configured by the Kubernetes operator. // // When running on Kubernetes, containerboot defaults to storing state in the // "tailscale" kube secret. To store state on local disk instead, set @@ -108,29 +117,31 @@ func newNetfilterRunner(logf logger.Logf) (linuxfw.NetfilterRunner, error) { func main() { log.SetPrefix("boot: ") tailscale.I_Acknowledge_This_API_Is_Unstable = true - cfg := &settings{ - AuthKey: defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""), - Hostname: defaultEnv("TS_HOSTNAME", ""), - Routes: defaultEnvStringPointer("TS_ROUTES"), - ServeConfigPath: defaultEnv("TS_SERVE_CONFIG", ""), - ProxyTo: defaultEnv("TS_DEST_IP", ""), - TailnetTargetIP: defaultEnv("TS_TAILNET_TARGET_IP", ""), - TailnetTargetFQDN: defaultEnv("TS_TAILNET_TARGET_FQDN", ""), - DaemonExtraArgs: defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""), - ExtraArgs: defaultEnv("TS_EXTRA_ARGS", ""), - InKubernetes: os.Getenv("KUBERNETES_SERVICE_HOST") != "", - UserspaceMode: defaultBool("TS_USERSPACE", true), - StateDir: defaultEnv("TS_STATE_DIR", ""), - AcceptDNS: defaultEnvBoolPointer("TS_ACCEPT_DNS"), - KubeSecret: defaultEnv("TS_KUBE_SECRET", "tailscale"), - SOCKSProxyAddr: defaultEnv("TS_SOCKS5_SERVER", ""), - HTTPProxyAddr: defaultEnv("TS_OUTBOUND_HTTP_PROXY_LISTEN", ""), - Socket: defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"), - AuthOnce: defaultBool("TS_AUTH_ONCE", false), - Root: defaultEnv("TS_TEST_ONLY_ROOT", "/"), - TailscaledConfigFilePath: defaultEnv("EXPERIMENTAL_TS_CONFIGFILE_PATH", ""), + AuthKey: defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""), + Hostname: defaultEnv("TS_HOSTNAME", ""), + Routes: defaultEnvStringPointer("TS_ROUTES"), + ServeConfigPath: defaultEnv("TS_SERVE_CONFIG", ""), + ProxyTo: defaultEnv("TS_DEST_IP", ""), + TailnetTargetIP: defaultEnv("TS_TAILNET_TARGET_IP", ""), + TailnetTargetFQDN: defaultEnv("TS_TAILNET_TARGET_FQDN", ""), + DaemonExtraArgs: defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""), + ExtraArgs: defaultEnv("TS_EXTRA_ARGS", ""), + InKubernetes: os.Getenv("KUBERNETES_SERVICE_HOST") != "", + UserspaceMode: defaultBool("TS_USERSPACE", true), + StateDir: defaultEnv("TS_STATE_DIR", ""), + AcceptDNS: defaultEnvBoolPointer("TS_ACCEPT_DNS"), + KubeSecret: defaultEnv("TS_KUBE_SECRET", "tailscale"), + SOCKSProxyAddr: defaultEnv("TS_SOCKS5_SERVER", ""), + HTTPProxyAddr: defaultEnv("TS_OUTBOUND_HTTP_PROXY_LISTEN", ""), + Socket: defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"), + AuthOnce: defaultBool("TS_AUTH_ONCE", false), + Root: defaultEnv("TS_TEST_ONLY_ROOT", "/"), + TailscaledConfigFilePath: defaultEnv("EXPERIMENTAL_TS_CONFIGFILE_PATH", ""), + AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false), + PodIP: defaultEnv("POD_IP", ""), } + if err := cfg.validate(); err != nil { log.Fatalf("invalid configuration: %v", err) } @@ -330,7 +341,7 @@ authLoop: } var ( - wantProxy = cfg.ProxyTo != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" + wantProxy = cfg.ProxyTo != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress wantDeviceInfo = cfg.InKubernetes && cfg.KubeSecret != "" && cfg.KubernetesCanPatch startupTasksDone = false currentIPs deephash.Sum // tailscale IPs assigned to device @@ -365,6 +376,7 @@ authLoop: } }() var wg sync.WaitGroup + runLoop: for { select { @@ -451,6 +463,18 @@ runLoop: log.Fatalf("installing egress proxy rules: %v", err) } } + // If this is a L7 cluster ingress proxy (set up + // by Kubernetes operator) and proxying of + // cluster traffic to the ingress target is + // enabled, set up proxy rule each time the + // tailnet IPs of this node change (including + // the first time they become available). + if cfg.AllowProxyingClusterTrafficViaIngress && cfg.ServeConfigPath != "" && ipsHaveChanged && len(addrs) > 0 { + log.Printf("installing rules to forward traffic for %s to node's tailnet IP", cfg.PodIP) + if err := installTSForwardingRuleForDestination(ctx, cfg.PodIP, addrs, nfr); err != nil { + log.Fatalf("installing rules to forward traffic to node's tailnet IP: %v", err) + } + } currentIPs = newCurrentIPs deviceInfo := []any{n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name()} @@ -837,6 +861,35 @@ func installEgressForwardingRule(ctx context.Context, dstStr string, tsIPs []net return nil } +// installTSForwardingRuleForDestination accepts a destination address and a +// list of node's tailnet addresses, sets up rules to forward traffic for +// destination to the tailnet IP matching the destination IP family. +// Destination can be Pod IP of this node. +func installTSForwardingRuleForDestination(ctx context.Context, dstFilter string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error { + dst, err := netip.ParseAddr(dstFilter) + if err != nil { + return err + } + var local netip.Addr + for _, pfx := range tsIPs { + if !pfx.IsSingleIP() { + continue + } + if pfx.Addr().Is4() != dst.Is4() { + continue + } + local = pfx.Addr() + break + } + if !local.IsValid() { + return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstFilter, tsIPs) + } + if err := nfr.AddDNATRule(dst, local); err != nil { + return fmt.Errorf("installing rule for forwarding traffic to tailnet IP: %w", err) + } + return nil +} + func installIngressForwardingRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error { dst, err := netip.ParseAddr(dstStr) if err != nil { @@ -897,6 +950,14 @@ type settings struct { Root string KubernetesCanPatch bool TailscaledConfigFilePath string + // If set to true and, if this containerboot instance is a Kubernetes + // ingress proxy, set up rules to forward incoming cluster traffic to be + // forwarded to the ingress target in cluster. + AllowProxyingClusterTrafficViaIngress bool + // PodIP is the IP of the Pod if running in Kubernetes. This is used + // when setting up rules to proxy cluster traffic to cluster ingress + // target. + PodIP string } func (s *settings) validate() error { @@ -920,6 +981,15 @@ func (s *settings) validate() error { if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") { return errors.New("EXPERIMENTAL_TS_CONFIGFILE_PATH cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.") } + if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode { + return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode") + } + if s.AllowProxyingClusterTrafficViaIngress && s.ServeConfigPath == "" { + return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but this is not a cluster ingress proxy") + } + if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" { + return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set") + } return nil } diff --git a/cmd/k8s-operator/deploy/manifests/proxy.yaml b/cmd/k8s-operator/deploy/manifests/proxy.yaml index ff9a44973..daa795b3e 100644 --- a/cmd/k8s-operator/deploy/manifests/proxy.yaml +++ b/cmd/k8s-operator/deploy/manifests/proxy.yaml @@ -30,6 +30,10 @@ spec: value: "false" - name: TS_AUTH_ONCE value: "true" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP securityContext: capabilities: add: diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index af09be075..f0c176924 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -255,6 +255,10 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga ChildResourceLabels: crl, } + if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" { + sts.ForwardClusterTrafficViaL7IngressProxy = true + } + if _, err := a.ssr.Provision(ctx, logger, sts); err != nil { return fmt.Errorf("failed to provision: %w", err) } diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go new file mode 100644 index 000000000..3a87e5184 --- /dev/null +++ b/cmd/k8s-operator/ingress_test.go @@ -0,0 +1,139 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "testing" + + "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "tailscale.com/ipn" + "tailscale.com/types/ptr" + "tailscale.com/util/mak" +) + +func TestTailscaleIngress(t *testing.T) { + tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}} + fc := fake.NewFakeClient(tsIngressClass) + ft := &fakeTSClient{} + fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}} + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + ingR := &IngressReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: ft, + tsnetServer: fakeTsnetServer, + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + logger: zl.Sugar(), + } + + // 1. Resources get created for regular Ingress + ing := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + // The apiserver is supposed to set the UID, but the fake client + // doesn't. So, set it explicitly because other code later depends + // on it being set. + UID: types.UID("1234-UID"), + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: ptr.To("tailscale"), + DefaultBackend: &networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "test", + Port: networkingv1.ServiceBackendPort{ + Number: 8080, + }, + }, + }, + TLS: []networkingv1.IngressTLS{ + {Hosts: []string{"default-test"}}, + }, + }, + } + mustCreate(t, fc, ing) + mustCreate(t, fc, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "1.2.3.4", + Ports: []corev1.ServicePort{{ + Port: 8080, + Name: "http"}, + }, + }, + }) + + expectReconciled(t, ingR, "default", "test") + + fullName, shortName := findGenName(t, fc, "default", "test", "ingress") + opts := configOpts{ + stsName: shortName, + secretName: fullName, + namespace: "default", + parentType: "ingress", + hostname: "default-test", + } + serveConfig := &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}}, + } + opts.serveConfig = serveConfig + + expectEqual(t, fc, expectedSecret(t, opts)) + expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) + expectEqual(t, fc, expectedSTSUserspace(opts)) + + // 2. Ingress status gets updated with ingress proxy's MagicDNS name + // once that becomes available. + mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) { + mak.Set(&secret.Data, "device_id", []byte("1234")) + mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net")) + }) + expectReconciled(t, ingR, "default", "test") + ing.Finalizers = append(ing.Finalizers, "tailscale.com/finalizer") + ing.Status.LoadBalancer = networkingv1.IngressLoadBalancerStatus{ + Ingress: []networkingv1.IngressLoadBalancerIngress{ + {Hostname: "foo.tailnetxyz.ts.net", Ports: []networkingv1.IngressPortStatus{{Port: 443, Protocol: "TCP"}}}, + }, + } + expectEqual(t, fc, ing) + + // 3. Resources get created for Ingress that should allow forwarding + // cluster traffic + mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { + mak.Set(&ing.ObjectMeta.Annotations, AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy, "true") + }) + opts.shouldEnableForwardingClusterTrafficViaIngress = true + expectReconciled(t, ingR, "default", "test") + expectEqual(t, fc, expectedSTS(opts)) + + // 4. Resources get cleaned up when Ingress class is unset + mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { + ing.Spec.IngressClassName = ptr.To("nginx") + }) + expectReconciled(t, ingR, "default", "test") + expectReconciled(t, ingR, "default", "test") // deleting Ingress STS requires two reconciles + expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName) + expectMissing[corev1.Service](t, fc, "operator-ns", shortName) + expectMissing[corev1.Secret](t, fc, "operator-ns", fullName) +} diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index 06adfe4cb..942147477 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -68,7 +68,7 @@ func TestLoadBalancerClass(t *testing.T) { } expectEqual(t, fc, expectedSecret(t, opts)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(opts)) // Normally the Tailscale proxy pod would come up here and write its info @@ -209,7 +209,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { } expectEqual(t, fc, expectedSecret(t, o)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) want := &corev1.Service{ TypeMeta: metav1.TypeMeta{ @@ -233,7 +233,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { } expectEqual(t, fc, want) expectEqual(t, fc, expectedSecret(t, o)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) // Change the tailscale-target-fqdn annotation which should update the @@ -319,7 +319,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { } expectEqual(t, fc, expectedSecret(t, o)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) want := &corev1.Service{ TypeMeta: metav1.TypeMeta{ @@ -343,7 +343,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { } expectEqual(t, fc, want) expectEqual(t, fc, expectedSecret(t, o)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) // Change the tailscale-target-ip annotation which should update the @@ -426,7 +426,7 @@ func TestAnnotations(t *testing.T) { } expectEqual(t, fc, expectedSecret(t, o)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) want := &corev1.Service{ TypeMeta: metav1.TypeMeta{ @@ -534,7 +534,7 @@ func TestAnnotationIntoLB(t *testing.T) { } expectEqual(t, fc, expectedSecret(t, o)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) // Normally the Tailscale proxy pod would come up here and write its info @@ -579,7 +579,7 @@ func TestAnnotationIntoLB(t *testing.T) { }) expectReconciled(t, sr, "default", "test") // None of the proxy machinery should have changed... - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) // ... but the service should have a LoadBalancer status. @@ -665,7 +665,7 @@ func TestLBIntoAnnotation(t *testing.T) { } expectEqual(t, fc, expectedSecret(t, o)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) // Normally the Tailscale proxy pod would come up here and write its info @@ -728,7 +728,7 @@ func TestLBIntoAnnotation(t *testing.T) { }) expectReconciled(t, sr, "default", "test") - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) want = &corev1.Service{ @@ -806,7 +806,7 @@ func TestCustomHostname(t *testing.T) { } expectEqual(t, fc, expectedSecret(t, o)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) want := &corev1.Service{ TypeMeta: metav1.TypeMeta{ @@ -964,7 +964,7 @@ func TestDefaultLoadBalancer(t *testing.T) { fullName, shortName := findGenName(t, fc, "default", "test", "svc") - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) o := configOpts{ stsName: shortName, secretName: fullName, diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 1c8063521..cd1a585ca 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -29,7 +29,6 @@ import ( "tailscale.com/ipn" "tailscale.com/net/netutil" "tailscale.com/tailcfg" - "tailscale.com/tsnet" "tailscale.com/types/opt" "tailscale.com/util/dnsname" "tailscale.com/util/mak" @@ -55,6 +54,19 @@ const ( // Annotations settable by users on ingresses. AnnotationFunnel = "tailscale.com/funnel" + // If set to true, set up iptables/nftables rules in the proxy forward + // cluster traffic to the tailnet IP of that proxy. This can only be set + // on an Ingress. This is useful in cases where a cluster target needs + // to be able to reach a cluster workload exposed to tailnet via Ingress + // using the same hostname as a tailnet workload (in this case, the + // MagicDNS name of the ingress proxy). This annotation is experimental. + // If it is set to true, the proxy set up for Ingress, will run + // tailscale in non-userspace, with NET_ADMIN cap for tailscale + // container and will also run a privileged init container that enables + // forwarding. + // Eventually this behaviour might become the default. + AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy = "tailscale.com/experimental-forward-cluster-traffic-via-ingress" + // Annotations set by the operator on pods to trigger restarts when the // hostname, IP, FQDN or tailscaled config changes. podAnnotationLastSetClusterIP = "tailscale.com/operator-last-set-cluster-ip" @@ -74,8 +86,11 @@ type tailscaleSTSConfig struct { ParentResourceUID string ChildResourceLabels map[string]string - ServeConfig *ipn.ServeConfig - ClusterTargetIP string // ingress target + ServeConfig *ipn.ServeConfig // if serve config is set, this is a proxy for Ingress + ClusterTargetIP string // ingress target + // If set to true, operator should configure containerboot to forward + // cluster traffic via the proxy set up for Kubernetes Ingress. + ForwardClusterTrafficViaL7IngressProxy bool TailnetTargetIP string // egress target IP @@ -95,10 +110,13 @@ type connector struct { // isExitNode defines whether this Connector should act as an exit node. isExitNode bool } +type tsnetServer interface { + CertDomains() []string +} type tailscaleSTSReconciler struct { client.Client - tsnetServer *tsnet.Server + tsnetServer tsnetServer tsClient tsClient defaultTags []string operatorNamespace string @@ -381,7 +399,7 @@ var userspaceProxyYaml []byte func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string) (*appsv1.StatefulSet, error) { var ss appsv1.StatefulSet - if sts.ServeConfig != nil { + if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil { return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err) } @@ -422,6 +440,12 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S Value: proxySecret, }, ) + if sts.ForwardClusterTrafficViaL7IngressProxy { + container.Env = append(container.Env, corev1.EnvVar{ + Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", + Value: "true", + }) + } if !shouldDoTailscaledDeclarativeConfig(sts) { container.Env = append(container.Env, corev1.EnvVar{ Name: "TS_HOSTNAME", diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index a5aea24cd..5d025539e 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -31,20 +31,22 @@ import ( // confgOpts contains configuration options for creating cluster resources for // Tailscale proxies. type configOpts struct { - stsName string - secretName string - hostname string - namespace string - parentType string - priorityClassName string - firewallMode string - tailnetTargetIP string - tailnetTargetFQDN string - clusterTargetIP string - subnetRoutes string - isExitNode bool - shouldUseDeclarativeConfig bool // tailscaled in proxy should be configured using config file - confFileHash string + stsName string + secretName string + hostname string + namespace string + parentType string + priorityClassName string + firewallMode string + tailnetTargetIP string + tailnetTargetFQDN string + clusterTargetIP string + subnetRoutes string + isExitNode bool + shouldUseDeclarativeConfig bool // tailscaled in proxy should be configured using config file + confFileHash string + serveConfig *ipn.ServeConfig + shouldEnableForwardingClusterTrafficViaIngress bool } func expectedSTS(opts configOpts) *appsv1.StatefulSet { @@ -54,6 +56,7 @@ func expectedSTS(opts configOpts) *appsv1.StatefulSet { Env: []corev1.EnvVar{ {Name: "TS_USERSPACE", Value: "false"}, {Name: "TS_AUTH_ONCE", Value: "true"}, + {Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, {Name: "TS_KUBE_SECRET", Value: opts.secretName}, }, SecurityContext: &corev1.SecurityContext{ @@ -63,6 +66,12 @@ func expectedSTS(opts configOpts) *appsv1.StatefulSet { }, ImagePullPolicy: "Always", } + if opts.shouldEnableForwardingClusterTrafficViaIngress { + tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{ + Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", + Value: "true", + }) + } annots := make(map[string]string) var volumes []corev1.Volume if opts.shouldUseDeclarativeConfig { @@ -122,6 +131,16 @@ func expectedSTS(opts configOpts) *appsv1.StatefulSet { }) annots["tailscale.com/operator-last-set-cluster-ip"] = opts.clusterTargetIP } + if opts.serveConfig != nil { + tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{ + Name: "TS_SERVE_CONFIG", + Value: "/etc/tailscaled/serve-config", + }) + volumes = append(volumes, corev1.Volume{ + Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Path: "serve-config", Key: "serve-config"}}}}, + }) + tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"}) + } return &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ Kind: "StatefulSet", @@ -177,7 +196,68 @@ func expectedSTS(opts configOpts) *appsv1.StatefulSet { } } -func expectedHeadlessService(name string) *corev1.Service { +func expectedSTSUserspace(opts configOpts) *appsv1.StatefulSet { + tsContainer := corev1.Container{ + Name: "tailscale", + Image: "tailscale/tailscale", + Env: []corev1.EnvVar{ + {Name: "TS_USERSPACE", Value: "true"}, + {Name: "TS_AUTH_ONCE", Value: "true"}, + {Name: "TS_KUBE_SECRET", Value: opts.secretName}, + {Name: "TS_HOSTNAME", Value: opts.hostname}, + {Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"}, + }, + ImagePullPolicy: "Always", + VolumeMounts: []corev1.VolumeMount{{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"}}, + } + annots := make(map[string]string) + volumes := []corev1.Volume{{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}}} + annots["tailscale.com/operator-last-set-hostname"] = opts.hostname + return &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: opts.stsName, + Namespace: "operator-ns", + Labels: map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource": "test", + "tailscale.com/parent-resource-ns": opts.namespace, + "tailscale.com/parent-resource-type": opts.parentType, + }, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To[int32](1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "1234-UID"}, + }, + ServiceName: opts.stsName, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annots, + DeletionGracePeriodSeconds: ptr.To[int64](10), + Labels: map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource": "test", + "tailscale.com/parent-resource-ns": opts.namespace, + "tailscale.com/parent-resource-type": opts.parentType, + "app": "1234-UID", + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "proxies", + PriorityClassName: opts.priorityClassName, + Containers: []corev1.Container{tsContainer}, + Volumes: volumes, + }, + }, + }, + } +} + +func expectedHeadlessService(name string, parentType string) *corev1.Service { return &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -191,7 +271,7 @@ func expectedHeadlessService(name string) *corev1.Service { "tailscale.com/managed": "true", "tailscale.com/parent-resource": "test", "tailscale.com/parent-resource-ns": "default", - "tailscale.com/parent-resource-type": "svc", + "tailscale.com/parent-resource-type": parentType, }, }, Spec: corev1.ServiceSpec{ @@ -220,6 +300,13 @@ func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret { Namespace: "operator-ns", }, } + if opts.serveConfig != nil { + serveConfigBs, err := json.Marshal(opts.serveConfig) + if err != nil { + t.Fatalf("error marshalling serve config: %v", err) + } + mak.Set(&s.StringData, "serve-config", string(serveConfigBs)) + } if !opts.shouldUseDeclarativeConfig { mak.Set(&s.StringData, "authkey", "secret-authkey") labels["tailscale.com/parent-resource-ns"] = opts.namespace @@ -384,6 +471,13 @@ type fakeTSClient struct { keyRequests []tailscale.KeyCapabilities deleted []string } +type fakeTSNetServer struct { + certDomains []string +} + +func (f *fakeTSNetServer) CertDomains() []string { + return f.certDomains +} func (c *fakeTSClient) CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error) { c.Lock()