diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 2dffcfb4b..5d14826d8 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -55,6 +55,15 @@ // and not `tailscale up` or `tailscale set`. // The config file contents are currently read once on container start. // NB: This env var is currently experimental and the logic will likely change! +// - EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS: if set to true +// and if this containerboot instance is an L7 ingress proxy (created by +// the Kubernetes operator), set up rules to allow proxying cluster traffic, +// received on the Pod IP of this node, to the ingress target in the cluster. +// This, in conjunction with MagicDNS name resolution in cluster, can be +// useful for cases where a cluster workload needs to access a target in +// cluster using the same hostname (in this case, the MagicDNS name of the ingress proxy) +// as a non-cluster workload on tailnet. +// This is only meant to be configured by the Kubernetes operator. // // When running on Kubernetes, containerboot defaults to storing state in the // "tailscale" kube secret. To store state on local disk instead, set @@ -108,29 +117,31 @@ func newNetfilterRunner(logf logger.Logf) (linuxfw.NetfilterRunner, error) { func main() { log.SetPrefix("boot: ") tailscale.I_Acknowledge_This_API_Is_Unstable = true - cfg := &settings{ - AuthKey: defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""), - Hostname: defaultEnv("TS_HOSTNAME", ""), - Routes: defaultEnvStringPointer("TS_ROUTES"), - ServeConfigPath: defaultEnv("TS_SERVE_CONFIG", ""), - ProxyTo: defaultEnv("TS_DEST_IP", ""), - TailnetTargetIP: defaultEnv("TS_TAILNET_TARGET_IP", ""), - TailnetTargetFQDN: defaultEnv("TS_TAILNET_TARGET_FQDN", ""), - DaemonExtraArgs: defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""), - ExtraArgs: defaultEnv("TS_EXTRA_ARGS", ""), - InKubernetes: os.Getenv("KUBERNETES_SERVICE_HOST") != "", - UserspaceMode: defaultBool("TS_USERSPACE", true), - StateDir: defaultEnv("TS_STATE_DIR", ""), - AcceptDNS: defaultEnvBoolPointer("TS_ACCEPT_DNS"), - KubeSecret: defaultEnv("TS_KUBE_SECRET", "tailscale"), - SOCKSProxyAddr: defaultEnv("TS_SOCKS5_SERVER", ""), - HTTPProxyAddr: defaultEnv("TS_OUTBOUND_HTTP_PROXY_LISTEN", ""), - Socket: defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"), - AuthOnce: defaultBool("TS_AUTH_ONCE", false), - Root: defaultEnv("TS_TEST_ONLY_ROOT", "/"), - TailscaledConfigFilePath: defaultEnv("EXPERIMENTAL_TS_CONFIGFILE_PATH", ""), + AuthKey: defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""), + Hostname: defaultEnv("TS_HOSTNAME", ""), + Routes: defaultEnvStringPointer("TS_ROUTES"), + ServeConfigPath: defaultEnv("TS_SERVE_CONFIG", ""), + ProxyTo: defaultEnv("TS_DEST_IP", ""), + TailnetTargetIP: defaultEnv("TS_TAILNET_TARGET_IP", ""), + TailnetTargetFQDN: defaultEnv("TS_TAILNET_TARGET_FQDN", ""), + DaemonExtraArgs: defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""), + ExtraArgs: defaultEnv("TS_EXTRA_ARGS", ""), + InKubernetes: os.Getenv("KUBERNETES_SERVICE_HOST") != "", + UserspaceMode: defaultBool("TS_USERSPACE", true), + StateDir: defaultEnv("TS_STATE_DIR", ""), + AcceptDNS: defaultEnvBoolPointer("TS_ACCEPT_DNS"), + KubeSecret: defaultEnv("TS_KUBE_SECRET", "tailscale"), + SOCKSProxyAddr: defaultEnv("TS_SOCKS5_SERVER", ""), + HTTPProxyAddr: defaultEnv("TS_OUTBOUND_HTTP_PROXY_LISTEN", ""), + Socket: defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"), + AuthOnce: defaultBool("TS_AUTH_ONCE", false), + Root: defaultEnv("TS_TEST_ONLY_ROOT", "/"), + TailscaledConfigFilePath: defaultEnv("EXPERIMENTAL_TS_CONFIGFILE_PATH", ""), + AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false), + PodIP: defaultEnv("POD_IP", ""), } + if err := cfg.validate(); err != nil { log.Fatalf("invalid configuration: %v", err) } @@ -330,7 +341,7 @@ authLoop: } var ( - wantProxy = cfg.ProxyTo != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" + wantProxy = cfg.ProxyTo != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress wantDeviceInfo = cfg.InKubernetes && cfg.KubeSecret != "" && cfg.KubernetesCanPatch startupTasksDone = false currentIPs deephash.Sum // tailscale IPs assigned to device @@ -365,6 +376,7 @@ authLoop: } }() var wg sync.WaitGroup + runLoop: for { select { @@ -451,6 +463,18 @@ runLoop: log.Fatalf("installing egress proxy rules: %v", err) } } + // If this is a L7 cluster ingress proxy (set up + // by Kubernetes operator) and proxying of + // cluster traffic to the ingress target is + // enabled, set up proxy rule each time the + // tailnet IPs of this node change (including + // the first time they become available). + if cfg.AllowProxyingClusterTrafficViaIngress && cfg.ServeConfigPath != "" && ipsHaveChanged && len(addrs) > 0 { + log.Printf("installing rules to forward traffic for %s to node's tailnet IP", cfg.PodIP) + if err := installTSForwardingRuleForDestination(ctx, cfg.PodIP, addrs, nfr); err != nil { + log.Fatalf("installing rules to forward traffic to node's tailnet IP: %v", err) + } + } currentIPs = newCurrentIPs deviceInfo := []any{n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name()} @@ -837,6 +861,35 @@ func installEgressForwardingRule(ctx context.Context, dstStr string, tsIPs []net return nil } +// installTSForwardingRuleForDestination accepts a destination address and a +// list of node's tailnet addresses, sets up rules to forward traffic for +// destination to the tailnet IP matching the destination IP family. +// Destination can be Pod IP of this node. +func installTSForwardingRuleForDestination(ctx context.Context, dstFilter string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error { + dst, err := netip.ParseAddr(dstFilter) + if err != nil { + return err + } + var local netip.Addr + for _, pfx := range tsIPs { + if !pfx.IsSingleIP() { + continue + } + if pfx.Addr().Is4() != dst.Is4() { + continue + } + local = pfx.Addr() + break + } + if !local.IsValid() { + return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstFilter, tsIPs) + } + if err := nfr.AddDNATRule(dst, local); err != nil { + return fmt.Errorf("installing rule for forwarding traffic to tailnet IP: %w", err) + } + return nil +} + func installIngressForwardingRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error { dst, err := netip.ParseAddr(dstStr) if err != nil { @@ -897,6 +950,14 @@ type settings struct { Root string KubernetesCanPatch bool TailscaledConfigFilePath string + // If set to true and, if this containerboot instance is a Kubernetes + // ingress proxy, set up rules to forward incoming cluster traffic to be + // forwarded to the ingress target in cluster. + AllowProxyingClusterTrafficViaIngress bool + // PodIP is the IP of the Pod if running in Kubernetes. This is used + // when setting up rules to proxy cluster traffic to cluster ingress + // target. + PodIP string } func (s *settings) validate() error { @@ -920,6 +981,15 @@ func (s *settings) validate() error { if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") { return errors.New("EXPERIMENTAL_TS_CONFIGFILE_PATH cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.") } + if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode { + return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode") + } + if s.AllowProxyingClusterTrafficViaIngress && s.ServeConfigPath == "" { + return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but this is not a cluster ingress proxy") + } + if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" { + return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set") + } return nil } diff --git a/cmd/k8s-operator/deploy/manifests/proxy.yaml b/cmd/k8s-operator/deploy/manifests/proxy.yaml index ff9a44973..daa795b3e 100644 --- a/cmd/k8s-operator/deploy/manifests/proxy.yaml +++ b/cmd/k8s-operator/deploy/manifests/proxy.yaml @@ -30,6 +30,10 @@ spec: value: "false" - name: TS_AUTH_ONCE value: "true" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP securityContext: capabilities: add: diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index af09be075..f0c176924 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -255,6 +255,10 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga ChildResourceLabels: crl, } + if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" { + sts.ForwardClusterTrafficViaL7IngressProxy = true + } + if _, err := a.ssr.Provision(ctx, logger, sts); err != nil { return fmt.Errorf("failed to provision: %w", err) } diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go new file mode 100644 index 000000000..3a87e5184 --- /dev/null +++ b/cmd/k8s-operator/ingress_test.go @@ -0,0 +1,139 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "testing" + + "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "tailscale.com/ipn" + "tailscale.com/types/ptr" + "tailscale.com/util/mak" +) + +func TestTailscaleIngress(t *testing.T) { + tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}} + fc := fake.NewFakeClient(tsIngressClass) + ft := &fakeTSClient{} + fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}} + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + ingR := &IngressReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: ft, + tsnetServer: fakeTsnetServer, + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + logger: zl.Sugar(), + } + + // 1. Resources get created for regular Ingress + ing := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + // The apiserver is supposed to set the UID, but the fake client + // doesn't. So, set it explicitly because other code later depends + // on it being set. + UID: types.UID("1234-UID"), + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: ptr.To("tailscale"), + DefaultBackend: &networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "test", + Port: networkingv1.ServiceBackendPort{ + Number: 8080, + }, + }, + }, + TLS: []networkingv1.IngressTLS{ + {Hosts: []string{"default-test"}}, + }, + }, + } + mustCreate(t, fc, ing) + mustCreate(t, fc, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "1.2.3.4", + Ports: []corev1.ServicePort{{ + Port: 8080, + Name: "http"}, + }, + }, + }) + + expectReconciled(t, ingR, "default", "test") + + fullName, shortName := findGenName(t, fc, "default", "test", "ingress") + opts := configOpts{ + stsName: shortName, + secretName: fullName, + namespace: "default", + parentType: "ingress", + hostname: "default-test", + } + serveConfig := &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}}, + } + opts.serveConfig = serveConfig + + expectEqual(t, fc, expectedSecret(t, opts)) + expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) + expectEqual(t, fc, expectedSTSUserspace(opts)) + + // 2. Ingress status gets updated with ingress proxy's MagicDNS name + // once that becomes available. + mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) { + mak.Set(&secret.Data, "device_id", []byte("1234")) + mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net")) + }) + expectReconciled(t, ingR, "default", "test") + ing.Finalizers = append(ing.Finalizers, "tailscale.com/finalizer") + ing.Status.LoadBalancer = networkingv1.IngressLoadBalancerStatus{ + Ingress: []networkingv1.IngressLoadBalancerIngress{ + {Hostname: "foo.tailnetxyz.ts.net", Ports: []networkingv1.IngressPortStatus{{Port: 443, Protocol: "TCP"}}}, + }, + } + expectEqual(t, fc, ing) + + // 3. Resources get created for Ingress that should allow forwarding + // cluster traffic + mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { + mak.Set(&ing.ObjectMeta.Annotations, AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy, "true") + }) + opts.shouldEnableForwardingClusterTrafficViaIngress = true + expectReconciled(t, ingR, "default", "test") + expectEqual(t, fc, expectedSTS(opts)) + + // 4. Resources get cleaned up when Ingress class is unset + mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { + ing.Spec.IngressClassName = ptr.To("nginx") + }) + expectReconciled(t, ingR, "default", "test") + expectReconciled(t, ingR, "default", "test") // deleting Ingress STS requires two reconciles + expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName) + expectMissing[corev1.Service](t, fc, "operator-ns", shortName) + expectMissing[corev1.Secret](t, fc, "operator-ns", fullName) +} diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index 06adfe4cb..942147477 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -68,7 +68,7 @@ func TestLoadBalancerClass(t *testing.T) { } expectEqual(t, fc, expectedSecret(t, opts)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(opts)) // Normally the Tailscale proxy pod would come up here and write its info @@ -209,7 +209,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { } expectEqual(t, fc, expectedSecret(t, o)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) want := &corev1.Service{ TypeMeta: metav1.TypeMeta{ @@ -233,7 +233,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { } expectEqual(t, fc, want) expectEqual(t, fc, expectedSecret(t, o)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) // Change the tailscale-target-fqdn annotation which should update the @@ -319,7 +319,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { } expectEqual(t, fc, expectedSecret(t, o)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) want := &corev1.Service{ TypeMeta: metav1.TypeMeta{ @@ -343,7 +343,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { } expectEqual(t, fc, want) expectEqual(t, fc, expectedSecret(t, o)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) // Change the tailscale-target-ip annotation which should update the @@ -426,7 +426,7 @@ func TestAnnotations(t *testing.T) { } expectEqual(t, fc, expectedSecret(t, o)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) want := &corev1.Service{ TypeMeta: metav1.TypeMeta{ @@ -534,7 +534,7 @@ func TestAnnotationIntoLB(t *testing.T) { } expectEqual(t, fc, expectedSecret(t, o)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) // Normally the Tailscale proxy pod would come up here and write its info @@ -579,7 +579,7 @@ func TestAnnotationIntoLB(t *testing.T) { }) expectReconciled(t, sr, "default", "test") // None of the proxy machinery should have changed... - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) // ... but the service should have a LoadBalancer status. @@ -665,7 +665,7 @@ func TestLBIntoAnnotation(t *testing.T) { } expectEqual(t, fc, expectedSecret(t, o)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) // Normally the Tailscale proxy pod would come up here and write its info @@ -728,7 +728,7 @@ func TestLBIntoAnnotation(t *testing.T) { }) expectReconciled(t, sr, "default", "test") - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) want = &corev1.Service{ @@ -806,7 +806,7 @@ func TestCustomHostname(t *testing.T) { } expectEqual(t, fc, expectedSecret(t, o)) - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(o)) want := &corev1.Service{ TypeMeta: metav1.TypeMeta{ @@ -964,7 +964,7 @@ func TestDefaultLoadBalancer(t *testing.T) { fullName, shortName := findGenName(t, fc, "default", "test", "svc") - expectEqual(t, fc, expectedHeadlessService(shortName)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) o := configOpts{ stsName: shortName, secretName: fullName, diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 1c8063521..cd1a585ca 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -29,7 +29,6 @@ import ( "tailscale.com/ipn" "tailscale.com/net/netutil" "tailscale.com/tailcfg" - "tailscale.com/tsnet" "tailscale.com/types/opt" "tailscale.com/util/dnsname" "tailscale.com/util/mak" @@ -55,6 +54,19 @@ const ( // Annotations settable by users on ingresses. AnnotationFunnel = "tailscale.com/funnel" + // If set to true, set up iptables/nftables rules in the proxy forward + // cluster traffic to the tailnet IP of that proxy. This can only be set + // on an Ingress. This is useful in cases where a cluster target needs + // to be able to reach a cluster workload exposed to tailnet via Ingress + // using the same hostname as a tailnet workload (in this case, the + // MagicDNS name of the ingress proxy). This annotation is experimental. + // If it is set to true, the proxy set up for Ingress, will run + // tailscale in non-userspace, with NET_ADMIN cap for tailscale + // container and will also run a privileged init container that enables + // forwarding. + // Eventually this behaviour might become the default. + AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy = "tailscale.com/experimental-forward-cluster-traffic-via-ingress" + // Annotations set by the operator on pods to trigger restarts when the // hostname, IP, FQDN or tailscaled config changes. podAnnotationLastSetClusterIP = "tailscale.com/operator-last-set-cluster-ip" @@ -74,8 +86,11 @@ type tailscaleSTSConfig struct { ParentResourceUID string ChildResourceLabels map[string]string - ServeConfig *ipn.ServeConfig - ClusterTargetIP string // ingress target + ServeConfig *ipn.ServeConfig // if serve config is set, this is a proxy for Ingress + ClusterTargetIP string // ingress target + // If set to true, operator should configure containerboot to forward + // cluster traffic via the proxy set up for Kubernetes Ingress. + ForwardClusterTrafficViaL7IngressProxy bool TailnetTargetIP string // egress target IP @@ -95,10 +110,13 @@ type connector struct { // isExitNode defines whether this Connector should act as an exit node. isExitNode bool } +type tsnetServer interface { + CertDomains() []string +} type tailscaleSTSReconciler struct { client.Client - tsnetServer *tsnet.Server + tsnetServer tsnetServer tsClient tsClient defaultTags []string operatorNamespace string @@ -381,7 +399,7 @@ var userspaceProxyYaml []byte func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string) (*appsv1.StatefulSet, error) { var ss appsv1.StatefulSet - if sts.ServeConfig != nil { + if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil { return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err) } @@ -422,6 +440,12 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S Value: proxySecret, }, ) + if sts.ForwardClusterTrafficViaL7IngressProxy { + container.Env = append(container.Env, corev1.EnvVar{ + Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", + Value: "true", + }) + } if !shouldDoTailscaledDeclarativeConfig(sts) { container.Env = append(container.Env, corev1.EnvVar{ Name: "TS_HOSTNAME", diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index a5aea24cd..5d025539e 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -31,20 +31,22 @@ import ( // confgOpts contains configuration options for creating cluster resources for // Tailscale proxies. type configOpts struct { - stsName string - secretName string - hostname string - namespace string - parentType string - priorityClassName string - firewallMode string - tailnetTargetIP string - tailnetTargetFQDN string - clusterTargetIP string - subnetRoutes string - isExitNode bool - shouldUseDeclarativeConfig bool // tailscaled in proxy should be configured using config file - confFileHash string + stsName string + secretName string + hostname string + namespace string + parentType string + priorityClassName string + firewallMode string + tailnetTargetIP string + tailnetTargetFQDN string + clusterTargetIP string + subnetRoutes string + isExitNode bool + shouldUseDeclarativeConfig bool // tailscaled in proxy should be configured using config file + confFileHash string + serveConfig *ipn.ServeConfig + shouldEnableForwardingClusterTrafficViaIngress bool } func expectedSTS(opts configOpts) *appsv1.StatefulSet { @@ -54,6 +56,7 @@ func expectedSTS(opts configOpts) *appsv1.StatefulSet { Env: []corev1.EnvVar{ {Name: "TS_USERSPACE", Value: "false"}, {Name: "TS_AUTH_ONCE", Value: "true"}, + {Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, {Name: "TS_KUBE_SECRET", Value: opts.secretName}, }, SecurityContext: &corev1.SecurityContext{ @@ -63,6 +66,12 @@ func expectedSTS(opts configOpts) *appsv1.StatefulSet { }, ImagePullPolicy: "Always", } + if opts.shouldEnableForwardingClusterTrafficViaIngress { + tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{ + Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", + Value: "true", + }) + } annots := make(map[string]string) var volumes []corev1.Volume if opts.shouldUseDeclarativeConfig { @@ -122,6 +131,16 @@ func expectedSTS(opts configOpts) *appsv1.StatefulSet { }) annots["tailscale.com/operator-last-set-cluster-ip"] = opts.clusterTargetIP } + if opts.serveConfig != nil { + tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{ + Name: "TS_SERVE_CONFIG", + Value: "/etc/tailscaled/serve-config", + }) + volumes = append(volumes, corev1.Volume{ + Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Path: "serve-config", Key: "serve-config"}}}}, + }) + tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"}) + } return &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ Kind: "StatefulSet", @@ -177,7 +196,68 @@ func expectedSTS(opts configOpts) *appsv1.StatefulSet { } } -func expectedHeadlessService(name string) *corev1.Service { +func expectedSTSUserspace(opts configOpts) *appsv1.StatefulSet { + tsContainer := corev1.Container{ + Name: "tailscale", + Image: "tailscale/tailscale", + Env: []corev1.EnvVar{ + {Name: "TS_USERSPACE", Value: "true"}, + {Name: "TS_AUTH_ONCE", Value: "true"}, + {Name: "TS_KUBE_SECRET", Value: opts.secretName}, + {Name: "TS_HOSTNAME", Value: opts.hostname}, + {Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"}, + }, + ImagePullPolicy: "Always", + VolumeMounts: []corev1.VolumeMount{{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"}}, + } + annots := make(map[string]string) + volumes := []corev1.Volume{{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}}} + annots["tailscale.com/operator-last-set-hostname"] = opts.hostname + return &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: opts.stsName, + Namespace: "operator-ns", + Labels: map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource": "test", + "tailscale.com/parent-resource-ns": opts.namespace, + "tailscale.com/parent-resource-type": opts.parentType, + }, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To[int32](1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "1234-UID"}, + }, + ServiceName: opts.stsName, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annots, + DeletionGracePeriodSeconds: ptr.To[int64](10), + Labels: map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource": "test", + "tailscale.com/parent-resource-ns": opts.namespace, + "tailscale.com/parent-resource-type": opts.parentType, + "app": "1234-UID", + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "proxies", + PriorityClassName: opts.priorityClassName, + Containers: []corev1.Container{tsContainer}, + Volumes: volumes, + }, + }, + }, + } +} + +func expectedHeadlessService(name string, parentType string) *corev1.Service { return &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -191,7 +271,7 @@ func expectedHeadlessService(name string) *corev1.Service { "tailscale.com/managed": "true", "tailscale.com/parent-resource": "test", "tailscale.com/parent-resource-ns": "default", - "tailscale.com/parent-resource-type": "svc", + "tailscale.com/parent-resource-type": parentType, }, }, Spec: corev1.ServiceSpec{ @@ -220,6 +300,13 @@ func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret { Namespace: "operator-ns", }, } + if opts.serveConfig != nil { + serveConfigBs, err := json.Marshal(opts.serveConfig) + if err != nil { + t.Fatalf("error marshalling serve config: %v", err) + } + mak.Set(&s.StringData, "serve-config", string(serveConfigBs)) + } if !opts.shouldUseDeclarativeConfig { mak.Set(&s.StringData, "authkey", "secret-authkey") labels["tailscale.com/parent-resource-ns"] = opts.namespace @@ -384,6 +471,13 @@ type fakeTSClient struct { keyRequests []tailscale.KeyCapabilities deleted []string } +type fakeTSNetServer struct { + certDomains []string +} + +func (f *fakeTSNetServer) CertDomains() []string { + return f.certDomains +} func (c *fakeTSClient) CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error) { c.Lock()