From d481ec4560653035f9d20c87b34763139dfbe3ff Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Tue, 27 Feb 2024 22:27:33 +0000 Subject: [PATCH] cmd/k8s-operator,k8s-operator: allow to optionally configure proxies to --accept-routes A new ProxyClass.tailscaledConfig.acceptRoutes field (defaults to false) can be used to configure proxies created by the operator to be ran with --accept-routes via the declarative config. Updates tailscale/tailscale#10684 Signed-off-by: Irbe Krumina --- cmd/k8s-operator/connector_test.go | 18 ++-- .../crds/tailscale.com_proxyclasses.yaml | 14 ++- .../deploy/manifests/operator.yaml | 14 ++- cmd/k8s-operator/ingress_test.go | 8 +- cmd/k8s-operator/operator_test.go | 100 +++++++++++++++--- cmd/k8s-operator/sts.go | 57 +++++----- cmd/k8s-operator/testutils_test.go | 13 ++- k8s-operator/api.md | 40 ++++++- .../apis/v1alpha1/types_proxyclass.go | 22 +++- .../apis/v1alpha1/zz_generated.deepcopy.go | 20 ++++ 10 files changed, 239 insertions(+), 67 deletions(-) diff --git a/cmd/k8s-operator/connector_test.go b/cmd/k8s-operator/connector_test.go index 7e8e9599f..cdc9de665 100644 --- a/cmd/k8s-operator/connector_test.go +++ b/cmd/k8s-operator/connector_test.go @@ -73,7 +73,7 @@ func TestConnector(t *testing.T) { hostname: "test-connector", isExitNode: true, subnetRoutes: "10.40.0.0/14", - confFileHash: "9321660203effb80983eaecc7b5ac5a8c53934926f46e895b9fe295dcfc5a904", + confFileHash: "9c66c269f3aa779e4e63ca414c147fd419caa62f9220a535da619f21b6bdf2b9", } expectEqual(t, fc, expectedSecret(t, opts)) expectEqual(t, fc, expectedSTS(t, fc, opts)) @@ -83,7 +83,7 @@ func TestConnector(t *testing.T) { conn.Spec.SubnetRouter.AdvertiseRoutes = []tsapi.Route{"10.40.0.0/14", "10.44.0.0/20"} }) opts.subnetRoutes = "10.40.0.0/14,10.44.0.0/20" - opts.confFileHash = "fb6c4daf67425f983985750cd8d6f2beae77e614fcb34176604571f5623d6862" + opts.confFileHash = "ff6e3c6ca4188a1eeab248397cd8155f08b5530ae0bf1bf0fcc7f28bdfee3217" expectReconciled(t, cr, "", "test") expectEqual(t, fc, expectedSTS(t, fc, opts)) @@ -93,7 +93,7 @@ func TestConnector(t *testing.T) { conn.Spec.SubnetRouter.AdvertiseRoutes = []tsapi.Route{"10.44.0.0/20"} }) opts.subnetRoutes = "10.44.0.0/20" - opts.confFileHash = "bacba177bcfe3849065cf6fee53d658a9bb4144197ac5b861727d69ea99742bb" + opts.confFileHash = "2e06dc5fc2b05207007dcc89a5b31b9f4362299af24aa2110d16481dba5d338b" expectReconciled(t, cr, "", "test") expectEqual(t, fc, expectedSTS(t, fc, opts)) @@ -102,7 +102,7 @@ func TestConnector(t *testing.T) { conn.Spec.SubnetRouter = nil }) opts.subnetRoutes = "" - opts.confFileHash = "7c421a99128eb80e79a285a82702f19f8f720615542a15bd794858a6275d8079" + opts.confFileHash = "4697dc19021cf7ce95feeaa49af81f1ac81420ef0023acdd1689df272d134c60" expectReconciled(t, cr, "", "test") expectEqual(t, fc, expectedSTS(t, fc, opts)) @@ -113,7 +113,7 @@ func TestConnector(t *testing.T) { } }) opts.subnetRoutes = "10.44.0.0/20" - opts.confFileHash = "bacba177bcfe3849065cf6fee53d658a9bb4144197ac5b861727d69ea99742bb" + opts.confFileHash = "2e06dc5fc2b05207007dcc89a5b31b9f4362299af24aa2110d16481dba5d338b" expectReconciled(t, cr, "", "test") expectEqual(t, fc, expectedSTS(t, fc, opts)) @@ -156,7 +156,7 @@ func TestConnector(t *testing.T) { parentType: "connector", subnetRoutes: "10.40.0.0/14", hostname: "test-connector", - confFileHash: "57d922331890c9b1c8c6ae664394cb254334c551d9cd9db14537b5d9da9fb17e", + confFileHash: "341dc67b44be0c81a0f31f2d3b9ae67084e88435894a0543b04d7fd97bfedf24", } expectEqual(t, fc, expectedSecret(t, opts)) expectEqual(t, fc, expectedSTS(t, fc, opts)) @@ -166,7 +166,7 @@ func TestConnector(t *testing.T) { conn.Spec.ExitNode = true }) opts.isExitNode = true - opts.confFileHash = "1499b591fd97a50f0330db6ec09979792c49890cf31f5da5bb6a3f50dba1e77a" + opts.confFileHash = "8f6a5d5895cc8a1ff7896d25ac5794b44f4d3c0e74588cab4674ecb323a676dc" expectReconciled(t, cr, "", "test") expectEqual(t, fc, expectedSTS(t, fc, opts)) @@ -243,7 +243,7 @@ func TestConnectorWithProxyClass(t *testing.T) { hostname: "test-connector", isExitNode: true, subnetRoutes: "10.40.0.0/14", - confFileHash: "9321660203effb80983eaecc7b5ac5a8c53934926f46e895b9fe295dcfc5a904", + confFileHash: "9c66c269f3aa779e4e63ca414c147fd419caa62f9220a535da619f21b6bdf2b9", } expectEqual(t, fc, expectedSecret(t, opts)) expectEqual(t, fc, expectedSTS(t, fc, opts)) @@ -272,7 +272,7 @@ func TestConnectorWithProxyClass(t *testing.T) { // We lose the auth key on second reconcile, because in code it's set to // StringData, but is actually read from Data. This works with a real // API server, but not with our test setup here. - opts.confFileHash = "1499b591fd97a50f0330db6ec09979792c49890cf31f5da5bb6a3f50dba1e77a" + opts.confFileHash = "8f6a5d5895cc8a1ff7896d25ac5794b44f4d3c0e74588cab4674ecb323a676dc" expectReconciled(t, cr, "", "test") expectEqual(t, fc, expectedSTS(t, fc, opts)) diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml index 83504f4c0..7e8ae35b7 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml @@ -35,15 +35,13 @@ spec: type: object spec: type: object - required: - - statefulSet properties: statefulSet: description: Proxy's StatefulSet spec. type: object properties: annotations: - description: Annotations that will be added to the StatefulSet created for the proxy. Any Annotations specified here will be merged with the default annotations applied to the StatefulSet by the Tailscale Kubernetes operator as well as any other annotations that might have been applied by other actors. Annotations must be valid Kubernetes annotations. https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + description: Annotations that will be added to the StatefulSet created for the proxy. Any annotations specified here will be merged with the default annotations applied to the StatefulSet by the Tailscale Kubernetes operator as well as any other annotations that might have been applied by other actors. Annotations must be valid Kubernetes annotations. https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set type: object additionalProperties: type: string @@ -452,6 +450,16 @@ spec: value: description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. type: string + tailscaledConfig: + description: Configuration for tailscaled running in the proxy. + type: object + properties: + acceptRoutes: + description: AcceptRoutes can be set to "true" to configure the proxy to accept routes advertized by by other nodes on your tailnet, such as subnet routers and app connectors. This is equivalent of running 'tailscale up --accept-routes'. https://tailscale.com/kb/1072/client-preferences#use-tailscale-subnets The value of this field must be a string ("true" or "false"), defaults to "false". + type: string + x-kubernetes-validations: + - rule: type(self) == string && (self=='true' || self=='false') + message: acceptRoutes must be set to a string value. Accepted values are 'true' and 'false' status: type: object properties: diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 62e444f82..99f3b8bb7 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -196,7 +196,7 @@ spec: annotations: additionalProperties: type: string - description: Annotations that will be added to the StatefulSet created for the proxy. Any Annotations specified here will be merged with the default annotations applied to the StatefulSet by the Tailscale Kubernetes operator as well as any other annotations that might have been applied by other actors. Annotations must be valid Kubernetes annotations. https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + description: Annotations that will be added to the StatefulSet created for the proxy. Any annotations specified here will be merged with the default annotations applied to the StatefulSet by the Tailscale Kubernetes operator as well as any other annotations that might have been applied by other actors. Annotations must be valid Kubernetes annotations. https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set type: object labels: additionalProperties: @@ -604,8 +604,16 @@ spec: type: array type: object type: object - required: - - statefulSet + tailscaledConfig: + description: Configuration for tailscaled running in the proxy. + properties: + acceptRoutes: + description: AcceptRoutes can be set to true to make the proxy to accept routes. from subnet routers and route traffic via exit nodes (defaults to false). https://tailscale.com/kb/1019/subnets + type: string + x-kubernetes-validations: + - message: acceptRoutes must be set to a string value. Accepted values are 'true' and 'false' + rule: type(self) == string && (self=='true' || self=='false') + type: object type: object status: properties: diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go index a7cec1a81..5b97e6bc3 100644 --- a/cmd/k8s-operator/ingress_test.go +++ b/cmd/k8s-operator/ingress_test.go @@ -93,7 +93,7 @@ func TestTailscaleIngress(t *testing.T) { namespace: "default", parentType: "ingress", hostname: "default-test", - confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", + confFileHash: "705e5ffd0bd5326237efdf542c850a65a54101284d5daa30775420fcc64d89c1", } serveConfig := &ipn.ServeConfig{ TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, @@ -128,7 +128,7 @@ func TestTailscaleIngress(t *testing.T) { opts.shouldEnableForwardingClusterTrafficViaIngress = true // configfile hash changed at this point in test env only because we // lost auth key due to how changes are applied in test client. - opts.confFileHash = "fb9006e30ecda75e88c29dcd0ca2dd28a2ae964d001c66e1be3efe159cc3821d" + opts.confFileHash = "5c7ed263a3bde718d485eb836f307d76f8778bda1f0303b0ea554d69b5829d80" expectReconciled(t, ingR, "default", "test") expectEqual(t, fc, expectedSTS(t, fc, opts)) @@ -228,7 +228,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { namespace: "default", parentType: "ingress", hostname: "default-test", - confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", + confFileHash: "705e5ffd0bd5326237efdf542c850a65a54101284d5daa30775420fcc64d89c1", } serveConfig := &ipn.ServeConfig{ TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, @@ -263,7 +263,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { opts.proxyClass = pc.Name // configfile hash changed at this point in test env only because we // lost auth key due to how changes are applied in test client. - opts.confFileHash = "fb9006e30ecda75e88c29dcd0ca2dd28a2ae964d001c66e1be3efe159cc3821d" + opts.confFileHash = "5c7ed263a3bde718d485eb836f307d76f8778bda1f0303b0ea554d69b5829d80" expectEqual(t, fc, expectedSTSUserspace(t, fc, opts)) // 4. tailscale.com/proxy-class label is removed from the Ingress, the diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index 3dba7325f..411835e18 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -71,7 +71,7 @@ func TestLoadBalancerClass(t *testing.T) { parentType: "svc", hostname: "default-test", clusterTargetIP: "10.20.30.40", - confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", + confFileHash: "705e5ffd0bd5326237efdf542c850a65a54101284d5daa30775420fcc64d89c1", } expectEqual(t, fc, expectedSecret(t, opts)) @@ -213,7 +213,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { parentType: "svc", tailnetTargetFQDN: tailnetTargetFQDN, hostname: "default-test", - confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", + confFileHash: "705e5ffd0bd5326237efdf542c850a65a54101284d5daa30775420fcc64d89c1", } expectEqual(t, fc, expectedSecret(t, o)) @@ -324,7 +324,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { parentType: "svc", tailnetTargetIP: tailnetTargetIP, hostname: "default-test", - confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", + confFileHash: "705e5ffd0bd5326237efdf542c850a65a54101284d5daa30775420fcc64d89c1", } expectEqual(t, fc, expectedSecret(t, o)) @@ -432,7 +432,7 @@ func TestAnnotations(t *testing.T) { parentType: "svc", hostname: "default-test", clusterTargetIP: "10.20.30.40", - confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", + confFileHash: "705e5ffd0bd5326237efdf542c850a65a54101284d5daa30775420fcc64d89c1", } expectEqual(t, fc, expectedSecret(t, o)) @@ -541,7 +541,7 @@ func TestAnnotationIntoLB(t *testing.T) { parentType: "svc", hostname: "default-test", clusterTargetIP: "10.20.30.40", - confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", + confFileHash: "705e5ffd0bd5326237efdf542c850a65a54101284d5daa30775420fcc64d89c1", } expectEqual(t, fc, expectedSecret(t, o)) @@ -591,7 +591,7 @@ func TestAnnotationIntoLB(t *testing.T) { expectReconciled(t, sr, "default", "test") // None of the proxy machinery should have changed... // (although configfile hash will change in test env only because we lose auth key due to out test not syncing secret.StringData -> secret.Data) - o.confFileHash = "fb9006e30ecda75e88c29dcd0ca2dd28a2ae964d001c66e1be3efe159cc3821d" + o.confFileHash = "5c7ed263a3bde718d485eb836f307d76f8778bda1f0303b0ea554d69b5829d80" expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(t, fc, o)) // ... but the service should have a LoadBalancer status. @@ -675,7 +675,7 @@ func TestLBIntoAnnotation(t *testing.T) { parentType: "svc", hostname: "default-test", clusterTargetIP: "10.20.30.40", - confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", + confFileHash: "705e5ffd0bd5326237efdf542c850a65a54101284d5daa30775420fcc64d89c1", } expectEqual(t, fc, expectedSecret(t, o)) @@ -745,7 +745,7 @@ func TestLBIntoAnnotation(t *testing.T) { // configfile hash changes on a re-apply in this case in tests only as // we lose the auth key due to the test apply not syncing // secret.StringData -> Data. - o.confFileHash = "fb9006e30ecda75e88c29dcd0ca2dd28a2ae964d001c66e1be3efe159cc3821d" + o.confFileHash = "5c7ed263a3bde718d485eb836f307d76f8778bda1f0303b0ea554d69b5829d80" expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) expectEqual(t, fc, expectedSTS(t, fc, o)) @@ -821,7 +821,7 @@ func TestCustomHostname(t *testing.T) { parentType: "svc", hostname: "reindeer-flotilla", clusterTargetIP: "10.20.30.40", - confFileHash: "42376226c7d76ed6d6318315dc6c402f7d993bc0b01a5b0e6c8a833106b7509e", + confFileHash: "37426401f55a3e9e48d7e076a2d859386df39138858f8e2e34280555759fb4d8", } expectEqual(t, fc, expectedSecret(t, o)) @@ -937,7 +937,7 @@ func TestCustomPriorityClassName(t *testing.T) { hostname: "tailscale-critical", priorityClassName: "custom-priority-class-name", clusterTargetIP: "10.20.30.40", - confFileHash: "13cdef0d5f6f0f2406af028710ea1e0f99f65aba4021e4e70ac75a73cf141fd1", + confFileHash: "3bf08bbd3c5ef664ce6c25f1ff6d307c411db4c8c5e05ba2aeefb81d7c9de79d", } expectEqual(t, fc, expectedSTS(t, fc, o)) @@ -1000,7 +1000,7 @@ func TestProxyClassForService(t *testing.T) { parentType: "svc", hostname: "default-test", clusterTargetIP: "10.20.30.40", - confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", + confFileHash: "705e5ffd0bd5326237efdf542c850a65a54101284d5daa30775420fcc64d89c1", } expectEqual(t, fc, expectedSecret(t, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) @@ -1030,7 +1030,7 @@ func TestProxyClassForService(t *testing.T) { // configfile hash changes on a second apply in test env only because we // lose auth key due to out test not syncing secret.StringData -> // secret.Data - opts.confFileHash = "fb9006e30ecda75e88c29dcd0ca2dd28a2ae964d001c66e1be3efe159cc3821d" + opts.confFileHash = "5c7ed263a3bde718d485eb836f307d76f8778bda1f0303b0ea554d69b5829d80" expectReconciled(t, sr, "default", "test") expectEqual(t, fc, expectedSTS(t, fc, opts)) @@ -1094,7 +1094,7 @@ func TestDefaultLoadBalancer(t *testing.T) { parentType: "svc", hostname: "default-test", clusterTargetIP: "10.20.30.40", - confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", + confFileHash: "705e5ffd0bd5326237efdf542c850a65a54101284d5daa30775420fcc64d89c1", } expectEqual(t, fc, expectedSTS(t, fc, o)) } @@ -1148,7 +1148,79 @@ func TestProxyFirewallMode(t *testing.T) { hostname: "default-test", firewallMode: "nftables", clusterTargetIP: "10.20.30.40", - confFileHash: "6cceb342cd3e1c56cd1bd94c29df63df3653c35fe98a7e7afcdee0dcaa2ad549", + confFileHash: "705e5ffd0bd5326237efdf542c850a65a54101284d5daa30775420fcc64d89c1", + } + expectEqual(t, fc, expectedSTS(t, fc, o)) + +} +func TestAcceptRoutes(t *testing.T) { + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{Name: "accept-routes"}, + Spec: tsapi.ProxyClassSpec{TailscaledConfig: &tsapi.TailscaledConfig{AcceptRoutes: "true"}}} + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pc). + WithStatusSubresource(pc). + Build() + mustUpdateStatus(t, fc, "", "accept-routes", func(pc *tsapi.ProxyClass) { + pc.Status = tsapi.ProxyClassStatus{ + Conditions: []tsapi.ConnectorCondition{{ + Status: metav1.ConditionTrue, + Type: tsapi.ProxyClassready, + ObservedGeneration: pc.Generation, + }}} + }) + ft := &fakeTSClient{} + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + sr := &ServiceReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: ft, + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + logger: zl.Sugar(), + } + + // Create a service that we should manage, and check that the initial round + // of objects looks right. + mustCreate(t, fc, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + // The apiserver is supposed to set the UID, but the fake client + // doesn't. So, set it explicitly because other code later depends + // on it being set. + UID: types.UID("1234-UID"), + Labels: map[string]string{ + LabelProxyClass: "accept-routes", + }, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "10.20.30.40", + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: ptr.To("tailscale"), + }, + }) + + expectReconciled(t, sr, "default", "test") + + fullName, shortName := findGenName(t, fc, "default", "test", "svc") + o := configOpts{ + stsName: shortName, + secretName: fullName, + namespace: "default", + parentType: "svc", + hostname: "default-test", + clusterTargetIP: "10.20.30.40", + proxyClass: "accept-routes", + acceptRoutes: true, + confFileHash: "5ec91709575f0f51af26f0ab6f86c7ebb3d6007ec59186c72da67a292d11f268", } expectEqual(t, fc, expectedSTS(t, fc, o)) diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 1c231b1f2..6eb5f818f 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -55,18 +55,19 @@ const ( FinalizerName = "tailscale.com/finalizer" - // Annotations settable by users on services. + // Annotations settable by users on tailscale Services. AnnotationExpose = "tailscale.com/expose" - AnnotationTags = "tailscale.com/tags" AnnotationHostname = "tailscale.com/hostname" annotationTailnetTargetIPOld = "tailscale.com/ts-tailnet-target-ip" AnnotationTailnetTargetIP = "tailscale.com/tailnet-ip" - //MagicDNS name of tailnet node. - AnnotationTailnetTargetFQDN = "tailscale.com/tailnet-fqdn" + AnnotationTailnetTargetFQDN = "tailscale.com/tailnet-fqdn" - // Annotations settable by users on ingresses. + // Annotations settable by users on tailscale Ingresses. AnnotationFunnel = "tailscale.com/funnel" + // Annotations settable by users on tailscale Ingresses and Services. + AnnotationTags = "tailscale.com/tags" + // If set to true, set up iptables/nftables rules in the proxy forward // cluster traffic to the tailnet IP of that proxy. This can only be set // on an Ingress. This is useful in cases where a cluster target needs @@ -134,6 +135,7 @@ type connector struct { // isExitNode defines whether this Connector should act as an exit node. isExitNode bool } + type tsnetServer interface { CertDomains() []string } @@ -171,11 +173,22 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga return nil, fmt.Errorf("failed to reconcile headless service: %w", err) } - secretName, tsConfigHash, err := a.createOrGetSecret(ctx, logger, sts, hsvc) + proxyClass := new(tsapi.ProxyClass) + if sts.ProxyClass != "" { + if err := a.Get(ctx, types.NamespacedName{Name: sts.ProxyClass}, proxyClass); err != nil { + return nil, fmt.Errorf("failed to get ProxyClass: %w", err) + } + if !tsoperator.ProxyClassIsReady(proxyClass) { + logger.Infof("ProxyClass %s specified for the proxy, but it is not (yet) in a ready state, waiting..") + return nil, nil + } + } + + secretName, tsConfigHash, err := a.createOrGetSecret(ctx, logger, sts, hsvc, proxyClass) if err != nil { return nil, fmt.Errorf("failed to create or get API key secret: %w", err) } - _, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName, tsConfigHash) + _, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName, tsConfigHash, proxyClass) if err != nil { return nil, fmt.Errorf("failed to reconcile statefulset: %w", err) } @@ -288,7 +301,7 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec }) } -func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (string, string, error) { +func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service, pc *tsapi.ProxyClass) (string, string, error) { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ // Hardcode a -0 suffix so that in future, if we support @@ -336,7 +349,7 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * return "", "", err } } - confFileBytes, h, err := tailscaledConfig(stsC, authKey, orig) + confFileBytes, h, err := tailscaledConfig(stsC, authKey, orig, pc) if err != nil { return "", "", fmt.Errorf("error creating tailscaled config: %w", err) } @@ -417,7 +430,7 @@ var proxyYaml []byte //go:embed deploy/manifests/userspace-proxy.yaml var userspaceProxyYaml []byte -func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string) (*appsv1.StatefulSet, error) { +func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) { ss := new(appsv1.StatefulSet) if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil { @@ -437,16 +450,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S } pod := &ss.Spec.Template container := &pod.Spec.Containers[0] - proxyClass := new(tsapi.ProxyClass) - if sts.ProxyClass != "" { - if err := a.Get(ctx, types.NamespacedName{Name: sts.ProxyClass}, proxyClass); err != nil { - return nil, fmt.Errorf("failed to get ProxyClass: %w", err) - } - if !tsoperator.ProxyClassIsReady(proxyClass) { - logger.Infof("ProxyClass %s specified for the proxy, but it is not (yet) in a ready state, waiting..") - return nil, nil - } - } container.Image = a.proxyImage ss.ObjectMeta = metav1.ObjectMeta{ Name: headlessSvc.Name, @@ -648,12 +651,16 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet) // tailscaledConfig takes a proxy config, a newly generated auth key if // generated and a Secret with the previous proxy state and auth key and // produces returns tailscaled configuration and a hash of that configuration. -func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) ([]byte, string, error) { +func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret, proxyClass *tsapi.ProxyClass) ([]byte, string, error) { conf := ipn.ConfigVAlpha{ - Version: "alpha0", - AcceptDNS: "false", - Locked: "false", - Hostname: &stsC.Hostname, + Version: "alpha0", + AcceptDNS: "false", + Locked: "false", + Hostname: &stsC.Hostname, + AcceptRoutes: opt.NewBool(false), // always set it explicitly + } + if proxyClass != nil && proxyClass.Spec.TailscaledConfig != nil { + conf.AcceptRoutes = opt.Bool(proxyClass.Spec.TailscaledConfig.AcceptRoutes) } if stsC.Connector != nil { routes, err := netutil.CalcAdvertiseRoutes(stsC.Connector.routes, stsC.Connector.isExitNode) diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index 58c150e5d..f1071b8ba 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -25,6 +25,7 @@ import ( "tailscale.com/client/tailscale" "tailscale.com/ipn" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/types/opt" "tailscale.com/types/ptr" "tailscale.com/util/mak" ) @@ -48,6 +49,7 @@ type configOpts struct { serveConfig *ipn.ServeConfig shouldEnableForwardingClusterTrafficViaIngress bool proxyClass string // configuration from the named ProxyClass should be applied to proxy resources + acceptRoutes bool // --accept-routes } func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet { @@ -339,11 +341,12 @@ func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret { mak.Set(&s.StringData, "serve-config", string(serveConfigBs)) } conf := &ipn.ConfigVAlpha{ - Version: "alpha0", - AcceptDNS: "false", - Hostname: &opts.hostname, - Locked: "false", - AuthKey: ptr.To("secret-authkey"), + Version: "alpha0", + AcceptDNS: "false", + Hostname: &opts.hostname, + Locked: "false", + AuthKey: ptr.To("secret-authkey"), + AcceptRoutes: opt.NewBool(opts.acceptRoutes), } var routes []netip.Prefix if opts.subnetRoutes != "" || opts.isExitNode { diff --git a/k8s-operator/api.md b/k8s-operator/api.md index e2cd47c25..b09c95775 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -335,7 +335,14 @@ ConnectorCondition contains condition information for a Connector. Proxy's StatefulSet spec.
- true + false + + tailscaledConfig + object + + Configuration for tailscaled running in the proxy.
+ + false @@ -360,7 +367,7 @@ Proxy's StatefulSet spec. annotations map[string]string - Annotations that will be added to the StatefulSet created for the proxy. Any Annotations specified here will be merged with the default annotations applied to the StatefulSet by the Tailscale Kubernetes operator as well as any other annotations that might have been applied by other actors. Annotations must be valid Kubernetes annotations. https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set
+ Annotations that will be added to the StatefulSet created for the proxy. Any annotations specified here will be merged with the default annotations applied to the StatefulSet by the Tailscale Kubernetes operator as well as any other annotations that might have been applied by other actors. Annotations must be valid Kubernetes annotations. https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set
false @@ -1551,6 +1558,35 @@ The pod this Toleration is attached to tolerates any taint that matches the trip +### ProxyClass.spec.tailscaledConfig +[↩ Parent](#proxyclassspec) + + + +Configuration for tailscaled running in the proxy. + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
acceptRoutesstring + AcceptRoutes can be set to "true" to configure the proxy to accept routes advertized by by other nodes on your tailnet, such as subnet routers and app connectors. This is equivalent of running 'tailscale up --accept-routes'. https://tailscale.com/kb/1072/client-preferences#use-tailscale-subnets The value of this field must be a string ("true" or "false"), defaults to "false".
+
+ Validations:
  • type(self) == string && (self=='true' || self=='false'): acceptRoutes must be set to a string value. Accepted values are 'true' and 'false'
    • +     		false
    + + ### ProxyClass.status [↩ Parent](#proxyclass) diff --git a/k8s-operator/apis/v1alpha1/types_proxyclass.go b/k8s-operator/apis/v1alpha1/types_proxyclass.go index 98d7b5753..81e911e9f 100644 --- a/k8s-operator/apis/v1alpha1/types_proxyclass.go +++ b/k8s-operator/apis/v1alpha1/types_proxyclass.go @@ -36,10 +36,28 @@ type ProxyClassList struct { } type ProxyClassSpec struct { + // Configuration for tailscaled running in the proxy. + // +optional + TailscaledConfig *TailscaledConfig `json:"tailscaledConfig,omitempty"` // Proxy's StatefulSet spec. - StatefulSet *StatefulSet `json:"statefulSet"` + // +optional + StatefulSet *StatefulSet `json:"statefulSet,omitempty"` } +type TailscaledConfig struct { + // AcceptRoutes can be set to "true" to configure the proxy to accept + // routes advertised by other nodes on your tailnet, such as subnet + // routers and app connectors. + // This is equivalent of running 'tailscale up --accept-routes'. + // https://tailscale.com/kb/1072/client-preferences#use-tailscale-subnets + // The value of this field must be a string ("true" or "false"), + // defaults to "false". + AcceptRoutes Bool `json:"acceptRoutes,omitempty"` +} + +// +kubebuilder:validation:XValidation:rule="type(self) == string && (self=='true' || self=='false')",message="acceptRoutes must be set to a string value. Accepted values are 'true' and 'false'" +type Bool string + type StatefulSet struct { // Labels that will be added to the StatefulSet created for the proxy. // Any labels specified here will be merged with the default labels @@ -51,7 +69,7 @@ type StatefulSet struct { // +optional Labels map[string]string `json:"labels,omitempty"` // Annotations that will be added to the StatefulSet created for the proxy. - // Any Annotations specified here will be merged with the default annotations + // Any annotations specified here will be merged with the default annotations // applied to the StatefulSet by the Tailscale Kubernetes operator as // well as any other annotations that might have been applied by other // actors. diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index efd202eee..3671ff667 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -283,6 +283,11 @@ func (in *ProxyClassList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) { *out = *in + if in.TailscaledConfig != nil { + in, out := &in.TailscaledConfig, &out.TailscaledConfig + *out = new(TailscaledConfig) + **out = **in + } if in.StatefulSet != nil { in, out := &in.StatefulSet, &out.StatefulSet *out = new(StatefulSet) @@ -413,3 +418,18 @@ func (in Tags) DeepCopy() Tags { in.DeepCopyInto(out) return *out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TailscaledConfig) DeepCopyInto(out *TailscaledConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailscaledConfig. +func (in *TailscaledConfig) DeepCopy() *TailscaledConfig { + if in == nil { + return nil + } + out := new(TailscaledConfig) + in.DeepCopyInto(out) + return out +}