From d86d1e7601e73ada7b87087361b6dff45de2e466 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Fri, 10 May 2024 16:32:37 +0100 Subject: [PATCH] cmd/k8s-operator,cmd/containerboot,ipn,k8s-operator: turn off stateful filter for egress proxies. (#12075) Turn off stateful filtering for egress proxies to allow cluster traffic to be forwarded to tailnet. Allow configuring stateful filter via tailscaled config file. Deprecate EXPERIMENTAL_TS_CONFIGFILE_PATH env var and introduce a new TS_EXPERIMENTAL_VERSIONED_CONFIG env var that can be used to provide containerboot a directory that should contain one or more tailscaled config files named cap-.hujson. Containerboot will pick the one with the newest capability version that is not newer than its current capability version. Proxies with this change will not work with older Tailscale Kubernetes operator versions - users must ensure that the deployed operator is at the same version or newer (up to 4 version skew) than the proxies. Updates tailscale/tailscale#12061 Signed-off-by: Irbe Krumina Co-authored-by: Maisem Ali --- cmd/containerboot/main.go | 58 ++++++++- cmd/containerboot/main_test.go | 10 +- cmd/k8s-operator/operator_test.go | 4 +- cmd/k8s-operator/sts.go | 188 +++++++++++++++++++---------- cmd/k8s-operator/testutils_test.go | 38 +++--- ipn/conf.go | 9 +- k8s-operator/tsdns.go | 23 ---- k8s-operator/utils.go | 49 ++++++++ 8 files changed, 256 insertions(+), 123 deletions(-) delete mode 100644 k8s-operator/tsdns.go create mode 100644 k8s-operator/utils.go diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 80033e138..abeb85d09 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -52,8 +52,10 @@ // ${TS_CERT_DOMAIN}, it will be replaced with the value of the available FQDN. // It cannot be used in conjunction with TS_DEST_IP. The file is watched for changes, // and will be re-applied when it changes. -// - EXPERIMENTAL_TS_CONFIGFILE_PATH: if specified, a path to tailscaled -// config. If this is set, TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, +// - TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR: if specified, a path to a +// directory that containers tailscaled config in file. The config file needs to be +// named cap-.hujson. If this is set, TS_HOSTNAME, +// TS_EXTRA_ARGS, TS_AUTHKEY, // TS_ROUTES, TS_ACCEPT_DNS env vars must not be set. If this is set, // containerboot only runs `tailscaled --config ` // and not `tailscale up` or `tailscale set`. @@ -92,6 +94,7 @@ import ( "os" "os/exec" "os/signal" + "path" "path/filepath" "reflect" "slices" @@ -107,6 +110,7 @@ import ( "tailscale.com/client/tailscale" "tailscale.com/ipn" "tailscale.com/ipn/conffile" + kubeutils "tailscale.com/k8s-operator" "tailscale.com/tailcfg" "tailscale.com/types/logger" "tailscale.com/types/ptr" @@ -145,7 +149,7 @@ func main() { Socket: defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"), AuthOnce: defaultBool("TS_AUTH_ONCE", false), Root: defaultEnv("TS_TEST_ONLY_ROOT", "/"), - TailscaledConfigFilePath: defaultEnv("EXPERIMENTAL_TS_CONFIGFILE_PATH", ""), + TailscaledConfigFilePath: tailscaledConfigFilePath(), AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false), PodIP: defaultEnv("POD_IP", ""), } @@ -1097,6 +1101,13 @@ type settings struct { func (s *settings) validate() error { if s.TailscaledConfigFilePath != "" { + dir, file := path.Split(s.TailscaledConfigFilePath) + if _, err := os.Stat(dir); err != nil { + return fmt.Errorf("error validating whether directory with tailscaled config file %s exists: %w", dir, err) + } + if _, err := os.Stat(s.TailscaledConfigFilePath); err != nil { + return fmt.Errorf("error validating whether tailscaled config directory %q contains tailscaled config for current capability version %q: %w. If this is a Tailscale Kubernetes operator proxy, please ensure that the version of the operator is not older than the version of the proxy", dir, file, err) + } if _, err := conffile.Load(s.TailscaledConfigFilePath); err != nil { return fmt.Errorf("error validating tailscaled configfile contents: %w", err) } @@ -1120,7 +1131,7 @@ func (s *settings) validate() error { return errors.New("Both TS_TAILNET_TARGET_IP and TS_TAILNET_FQDN cannot be set") } if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") { - return errors.New("EXPERIMENTAL_TS_CONFIGFILE_PATH cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.") + return errors.New("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.") } if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode { return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode") @@ -1252,3 +1263,42 @@ func isTwoStepConfigAlwaysAuth(cfg *settings) bool { func isOneStepConfig(cfg *settings) bool { return cfg.TailscaledConfigFilePath != "" } + +// tailscaledConfigFilePath returns the path to the tailscaled config file that +// should be used for the current capability version. It is determined by the +// TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR environment variable and looks for a +// file named cap-.hujson in the directory. It searches for +// the highest capability version that is less than or equal to the current +// capability version. +func tailscaledConfigFilePath() string { + dir := os.Getenv("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR") + if dir == "" { + return "" + } + fe, err := os.ReadDir(dir) + if err != nil { + log.Fatalf("error reading tailscaled config directory %q: %v", dir, err) + } + maxCompatVer := tailcfg.CapabilityVersion(-1) + for _, e := range fe { + // We don't check if type if file as in most cases this will + // come from a mounted kube Secret, where the directory contents + // will be various symlinks. + if e.Type().IsDir() { + continue + } + cv, err := kubeutils.CapVerFromFileName(e.Name()) + if err != nil { + log.Printf("skipping file %q in tailscaled config directory %q: %v", e.Name(), dir, err) + continue + } + if cv > maxCompatVer && cv <= tailcfg.CurrentCapabilityVersion { + maxCompatVer = cv + } + } + if maxCompatVer == -1 { + log.Fatalf("no tailscaled config file found in %q for current capability version %q", dir, tailcfg.CurrentCapabilityVersion) + } + log.Printf("Using tailscaled config file %q for capability version %q", maxCompatVer, tailcfg.CurrentCapabilityVersion) + return path.Join(dir, kubeutils.TailscaledConfigFileNameForCap(maxCompatVer)) +} diff --git a/cmd/containerboot/main_test.go b/cmd/containerboot/main_test.go index 1bc423fdc..e0ddd62bf 100644 --- a/cmd/containerboot/main_test.go +++ b/cmd/containerboot/main_test.go @@ -65,7 +65,7 @@ func TestContainerBoot(t *testing.T) { "dev/net", "proc/sys/net/ipv4", "proc/sys/net/ipv6/conf/all", - "etc", + "etc/tailscaled", } for _, path := range dirs { if err := os.MkdirAll(filepath.Join(d, path), 0700); err != nil { @@ -80,7 +80,7 @@ func TestContainerBoot(t *testing.T) { "dev/net/tun": []byte(""), "proc/sys/net/ipv4/ip_forward": []byte("0"), "proc/sys/net/ipv6/conf/all/forwarding": []byte("0"), - "etc/tailscaled": tailscaledConfBytes, + "etc/tailscaled/cap-95.hujson": tailscaledConfBytes, } resetFiles := func() { for path, content := range files { @@ -638,14 +638,14 @@ func TestContainerBoot(t *testing.T) { }, }, { - Name: "experimental tailscaled configfile", + Name: "experimental tailscaled config path", Env: map[string]string{ - "EXPERIMENTAL_TS_CONFIGFILE_PATH": filepath.Join(d, "etc/tailscaled"), + "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR": filepath.Join(d, "etc/tailscaled/"), }, Phases: []phase{ { WantCmds: []string{ - "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --config=/etc/tailscaled", + "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --config=/etc/tailscaled/cap-95.hujson", }, }, { Notify: runningNotify, diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index 6556b0250..b9bd9f783 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -1182,7 +1182,7 @@ func TestTailscaledConfigfileHash(t *testing.T) { parentType: "svc", hostname: "default-test", clusterTargetIP: "10.20.30.40", - confFileHash: "705e5ffd0bd5326237efdf542c850a65a54101284d5daa30775420fcc64d89c1", + confFileHash: "e09bededa0379920141cbd0b0dbdf9b8b66545877f9e8397423f5ce3e1ba439e", } expectEqual(t, fc, expectedSTS(t, fc, o), nil) @@ -1192,7 +1192,7 @@ func TestTailscaledConfigfileHash(t *testing.T) { mak.Set(&svc.Annotations, AnnotationHostname, "another-test") }) o.hostname = "another-test" - o.confFileHash = "1a087f887825d2b75d3673c7c2b0131f8ec1f0b1cb761d33e236dd28350dfe23" + o.confFileHash = "5d754cf55463135ee34aa9821f2fd8483b53eb0570c3740c84a086304f427684" expectReconciled(t, sr, "default", "test") expectEqual(t, fc, expectedSTS(t, fc, o), nil) } diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index defbbfd23..4d9417ac0 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -29,6 +29,7 @@ import ( "sigs.k8s.io/yaml" "tailscale.com/client/tailscale" "tailscale.com/ipn" + kubeutils "tailscale.com/k8s-operator" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/net/netutil" @@ -92,10 +93,6 @@ const ( podAnnotationLastSetTailnetTargetFQDN = "tailscale.com/operator-last-set-ts-tailnet-target-fqdn" // podAnnotationLastSetConfigFileHash is sha256 hash of the current tailscaled configuration contents. podAnnotationLastSetConfigFileHash = "tailscale.com/operator-last-set-config-file-hash" - - // tailscaledConfigKey is the name of the key in proxy Secret Data that - // holds the tailscaled config contents. - tailscaledConfigKey = "tailscaled" ) var ( @@ -174,11 +171,11 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga return nil, fmt.Errorf("failed to reconcile headless service: %w", err) } - secretName, tsConfigHash, err := a.createOrGetSecret(ctx, logger, sts, hsvc) + secretName, tsConfigHash, configs, err := a.createOrGetSecret(ctx, logger, sts, hsvc) if err != nil { return nil, fmt.Errorf("failed to create or get API key secret: %w", err) } - _, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName, tsConfigHash) + _, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName, tsConfigHash, configs) if err != nil { return nil, fmt.Errorf("failed to reconcile statefulset: %w", err) } @@ -291,7 +288,7 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec }) } -func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (string, string, error) { +func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName, hash string, configs tailscaleConfigs, _ error) { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ // Hardcode a -0 suffix so that in future, if we support @@ -307,25 +304,23 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName()) orig = secret.DeepCopy() } else if !apierrors.IsNotFound(err) { - return "", "", err + return "", "", nil, err } - var ( - authKey, hash string - ) + var authKey string if orig == nil { // Initially it contains only tailscaled config, but when the // proxy starts, it will also store there the state, certs and // ACME account key. sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, stsC.ChildResourceLabels) if err != nil { - return "", "", err + return "", "", nil, err } if sts != nil { // StatefulSet exists, so we have already created the secret. // If the secret is missing, they should delete the StatefulSet. logger.Errorf("Tailscale proxy secret doesn't exist, but the corresponding StatefulSet %s/%s already does. Something is wrong, please delete the StatefulSet.", sts.GetNamespace(), sts.GetName()) - return "", "", nil + return "", "", nil, nil } // Create API Key secret which is going to be used by the statefulset // to authenticate with Tailscale. @@ -336,45 +331,58 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * } authKey, err = a.newAuthKey(ctx, tags) if err != nil { - return "", "", err + return "", "", nil, err } } - confFileBytes, h, err := tailscaledConfig(stsC, authKey, orig) + configs, err := tailscaledConfig(stsC, authKey, orig) if err != nil { - return "", "", fmt.Errorf("error creating tailscaled config: %w", err) + return "", "", nil, fmt.Errorf("error creating tailscaled config: %w", err) + } + hash, err = tailscaledConfigHash(configs) + if err != nil { + return "", "", nil, fmt.Errorf("error calculating hash of tailscaled configs: %w", err) + } + + latest := tailcfg.CapabilityVersion(-1) + var latestConfig ipn.ConfigVAlpha + for key, val := range configs { + fn := kubeutils.TailscaledConfigFileNameForCap(key) + b, err := json.Marshal(val) + if err != nil { + return "", "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err) + } + mak.Set(&secret.StringData, fn, string(b)) + if key > latest { + latest = key + latestConfig = val + } } - hash = h - mak.Set(&secret.StringData, tailscaledConfigKey, string(confFileBytes)) if stsC.ServeConfig != nil { j, err := json.Marshal(stsC.ServeConfig) if err != nil { - return "", "", err + return "", "", nil, err } mak.Set(&secret.StringData, "serve-config", string(j)) } if orig != nil { - logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(secret.Data[tailscaledConfigKey])) + logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig)) if err := a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil { - return "", "", err + return "", "", nil, err } } else { - logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes([]byte(secret.StringData[tailscaledConfigKey]))) + logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig)) if err := a.Create(ctx, secret); err != nil { - return "", "", err + return "", "", nil, err } } - return secret.Name, hash, nil + return secret.Name, hash, configs, nil } // sanitizeConfigBytes returns ipn.ConfigVAlpha in string form with redacted // auth key. -func sanitizeConfigBytes(bs []byte) string { - c := &ipn.ConfigVAlpha{} - if err := json.Unmarshal(bs, c); err != nil { - return "invalid config" - } +func sanitizeConfigBytes(c ipn.ConfigVAlpha) string { if c.AuthKey != nil { c.AuthKey = ptr.To("**redacted**") } @@ -437,7 +445,7 @@ var proxyYaml []byte //go:embed deploy/manifests/userspace-proxy.yaml var userspaceProxyYaml []byte -func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string) (*appsv1.StatefulSet, error) { +func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string, configs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha) (*appsv1.StatefulSet, error) { ss := new(appsv1.StatefulSet) if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil { @@ -493,9 +501,15 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S Value: proxySecret, }, corev1.EnvVar{ + // Old tailscaled config key is still used for backwards compatibility. Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled", }, + corev1.EnvVar{ + // New style is in the form of cap-.hujson. + Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", + Value: "/etc/tsconfig", + }, ) if sts.ForwardClusterTrafficViaL7IngressProxy { container.Env = append(container.Env, corev1.EnvVar{ @@ -505,18 +519,16 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S } // Configure containeboot to run tailscaled with a configfile read from the state Secret. mak.Set(&ss.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, tsConfigHash) - pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ + + configVolume := corev1.Volume{ Name: "tailscaledconfig", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: proxySecret, - Items: []corev1.KeyToPath{{ - Key: tailscaledConfigKey, - Path: tailscaledConfigKey, - }}, }, }, - }) + } + pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, configVolume) container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ Name: "tailscaledconfig", ReadOnly: true, @@ -571,10 +583,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: proxySecret, - Items: []corev1.KeyToPath{{ - Key: "serve-config", - Path: "serve-config", - }}, + Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}, }, }, }) @@ -716,42 +725,82 @@ func enableMetrics(ss *appsv1.StatefulSet, pc *tsapi.ProxyClass) { } } +func readAuthKey(secret *corev1.Secret, key string) (*string, error) { + origConf := &ipn.ConfigVAlpha{} + if err := json.Unmarshal([]byte(secret.Data[key]), origConf); err != nil { + return nil, fmt.Errorf("error unmarshaling previous tailscaled config in %q: %w", key, err) + } + return origConf.AuthKey, nil +} + // tailscaledConfig takes a proxy config, a newly generated auth key if // generated and a Secret with the previous proxy state and auth key and -// produces returns tailscaled configuration and a hash of that configuration. -func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) ([]byte, string, error) { - conf := ipn.ConfigVAlpha{ - Version: "alpha0", - AcceptDNS: "false", - AcceptRoutes: "false", // AcceptRoutes defaults to true - Locked: "false", - Hostname: &stsC.Hostname, +// returns tailscaled configuration and a hash of that configuration. +// +// As of 2024-05-09 it also returns legacy tailscaled config without the +// later added NoStatefulFilter field to support proxies older than cap95. +// TODO (irbekrm): remove the legacy config once we no longer need to support +// versions older than cap94, +// https://tailscale.com/kb/1236/kubernetes-operator#operator-and-proxies +func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaleConfigs, error) { + conf := &ipn.ConfigVAlpha{ + Version: "alpha0", + AcceptDNS: "false", + AcceptRoutes: "false", // AcceptRoutes defaults to true + Locked: "false", + Hostname: &stsC.Hostname, + NoStatefulFiltering: "false", + } + + // For egress proxies only, we need to ensure that stateful filtering is + // not in place so that traffic from cluster can be forwarded via + // Tailscale IPs. + if stsC.TailnetTargetFQDN != "" || stsC.TailnetTargetIP != "" { + conf.NoStatefulFiltering = "true" } if stsC.Connector != nil { routes, err := netutil.CalcAdvertiseRoutes(stsC.Connector.routes, stsC.Connector.isExitNode) if err != nil { - return nil, "", fmt.Errorf("error calculating routes: %w", err) + return nil, fmt.Errorf("error calculating routes: %w", err) } conf.AdvertiseRoutes = routes } if newAuthkey != "" { conf.AuthKey = &newAuthkey - } else if oldSecret != nil && len(oldSecret.Data[tailscaledConfigKey]) > 0 { // write to StringData, read from Data as StringData is write-only - origConf := &ipn.ConfigVAlpha{} - if err := json.Unmarshal([]byte(oldSecret.Data[tailscaledConfigKey]), origConf); err != nil { - return nil, "", fmt.Errorf("error unmarshaling previous tailscaled config: %w", err) + } else if oldSecret != nil { + var err error + latest := tailcfg.CapabilityVersion(-1) + latestStr := "" + for k, data := range oldSecret.Data { + // write to StringData, read from Data as StringData is write-only + if len(data) == 0 { + continue + } + v, err := kubeutils.CapVerFromFileName(k) + if err != nil { + continue + } + if v > latest { + latestStr = k + latest = v + } + } + // Allow for configs that don't contain an auth key. Perhaps + // users have some mechanisms to delete them. Auth key is + // normally not needed after the initial login. + if latestStr != "" { + conf.AuthKey, err = readAuthKey(oldSecret, latestStr) + if err != nil { + return nil, err + } } - conf.AuthKey = origConf.AuthKey - } - confFileBytes, err := json.Marshal(conf) - if err != nil { - return nil, "", fmt.Errorf("error marshaling tailscaled config : %w", err) - } - hash, err := hashBytes(confFileBytes) - if err != nil { - return nil, "", fmt.Errorf("error calculating config hash: %w", err) } - return confFileBytes, hash, nil + capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha) + capVerConfigs[95] = *conf + // legacy config should not contain NoStatefulFiltering field. + conf.NoStatefulFiltering.Clear() + capVerConfigs[94] = *conf + return capVerConfigs, nil } // ptrObject is a type constraint for pointer types that implement @@ -761,7 +810,9 @@ type ptrObject[T any] interface { *T } -// hashBytes produces a hash for the provided bytes that is the same across +type tailscaleConfigs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha + +// hashBytes produces a hash for the provided tailscaled config that is the same across // different invocations of this code. We do not use the // tailscale.com/deephash.Hash here because that produces a different hash for // the same value in different tailscale builds. The hash we are producing here @@ -770,10 +821,13 @@ type ptrObject[T any] interface { // thing that changed is operator version (the hash is also exposed to users via // an annotation and might be confusing if it changes without the config having // changed). -func hashBytes(b []byte) (string, error) { - h := sha256.New() - _, err := h.Write(b) +func tailscaledConfigHash(c tailscaleConfigs) (string, error) { + b, err := json.Marshal(c) if err != nil { + return "", fmt.Errorf("error marshalling tailscaled configs: %w", err) + } + h := sha256.New() + if _, err = h.Write(b); err != nil { return "", fmt.Errorf("error calculating hash: %w", err) } return fmt.Sprintf("%x", h.Sum(nil)), nil diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index ae9eaebd1..684b75433 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -67,6 +67,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef {Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, {Name: "TS_KUBE_SECRET", Value: opts.secretName}, {Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"}, + {Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"}, }, SecurityContext: &corev1.SecurityContext{ Capabilities: &corev1.Capabilities{ @@ -89,12 +90,6 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: opts.secretName, - Items: []corev1.KeyToPath{ - { - Key: "tailscaled", - Path: "tailscaled", - }, - }, }, }, }, @@ -144,9 +139,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config", }) - volumes = append(volumes, corev1.Volume{ - Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Path: "serve-config", Key: "serve-config"}}}}, - }) + volumes = append(volumes, corev1.Volume{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}}) tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"}) } ss := &appsv1.StatefulSet{ @@ -229,6 +222,7 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps {Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, {Name: "TS_KUBE_SECRET", Value: opts.secretName}, {Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"}, + {Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"}, {Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"}, }, ImagePullPolicy: "Always", @@ -243,20 +237,12 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: opts.secretName, - Items: []corev1.KeyToPath{ - { - Key: "tailscaled", - Path: "tailscaled", - }, - }, }, }, }, {Name: "serve-config", VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, - Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}, - }, + Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}}, } ss := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ @@ -388,7 +374,17 @@ func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret { if err != nil { t.Fatalf("error marshalling tailscaled config") } + if opts.tailnetTargetFQDN != "" || opts.tailnetTargetIP != "" { + conf.NoStatefulFiltering = "true" + } else { + conf.NoStatefulFiltering = "false" + } + bn, err := json.Marshal(conf) + if err != nil { + t.Fatalf("error marshalling tailscaled config") + } mak.Set(&s.StringData, "tailscaled", string(b)) + mak.Set(&s.StringData, "cap-95.hujson", string(bn)) labels := map[string]string{ "tailscale.com/managed": "true", "tailscale.com/parent-resource": "test", @@ -463,7 +459,7 @@ func mustUpdateStatus[T any, O ptrObject[T]](t *testing.T, client client.Client, // they are not present in the passed object and use the modify func to remove // them from the cluster object. If no such modifications are needed, you can // pass nil in place of the modify function. -func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want O, modify func(O)) { +func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want O, modifier func(O)) { t.Helper() got := O(new(T)) if err := client.Get(context.Background(), types.NamespacedName{ @@ -477,8 +473,8 @@ func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want // so just remove it from both got and want. got.SetResourceVersion("") want.SetResourceVersion("") - if modify != nil { - modify(got) + if modifier != nil { + modifier(got) } if diff := cmp.Diff(got, want); diff != "" { t.Fatalf("unexpected object (-got +want):\n%s", diff) diff --git a/ipn/conf.go b/ipn/conf.go index ffd4c7269..ed6e3c871 100644 --- a/ipn/conf.go +++ b/ipn/conf.go @@ -32,7 +32,8 @@ type ConfigVAlpha struct { AdvertiseRoutes []netip.Prefix `json:",omitempty"` DisableSNAT opt.Bool `json:",omitempty"` - NetfilterMode *string `json:",omitempty"` // "on", "off", "nodivert" + NetfilterMode *string `json:",omitempty"` // "on", "off", "nodivert" + NoStatefulFiltering opt.Bool `json:",omitempty"` PostureChecking opt.Bool `json:",omitempty"` RunSSHServer opt.Bool `json:",omitempty"` // Tailscale SSH @@ -50,6 +51,7 @@ func (c *ConfigVAlpha) ToPrefs() (MaskedPrefs, error) { if c == nil { return mp, nil } + mp.WantRunning = !c.Enabled.EqualBool(false) mp.WantRunningSet = mp.WantRunning || c.Enabled != "" if c.ServerURL != nil { @@ -98,6 +100,11 @@ func (c *ConfigVAlpha) ToPrefs() (MaskedPrefs, error) { mp.NoSNAT = c.DisableSNAT.EqualBool(true) mp.NoSNAT = true } + if c.NoStatefulFiltering != "" { + mp.NoStatefulFiltering = c.NoStatefulFiltering + mp.NoStatefulFilteringSet = true + } + if c.NetfilterMode != nil { m, err := preftype.ParseNetfilterMode(*c.NetfilterMode) if err != nil { diff --git a/k8s-operator/tsdns.go b/k8s-operator/tsdns.go deleted file mode 100644 index 0269bca11..000000000 --- a/k8s-operator/tsdns.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -//go:build !plan9 - -package kube - -const ( - Alpha1Version = "v1alpha1" - - DNSRecordsCMName = "dnsrecords" - DNSRecordsCMKey = "records.json" -) - -type Records struct { - // Version is the version of this Records configuration. Version is - // written by the operator, i.e when it first populates the Records. - // k8s-nameserver must verify that it knows how to parse a given - // version. - Version string `json:"version"` - // IP4 contains a mapping of DNS names to IPv4 address(es). - IP4 map[string][]string `json:"ip4"` -} diff --git a/k8s-operator/utils.go b/k8s-operator/utils.go new file mode 100644 index 000000000..7d755f8eb --- /dev/null +++ b/k8s-operator/utils.go @@ -0,0 +1,49 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package kube + +import ( + "fmt" + + "tailscale.com/tailcfg" +) + +const ( + Alpha1Version = "v1alpha1" + + DNSRecordsCMName = "dnsrecords" + DNSRecordsCMKey = "records.json" +) + +type Records struct { + // Version is the version of this Records configuration. Version is + // written by the operator, i.e when it first populates the Records. + // k8s-nameserver must verify that it knows how to parse a given + // version. + Version string `json:"version"` + // IP4 contains a mapping of DNS names to IPv4 address(es). + IP4 map[string][]string `json:"ip4"` +} + +// TailscaledConfigFileNameForCap returns a tailscaled config file name in +// format expected by containerboot for the given CapVer. +func TailscaledConfigFileNameForCap(cap tailcfg.CapabilityVersion) string { + if cap < 95 { + return "tailscaled" + } + return fmt.Sprintf("cap-%v.hujson", cap) +} + +// CapVerFromFileName parses the capability version from a tailscaled +// config file name previously generated by TailscaledConfigFileNameForCap. +func CapVerFromFileName(name string) (tailcfg.CapabilityVersion, error) { + if name == "tailscaled" { + return 0, nil + } + var cap tailcfg.CapabilityVersion + _, err := fmt.Sscanf(name, "cap-%d.hujson", &cap) + return cap, err +}