From 23d8ccce34ef4514888d02082dbb69cddd2152c1 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Fri, 5 Jan 2024 09:14:02 +0000 Subject: [PATCH] Experimental Signed-off-by: Irbe Krumina --- cmd/k8s-operator/operator.go | 104 +++++----- cmd/k8s-operator/proxy.go | 375 ++++++++++++++++++++++++++++++----- cmd/k8s-operator/sts.go | 40 ++-- 3 files changed, 408 insertions(+), 111 deletions(-) diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index d762acd9a..3fa26c804 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -9,7 +9,6 @@ package main import ( "context" - "os" "regexp" "strings" "time" @@ -17,7 +16,6 @@ import ( "github.com/go-logr/zapr" "go.uber.org/zap" "go.uber.org/zap/zapcore" - "golang.org/x/oauth2/clientcredentials" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" @@ -41,6 +39,7 @@ import ( "tailscale.com/tsnet" "tailscale.com/tstime" "tailscale.com/types/logger" + "tailscale.com/util/must" "tailscale.com/version" ) @@ -56,13 +55,13 @@ func main() { tailscale.I_Acknowledge_This_API_Is_Unstable = true var ( - tsNamespace = defaultEnv("OPERATOR_NAMESPACE", "") - tslogging = defaultEnv("OPERATOR_LOGGING", "info") - image = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest") - priorityClassName = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "") - tags = defaultEnv("PROXY_TAGS", "tag:k8s") - tsFirewallMode = defaultEnv("PROXY_FIREWALL_MODE", "") - tsEnableConnector = defaultBool("ENABLE_CONNECTOR", false) + // tsNamespace = defaultEnv("OPERATOR_NAMESPACE", "") + tslogging = defaultEnv("OPERATOR_LOGGING", "info") + // image = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest") + // priorityClassName = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "") + // tags = defaultEnv("PROXY_TAGS", "tag:k8s") + // tsFirewallMode = defaultEnv("PROXY_FIREWALL_MODE", "") + // tsEnableConnector = defaultBool("ENABLE_CONNECTOR", false) ) var opts []kzap.Opts @@ -80,20 +79,22 @@ func main() { // The operator can run either as a plain operator or it can // additionally act as api-server proxy // https://tailscale.com/kb/1236/kubernetes-operator/?q=kubernetes#accessing-the-kubernetes-control-plane-using-an-api-server-proxy. - mode := parseAPIProxyMode() + // mode := parseAPIProxyMode() + mode := apiserverProxyModeNoAuth if mode == apiserverProxyModeDisabled { hostinfo.SetApp("k8s-operator") } else { hostinfo.SetApp("k8s-operator-proxy") } - s, tsClient := initTSNet(zlog) + s, _ := initTSNet(zlog) defer s.Close() - restConfig := config.GetConfigOrDie() + restConfig := must.Get(config.GetConfigWithContext("")) maybeLaunchAPIServerProxy(zlog, restConfig, s, mode) + select {} // TODO (irbekrm): gather the reconciler options into an opts struct // rather than passing a million of them in one by one. - runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags, tsFirewallMode, tsEnableConnector) + // runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags, tsFirewallMode, tsEnableConnector) } // initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the @@ -101,31 +102,31 @@ func main() { // with Tailscale. func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, *tailscale.Client) { var ( - clientIDPath = defaultEnv("CLIENT_ID_FILE", "") - clientSecretPath = defaultEnv("CLIENT_SECRET_FILE", "") - hostname = defaultEnv("OPERATOR_HOSTNAME", "tailscale-operator") - kubeSecret = defaultEnv("OPERATOR_SECRET", "") - operatorTags = defaultEnv("OPERATOR_INITIAL_TAGS", "tag:k8s-operator") + // clientIDPath = defaultEnv("CLIENT_ID_FILE", "") + // clientSecretPath = defaultEnv("CLIENT_SECRET_FILE", "") + hostname = defaultEnv("OPERATOR_HOSTNAME", "tailscale-operator") + kubeSecret = defaultEnv("OPERATOR_SECRET", "") + // operatorTags = defaultEnv("OPERATOR_INITIAL_TAGS", "tag:k8s-operator") ) startlog := zlog.Named("startup") - if clientIDPath == "" || clientSecretPath == "" { - startlog.Fatalf("CLIENT_ID_FILE and CLIENT_SECRET_FILE must be set") - } - clientID, err := os.ReadFile(clientIDPath) - if err != nil { - startlog.Fatalf("reading client ID %q: %v", clientIDPath, err) - } - clientSecret, err := os.ReadFile(clientSecretPath) - if err != nil { - startlog.Fatalf("reading client secret %q: %v", clientSecretPath, err) - } - credentials := clientcredentials.Config{ - ClientID: string(clientID), - ClientSecret: string(clientSecret), - TokenURL: "https://login.tailscale.com/api/v2/oauth/token", - } - tsClient := tailscale.NewClient("-", nil) - tsClient.HTTPClient = credentials.Client(context.Background()) + // if clientIDPath == "" || clientSecretPath == "" { + // startlog.Fatalf("CLIENT_ID_FILE and CLIENT_SECRET_FILE must be set") + // } + // clientID, err := os.ReadFile(clientIDPath) + // if err != nil { + // startlog.Fatalf("reading client ID %q: %v", clientIDPath, err) + // } + // clientSecret, err := os.ReadFile(clientSecretPath) + // if err != nil { + // startlog.Fatalf("reading client secret %q: %v", clientSecretPath, err) + // } + // credentials := clientcredentials.Config{ + // ClientID: string(clientID), + // ClientSecret: string(clientSecret), + // TokenURL: "https://login.tailscale.com/api/v2/oauth/token", + // } + // tsClient := tailscale.NewClient("-", nil) + // tsClient.HTTPClient = credentials.Client(context.Background()) s := &tsnet.Server{ Hostname: hostname, @@ -163,21 +164,22 @@ waitOnline: if loginDone { break } - caps := tailscale.KeyCapabilities{ - Devices: tailscale.KeyDeviceCapabilities{ - Create: tailscale.KeyDeviceCreateCapabilities{ - Reusable: false, - Preauthorized: true, - Tags: strings.Split(operatorTags, ","), - }, - }, - } - authkey, _, err := tsClient.CreateKey(ctx, caps) - if err != nil { - startlog.Fatalf("creating operator authkey: %v", err) - } + // caps := tailscale.KeyCapabilities{ + // Devices: tailscale.KeyDeviceCapabilities{ + // Create: tailscale.KeyDeviceCreateCapabilities{ + // Reusable: false, + // Preauthorized: true, + // Tags: strings.Split(operatorTags, ","), + // }, + // }, + // } + // authkey := "" + // authkey, _, err := tsClient.CreateKey(ctx, caps) + // if err != nil { + // startlog.Fatalf("creating operator authkey: %v", err) + // } if err := lc.Start(ctx, ipn.Options{ - AuthKey: authkey, + // AuthKey: authkey, }); err != nil { startlog.Fatalf("starting tailscale: %v", err) } @@ -196,7 +198,7 @@ waitOnline: } time.Sleep(time.Second) } - return s, tsClient + return s, nil } // runReconcilers starts the controller-runtime manager and registers the diff --git a/cmd/k8s-operator/proxy.go b/cmd/k8s-operator/proxy.go index 9a6526cc9..a707e6254 100644 --- a/cmd/k8s-operator/proxy.go +++ b/cmd/k8s-operator/proxy.go @@ -6,16 +6,25 @@ package main import ( + "bufio" "context" "crypto/tls" + "encoding/json" "fmt" + "io" "log" + "net" "net/http" "net/http/httputil" + "net/textproto" "net/url" "os" + "path" "strings" + "sync" + "time" + "github.com/pkg/errors" "go.uber.org/zap" "k8s.io/client-go/rest" "k8s.io/client-go/transport" @@ -87,9 +96,9 @@ func maybeLaunchAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, return } startlog := zlog.Named("launchAPIProxy") - if mode == apiserverProxyModeNoAuth { - restConfig = rest.AnonymousClientConfig(restConfig) - } + // if mode == apiserverProxyModeNoAuth { + // restConfig = rest.AnonymousClientConfig(restConfig) + // } cfg, err := restConfig.TransportConfig() if err != nil { startlog.Fatalf("could not get rest.TransportConfig(): %v", err) @@ -108,15 +117,89 @@ func maybeLaunchAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, if err != nil { startlog.Fatalf("could not get rest.TransportConfig(): %v", err) } - go runAPIServerProxy(s, rt, zlog.Named("apiserver-proxy"), mode) + go runAPIServerProxy(s, rt, zlog.Named("apiserver-proxy"), mode, restConfig.Host) } // apiserverProxy is an http.Handler that authenticates requests using the Tailscale // LocalAPI and then proxies them to the Kubernetes API. type apiserverProxy struct { - log *zap.SugaredLogger - lc *tailscale.LocalClient - rp *httputil.ReverseProxy + log *zap.SugaredLogger + lc *tailscale.LocalClient + rp *httputil.ReverseProxy + mode apiServerProxyMode + + upstreamURL *url.URL + upstreamClient *http.Client +} + +// Hop-by-hop headers. These are removed when sent to the backend. +// As of RFC 7230, hop-by-hop headers are required to appear in the +// Connection header field. These are the headers defined by the +// obsoleted RFC 2616 (section 13.5.1) and are used for backward +// compatibility. +var hopHeaders = []string{ + "Connection", + "Proxy-Connection", // non-standard but still sent by libcurl and rejected by e.g. google + "Keep-Alive", + "Proxy-Authenticate", + "Proxy-Authorization", + "Te", // canonicalized version of "TE" + "Trailer", // not Trailers per URL above; https://www.rfc-editor.org/errata_search.php?eid=4522 + "Transfer-Encoding", + "Upgrade", +} + +// removeHopByHopHeaders removes hop-by-hop headers. +func removeHopByHopHeaders(h http.Header) { + // RFC 7230, section 6.1: Remove headers listed in the "Connection" header. + for _, f := range h["Connection"] { + for _, sf := range strings.Split(f, ",") { + if sf = textproto.TrimString(sf); sf != "" { + h.Del(sf) + } + } + } + // RFC 2616, section 13.5.1: Remove a set of known hop-by-hop headers. + // This behavior is superseded by the RFC 7230 Connection header, but + // preserve it for backwards compatibility. + for _, f := range hopHeaders { + h.Del(f) + } +} + +func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) { + // Replace the URL with the Kubernetes APIServer. + + r.URL.Scheme = h.upstreamURL.Scheme + r.URL.Host = h.upstreamURL.Host + if h.mode == apiserverProxyModeNoAuth { + // If we are not providing authentication, then we are just + // proxying to the Kubernetes API, so we don't need to do + // anything else. + return + } + + // We want to proxy to the Kubernetes API, but we want to use + // the caller's identity to do so. We do this by impersonating + // the caller using the Kubernetes User Impersonation feature: + // https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation + + // Out of paranoia, remove all authentication headers that might + // have been set by the client. + r.Header.Del("Authorization") + r.Header.Del("Impersonate-Group") + r.Header.Del("Impersonate-User") + r.Header.Del("Impersonate-Uid") + for k := range r.Header { + if strings.HasPrefix(k, "Impersonate-Extra-") { + r.Header.Del(k) + } + } + + // Now add the impersonation headers that we want. + if err := addImpersonationHeaders(r, h.log); err != nil { + panic("failed to add impersonation headers: " + err.Error()) + } } func (h *apiserverProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -127,7 +210,84 @@ func (h *apiserverProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } counterNumRequestsProxied.Add(1) - h.rp.ServeHTTP(w, addWhoIsToRequest(r, who)) + r = addWhoIsToRequest(r, who) + if r.Method != "POST" || path.Base(r.URL.Path) != "exec" { // also check for pod + h.rp.ServeHTTP(w, r) + return + } + // hj := w.(http.Hijacker) + // reqConn, brw, err := hj.Hijack() + // if err != nil { + // return + // } + // defer reqConn.Close() + // if err := brw.Flush(); err != nil { + // return + // } + // reqConn = netutil.NewDrainBufConn(reqConn, brw.Reader) + // respConn, err := net.Dial("tcp", h.upstreamURL.Host) + // if err != nil { + // h.log.Errorf("failed to dial upstream: %v", err) + // return + // } + // defer respConn.Close() + + req2 := r.Clone(r.Context()) + h.addImpersonationHeadersAsRequired(req2) + + req2.Body = io.NopCloser(io.TeeReader(r.Body, os.Stdout)) + defer r.Body.Close() + + h.rp.ServeHTTP(&teeResponseWriter{ + ResponseWriter: w, + hj: w.(http.Hijacker), + multiWriter: io.MultiWriter(os.Stdout, w), + }, req2) +} + +type teeResponseWriter struct { + http.ResponseWriter + hj http.Hijacker + multiWriter io.Writer +} + +func (w *teeResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + reqConn, brw, err := w.hj.Hijack() + if err != nil { + return nil, nil, err + } + f, err := os.OpenFile("/tmp/recording.cast", os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return nil, nil, err + } + r := &recording{ + start: time.Now(), + failOpen: true, + out: f, + } + lc := &loggingConn{Conn: reqConn, lw: &loggingWriter{ + r: r, + recordingFailedOpen: false, + }} + + ch := CastHeader{ + Version: 2, + Timestamp: r.start.Unix(), + } + j, err := json.Marshal(ch) + if err != nil { + return nil, nil, err + } + j = append(j, '\n') + if _, err := f.Write(j); err != nil { + return nil, nil, err + } + + return lc, brw, nil +} + +func (w *teeResponseWriter) Write(b []byte) (int, error) { + return w.multiWriter.Write(b) } // runAPIServerProxy runs an HTTP server that authenticates requests using the @@ -144,7 +304,7 @@ func (h *apiserverProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { // are passed through to the Kubernetes API. // // It never returns. -func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLogger, mode apiServerProxyMode) { +func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLogger, mode apiServerProxyMode, host string) { if mode == apiserverProxyModeDisabled { return } @@ -152,7 +312,7 @@ func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLo if err != nil { log.Fatalf("could not listen on :443: %v", err) } - u, err := url.Parse(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS"))) + u, err := url.Parse(host) if err != nil { log.Fatalf("runAPIServerProxy: failed to parse URL %v", err) } @@ -162,45 +322,16 @@ func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLo log.Fatalf("could not get local client: %v", err) } ap := &apiserverProxy{ - log: log, - lc: lc, - rp: &httputil.ReverseProxy{ - Rewrite: func(r *httputil.ProxyRequest) { - // Replace the URL with the Kubernetes APIServer. - - r.Out.URL.Scheme = u.Scheme - r.Out.URL.Host = u.Host - if mode == apiserverProxyModeNoAuth { - // If we are not providing authentication, then we are just - // proxying to the Kubernetes API, so we don't need to do - // anything else. - return - } - - // We want to proxy to the Kubernetes API, but we want to use - // the caller's identity to do so. We do this by impersonating - // the caller using the Kubernetes User Impersonation feature: - // https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation - - // Out of paranoia, remove all authentication headers that might - // have been set by the client. - r.Out.Header.Del("Authorization") - r.Out.Header.Del("Impersonate-Group") - r.Out.Header.Del("Impersonate-User") - r.Out.Header.Del("Impersonate-Uid") - for k := range r.Out.Header { - if strings.HasPrefix(k, "Impersonate-Extra-") { - r.Out.Header.Del(k) - } - } - - // Now add the impersonation headers that we want. - if err := addImpersonationHeaders(r.Out, log); err != nil { - panic("failed to add impersonation headers: " + err.Error()) - } - }, - Transport: rt, + log: log, + lc: lc, + mode: mode, + upstreamURL: u, + } + ap.rp = &httputil.ReverseProxy{ + Rewrite: func(pr *httputil.ProxyRequest) { + ap.addImpersonationHeadersAsRequired(pr.Out) }, + Transport: rt, } hs := &http.Server{ // Kubernetes uses SPDY for exec and port-forward, however SPDY is @@ -285,3 +416,151 @@ func addImpersonationHeaders(r *http.Request, log *zap.SugaredLogger) error { } return nil } + +// CastHeader is the header of an asciinema file. +type CastHeader struct { + // Version is the asciinema file format version. + Version int `json:"version"` + + // Width is the terminal width in characters. + // It is non-zero for Pty sessions. + Width int `json:"width"` + + // Height is the terminal height in characters. + // It is non-zero for Pty sessions. + Height int `json:"height"` + + // Timestamp is the unix timestamp of when the recording started. + Timestamp int64 `json:"timestamp"` + + // Env is the environment variables of the session. + // Only "TERM" is set (2023-03-22). + Env map[string]string `json:"env"` + + // Command is the command that was executed. + // Typically empty for shell sessions. + Command string `json:"command,omitempty"` + + // Tailscale-specific fields: + // SrcNode is the FQDN of the node originating the connection. + // It is also the MagicDNS name for the node. + // It does not have a trailing dot. + // e.g. "host.tail-scale.ts.net" + SrcNode string `json:"srcNode"` + + // SrcNodeID is the node ID of the node originating the connection. + SrcNodeID tailcfg.StableNodeID `json:"srcNodeID"` + + // SrcNodeTags is the list of tags on the node originating the connection (if any). + SrcNodeTags []string `json:"srcNodeTags,omitempty"` + + // SrcNodeUserID is the user ID of the node originating the connection (if not tagged). + SrcNodeUserID tailcfg.UserID `json:"srcNodeUserID,omitempty"` // if not tagged + + // SrcNodeUser is the LoginName of the node originating the connection (if not tagged). + SrcNodeUser string `json:"srcNodeUser,omitempty"` + + // SSHUser is the username as presented by the client. + SSHUser string `json:"sshUser"` // as presented by the client + + // LocalUser is the effective username on the server. + LocalUser string `json:"localUser"` + + // ConnectionID uniquely identifies a connection made to the SSH server. + // It may be shared across multiple sessions over the same connection in + // case of SSH multiplexing. + ConnectionID string `json:"connectionID"` +} + +// loggingWriter is an io.Writer wrapper that writes first an +// asciinema JSON cast format recording line, and then writes to w. +type loggingWriter struct { + r *recording + + // recordingFailedOpen specifies whether we've failed to write to + // r.out and should stop trying. It is set to true if we fail to write + // to r.out and r.failOpen is set. + recordingFailedOpen bool +} + +func (w *loggingWriter) Write(p []byte) (n int, err error) { + if w.recordingFailedOpen { + return 0, nil + } + j, err := json.Marshal([]any{ + time.Since(w.r.start).Seconds(), + "o", + string(p), + }) + if err != nil { + return 0, err + } + j = append(j, '\n') + if err := w.writeCastLine(j); err != nil { + if !w.r.failOpen { + return 0, err + } + w.recordingFailedOpen = true + } + return len(p), nil +} + +func (w *loggingWriter) writeCastLine(j []byte) error { + w.r.mu.Lock() + defer w.r.mu.Unlock() + if w.r.out == nil { + return errors.New("logger closed") + } + _, err := w.r.out.Write(j) + if err != nil { + return fmt.Errorf("logger Write: %w", err) + } + return nil +} + +type loggingConn struct { + mu sync.Mutex // guards writes to r.out + closed bool + net.Conn + lw *loggingWriter +} + +func (c *loggingConn) Write(b []byte) (int, error) { + n, err := c.Conn.Write(b) + c.lw.Write(b[:n]) + return n, err +} + +func (c *loggingConn) Close() error { + c.mu.Lock() + defer c.mu.Unlock() + if c.closed { + return nil + } + c.closed = true + c.lw.r.Close() + return c.Conn.Close() +} + +// recording is the state for an SSH session recording. +type recording struct { + start time.Time + + // failOpen specifies whether the session should be allowed to + // continue if writing to the recording fails. + failOpen bool + + mu sync.Mutex // guards writes to, close of out + out io.WriteCloser +} + +func (r *recording) Close() error { + r.mu.Lock() + defer r.mu.Unlock() + if r.out == nil { + return nil + } + err := r.out.Close() + r.out = nil + return err +} diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index d24a6eb02..a9d7df82c 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -60,6 +60,7 @@ const ( podAnnotationLastSetHostname = "tailscale.com/operator-last-set-hostname" podAnnotationLastSetTailnetTargetIP = "tailscale.com/operator-last-set-ts-tailnet-target-ip" podAnnotationLastSetTailnetTargetFQDN = "tailscale.com/operator-last-set-ts-tailnet-target-fqdn" + podAnnotationLastSetConfigFileHash = "tailscale.com/operator-last-set-config-file-hash" ) type tailscaleSTSConfig struct { @@ -127,10 +128,11 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga return nil, fmt.Errorf("failed to reconcile headless service: %w", err) } - secretName, err := a.createOrGetSecret(ctx, logger, sts, hsvc) + secretName, key, err := a.createOrGetSecret(ctx, logger, sts, hsvc) if err != nil { return nil, fmt.Errorf("failed to create or get API key secret: %w", err) } + sts.key = key _, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName) if err != nil { return nil, fmt.Errorf("failed to reconcile statefulset: %w", err) @@ -245,7 +247,7 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec }) } -func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (string, error) { +func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (string, string, error) { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ // Hardcode a -0 suffix so that in future, if we support @@ -261,22 +263,23 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName()) orig = secret.DeepCopy() } else if !apierrors.IsNotFound(err) { - return "", err + return "", "", err } + var authKey string if orig == nil { // Secret doesn't exist yet, create one. Initially it contains // only the Tailscale authkey, but once Tailscale starts it'll // also store the daemon state. sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, stsC.ChildResourceLabels) if err != nil { - return "", err + return "", "", err } if sts != nil { // StatefulSet exists, so we have already created the secret. // If the secret is missing, they should delete the StatefulSet. logger.Errorf("Tailscale proxy secret doesn't exist, but the corresponding StatefulSet %s/%s already does. Something is wrong, please delete the StatefulSet.", sts.GetNamespace(), sts.GetName()) - return "", nil + return "", "", nil } // Create API Key secret which is going to be used by the statefulset // to authenticate with Tailscale. @@ -287,31 +290,32 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * } authKey, err := a.newAuthKey(ctx, tags) if err != nil { - return "", err + return "", "", err } mak.Set(&secret.StringData, "authkey", authKey) + } else { + authKey = string(orig.Data["authkey"]) } if stsC.ServeConfig != nil { j, err := json.Marshal(stsC.ServeConfig) if err != nil { - return "", err + return "", "", err } mak.Set(&secret.StringData, "serve-config", string(j)) } if orig != nil { log.Printf("Patching existing secret %s", secret.Name) if err := a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil { - return "", err + return "", "", err } } else { if err := a.Create(ctx, secret); err != nil { - return "", err + return "", authKey, err } log.Printf("Created secret %s", secret.Name) } - log.Printf("Created secret %s", secret.Name) - return secret.Name, nil + return secret.Name, authKey, nil } // DeviceInfo returns the device ID and hostname for the Tailscale device @@ -396,6 +400,8 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S Name: "TS_HOSTNAME", Value: sts.Hostname, }) + + var configFileHash string if sts.ClusterTargetIP != "" { container.Env = append(container.Env, corev1.EnvVar{ Name: "TS_DEST_IP", @@ -435,7 +441,11 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S }) } else if sts.Connector != nil { // TODO: definitely not the right place for this - a.tsConfigCM(ctx, headlessSvc.Name, a.operatorNamespace, logger, sts) + var err error + configFileHash, err = a.tsConfigCM(ctx, headlessSvc.Name, a.operatorNamespace, logger, sts) + if err != nil { + return nil, fmt.Errorf("failed to create configmap: %w", err) + } container.Env = append(container.Env, corev1.EnvVar{ Name: "TS_CONFIGFILE_PATH", Value: "/tsconfig/tailscaled", @@ -454,6 +464,9 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S Name: "configfile", MountPath: "/tsconfig", }) + if sts.key != "" { + + } // We need to provide these env vars even if the values are empty to // ensure that a transition from a Connector with a defined subnet // router or exit node to one without succeeds. @@ -500,6 +513,9 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S if sts.TailnetTargetFQDN != "" { ss.Spec.Template.Annotations[podAnnotationLastSetTailnetTargetFQDN] = sts.TailnetTargetFQDN } + if configFileHash != "" { + ss.Spec.Template.Annotations[podAnnotationLastSetTailnetTargetFQDN] = configFileHash + } ss.Spec.Template.Labels = map[string]string{ "app": sts.ParentResourceUID, }