From 320f77bd243f4e749e9dcee56a2ff25199a0b9f5 Mon Sep 17 00:00:00 2001 From: Maisem Ali Date: Thu, 24 Aug 2023 12:08:50 -0400 Subject: [PATCH] cmd/containerboot: add support for setting ServeConfig This watches the provided path for a JSON encoded ipn.ServeConfig. Everytime the file changes, or the nodes FQDN changes it reapplies the ServeConfig. At boot time, it nils out any previous ServeConfig just like tsnet does. As the ServeConfig requires pre-existing knowledge of the nodes FQDN to do SNI matching, it introduces a special `${TS_CERT_DOMAIN}` value in the JSON file which is replaced with the known CertDomain before it is applied. Updates #502 Updates #7895 Signed-off-by: Maisem Ali --- cmd/containerboot/main.go | 126 +++++++++++++++++++++++++++++++++----- go.mod | 2 +- 2 files changed, 113 insertions(+), 15 deletions(-) diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 658305400..4e86bd6cd 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -37,6 +37,10 @@ // logged in. If false (the default, for backwards // compatibility), forcibly log in every time the // container starts. +// - TS_SERVE_CONFIG: if specified, is the file path where the ipn.ServeConfig is located. +// It will be applied once tailscaled is up and running. If the file contains +// ${TS_CERT_DOMAIN}, it will be replaced with the value of the available FQDN. +// It cannot be used in conjunction with TS_DEST_IP. // // When running on Kubernetes, containerboot defaults to storing state in the // "tailscale" kube secret. To store state on local disk instead, set @@ -48,7 +52,9 @@ package main import ( + "bytes" "context" + "encoding/json" "errors" "fmt" "io/fs" @@ -60,12 +66,15 @@ import ( "path/filepath" "strconv" "strings" + "sync/atomic" "syscall" "time" + "github.com/fsnotify/fsnotify" "golang.org/x/sys/unix" "tailscale.com/client/tailscale" "tailscale.com/ipn" + "tailscale.com/types/ptr" "tailscale.com/util/deephash" ) @@ -78,6 +87,7 @@ func main() { Hostname: defaultEnv("TS_HOSTNAME", ""), Routes: defaultEnv("TS_ROUTES", ""), ProxyTo: defaultEnv("TS_DEST_IP", ""), + ServeConfigPath: defaultEnv("TS_SERVE_CONFIG", ""), DaemonExtraArgs: defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""), ExtraArgs: defaultEnv("TS_EXTRA_ARGS", ""), InKubernetes: os.Getenv("KUBERNETES_SERVICE_HOST") != "", @@ -95,6 +105,9 @@ func main() { if cfg.ProxyTo != "" && cfg.UserspaceMode { log.Fatal("TS_DEST_IP is not supported with TS_USERSPACE") } + if cfg.ProxyTo != "" && cfg.ServeConfigPath != "" { + log.Fatal("TS_DEST_IP is not supported with TS_SERVE_CONFIG") + } if !cfg.UserspaceMode { if err := ensureTunFile(cfg.Root); err != nil { @@ -120,18 +133,18 @@ func main() { // Context is used for all setup stuff until we're in steady // state, so that if something is hanging we eventually time out // and crashloop the container. - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + bootCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() if cfg.InKubernetes && cfg.KubeSecret != "" { - canPatch, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret) + canPatch, err := kc.CheckSecretPermissions(bootCtx, cfg.KubeSecret) if err != nil { log.Fatalf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err) } cfg.KubernetesCanPatch = canPatch if cfg.AuthKey == "" { - key, err := findKeyInKubeSecret(ctx, cfg.KubeSecret) + key, err := findKeyInKubeSecret(bootCtx, cfg.KubeSecret) if err != nil { log.Fatalf("Getting authkey from kube secret: %v", err) } @@ -154,12 +167,12 @@ func main() { } } - client, daemonPid, err := startTailscaled(ctx, cfg) + client, daemonPid, err := startTailscaled(bootCtx, cfg) if err != nil { log.Fatalf("failed to bring up tailscale: %v", err) } - w, err := client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialPrefs|ipn.NotifyInitialState) + w, err := client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialPrefs|ipn.NotifyInitialState) if err != nil { log.Fatalf("failed to watch tailscaled for updates: %v", err) } @@ -178,10 +191,10 @@ func main() { } didLogin = true w.Close() - if err := tailscaleLogin(ctx, cfg); err != nil { + if err := tailscaleLogin(bootCtx, cfg); err != nil { return fmt.Errorf("failed to auth tailscale: %v", err) } - w, err = client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState) + w, err = client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState) if err != nil { return fmt.Errorf("rewatching tailscaled for updates after auth: %v", err) } @@ -210,12 +223,6 @@ authLoop: case ipn.NeedsMachineAuth: log.Printf("machine authorization required, please visit the admin panel") case ipn.Running: - // Now that we are authenticated, we can set/reset any of the - // settings that we need to. - if err := tailscaleSet(ctx, cfg); err != nil { - log.Fatalf("failed to auth tailscale: %v", err) - } - // Technically, all we want is to keep monitoring the bus for // netmap updates. However, in order to make the container crash // if tailscale doesn't initially come up, the watch has a @@ -231,6 +238,20 @@ authLoop: w.Close() + ctx, cancel := context.WithCancel(context.Background()) // no deadline now that we're in steady state + defer cancel() + + // Now that we are authenticated, we can set/reset any of the + // settings that we need to. + if err := tailscaleSet(ctx, cfg); err != nil { + log.Fatalf("failed to auth tailscale: %v", err) + } + // Remove any serve config that may have been set by a previous + // run of containerboot. + if err := client.SetServeConfig(ctx, new(ipn.ServeConfig)); err != nil { + log.Fatalf("failed to unset serve config: %v", err) + } + if cfg.InKubernetes && cfg.KubeSecret != "" && cfg.KubernetesCanPatch && cfg.AuthOnce { // We were told to only auth once, so any secret-bound // authkey is no longer needed. We don't strictly need to @@ -241,7 +262,7 @@ authLoop: } } - w, err = client.WatchIPNBus(context.Background(), ipn.NotifyInitialNetMap|ipn.NotifyInitialState) + w, err = client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState) if err != nil { log.Fatalf("rewatching tailscaled for updates after auth: %v", err) } @@ -252,7 +273,13 @@ authLoop: startupTasksDone = false currentIPs deephash.Sum // tailscale IPs assigned to device currentDeviceInfo deephash.Sum // device ID and fqdn + + certDomain = new(atomic.Pointer[string]) + certDomainChanged = make(chan bool, 1) ) + if cfg.ServeConfigPath != "" { + go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client) + } for { n, err := w.Next() if err != nil { @@ -273,6 +300,16 @@ authLoop: log.Fatalf("installing proxy rules: %v", err) } } + if cfg.ServeConfigPath != "" && len(n.NetMap.DNS.CertDomains) > 0 { + cd := n.NetMap.DNS.CertDomains[0] + prev := certDomain.Swap(ptr.To(cd)) + if prev == nil || *prev != cd { + select { + case certDomainChanged <- true: + default: + } + } + } deviceInfo := []any{n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name()} if cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != "" && deephash.Update(¤tDeviceInfo, &deviceInfo) { if err := storeDeviceInfo(ctx, cfg.KubeSecret, n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name()); err != nil { @@ -312,6 +349,66 @@ authLoop: } } +// watchServeConfigChanges watches path for changes, and when it sees one, reads +// the serve config from it, replacing ${TS_CERT_DOMAIN} with certDomain, and +// applies it to lc. It exits when ctx is canceled. cdChanged is a channel that +// is written to when the certDomain changes, causing the serve config to be +// re-read and applied. +func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient) { + if certDomainAtomic == nil { + panic("cd must not be nil") + } + w, err := fsnotify.NewWatcher() + if err != nil { + log.Fatalf("failed to create fsnotify watcher: %v", err) + } + defer w.Close() + if err := w.Add(filepath.Dir(path)); err != nil { + log.Fatalf("failed to add fsnotify watch: %v", err) + } + var certDomain string + for { + select { + case <-ctx.Done(): + return + case <-cdChanged: + certDomain = *certDomainAtomic.Load() + case e := <-w.Events: + if e.Name != path { + continue + } + } + if certDomain == "" { + continue + } + sc, err := readServeConfig(path, certDomain) + if err != nil { + log.Fatalf("failed to read serve config: %v", err) + } + if err := lc.SetServeConfig(ctx, sc); err != nil { + log.Fatalf("failed to set serve config: %v", err) + } + } +} + +// readServeConfig reads the ipn.ServeConfig from path, replacing +// ${TS_CERT_DOMAIN} with certDomain. +func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) { + if path == "" { + return nil, nil + } + j, err := os.ReadFile(path) + if err != nil { + return nil, err + } + j = bytes.ReplaceAll(j, []byte("${TS_CERT_DOMAIN}"), []byte(certDomain)) + var sc ipn.ServeConfig + if err := json.Unmarshal(j, &sc); err != nil { + return nil, err + } + return &sc, nil +} + func startTailscaled(ctx context.Context, cfg *settings) (*tailscale.LocalClient, int, error) { args := tailscaledArgs(cfg) sigCh := make(chan os.Signal, 1) @@ -556,6 +653,7 @@ type settings struct { Hostname string Routes string ProxyTo string + ServeConfigPath string DaemonExtraArgs string ExtraArgs string InKubernetes bool diff --git a/go.mod b/go.mod index 91399e77a..d0e175b46 100644 --- a/go.mod +++ b/go.mod @@ -171,7 +171,7 @@ require ( github.com/fatih/color v1.15.0 // indirect github.com/fatih/structtag v1.2.0 // indirect github.com/firefart/nonamedreturns v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 github.com/fzipp/gocyclo v0.6.0 // indirect github.com/go-critic/go-critic v0.8.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect