diff --git a/cmd/containerboot/kube.go b/cmd/containerboot/kube.go index c7c86f472..921e11fa0 100644 --- a/cmd/containerboot/kube.go +++ b/cmd/containerboot/kube.go @@ -8,6 +8,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "log" "net/http" @@ -18,20 +19,6 @@ import ( "tailscale.com/tailcfg" ) -// findKeyInKubeSecret inspects the kube secret secretName for a data -// field called "authkey", and returns its value if present. -func findKeyInKubeSecret(ctx context.Context, secretName string) (string, error) { - s, err := kc.GetSecret(ctx, secretName) - if err != nil { - return "", err - } - ak, ok := s.Data["authkey"] - if !ok { - return "", nil - } - return string(ak), nil -} - // storeDeviceInfo writes deviceID into the "device_id" data field of the kube // secret secretName. func storeDeviceInfo(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID, fqdn string, addresses []netip.Prefix) error { @@ -88,9 +75,59 @@ func deleteAuthKey(ctx context.Context, secretName string) error { return nil } -var kc *kube.Client +var kc kube.Client + +// setupKube is responsible for doing any necessary configuration and checks to +// ensure that tailscale state storage and authentication mechanism will work on +// Kubernetes. +func (cfg *settings) setupKube(ctx context.Context) error { + if cfg.KubeSecret == "" { + return nil + } + canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret) + if err != nil { + return fmt.Errorf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err) + } + cfg.KubernetesCanPatch = canPatch + + s, err := kc.GetSecret(ctx, cfg.KubeSecret) + if err != nil && kube.IsNotFoundErr(err) && !canCreate { + return fmt.Errorf("Tailscale state Secret %s does not exist and we don't have permissions to create it. "+ + "If you intend to store tailscale state elsewhere than a Kubernetes Secret, "+ + "you can explicitly set TS_KUBE_SECRET env var to an empty string. "+ + "Else ensure that RBAC is set up that allows the service account associated with this installation to create Secrets.", cfg.KubeSecret) + } else if err != nil && !kube.IsNotFoundErr(err) { + return fmt.Errorf("Getting Tailscale state Secret %s: %v", cfg.KubeSecret, err) + } + + if cfg.AuthKey == "" && !isOneStepConfig(cfg) { + if s == nil { + log.Print("TS_AUTHKEY not provided and kube secret does not exist, login will be interactive if needed.") + return nil + } + keyBytes, _ := s.Data["authkey"] + key := string(keyBytes) + + if key != "" { + // This behavior of pulling authkeys from kube secrets was added + // at the same time as the patch permission, so we can enforce + // that we must be able to patch out the authkey after + // authenticating if you want to use this feature. This avoids + // us having to deal with the case where we might leave behind + // an unnecessary reusable authkey in a secret, like a rake in + // the grass. + if !cfg.KubernetesCanPatch { + return errors.New("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.") + } + cfg.AuthKey = key + } else { + log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.") + } + } + return nil +} -func initKube(root string) { +func initKubeClient(root string) { if root != "/" { // If we are running in a test, we need to set the root path to the fake // service account directory. diff --git a/cmd/containerboot/kube_test.go b/cmd/containerboot/kube_test.go new file mode 100644 index 000000000..1f030959f --- /dev/null +++ b/cmd/containerboot/kube_test.go @@ -0,0 +1,206 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build linux + +package main + +import ( + "context" + "errors" + "testing" + + "github.com/google/go-cmp/cmp" + "tailscale.com/kube" +) + +func TestSetupKube(t *testing.T) { + tests := []struct { + name string + cfg *settings + wantErr bool + wantCfg *settings + kc kube.Client + }{ + { + name: "TS_AUTHKEY set, state Secret exists", + cfg: &settings{ + AuthKey: "foo", + KubeSecret: "foo", + }, + kc: &kube.FakeClient{ + CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) { + return false, false, nil + }, + GetSecretImpl: func(context.Context, string) (*kube.Secret, error) { + return nil, nil + }, + }, + wantCfg: &settings{ + AuthKey: "foo", + KubeSecret: "foo", + }, + }, + { + name: "TS_AUTHKEY set, state Secret does not exist, we have permissions to create it", + cfg: &settings{ + AuthKey: "foo", + KubeSecret: "foo", + }, + kc: &kube.FakeClient{ + CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) { + return false, true, nil + }, + GetSecretImpl: func(context.Context, string) (*kube.Secret, error) { + return nil, &kube.Status{Code: 404} + }, + }, + wantCfg: &settings{ + AuthKey: "foo", + KubeSecret: "foo", + }, + }, + { + name: "TS_AUTHKEY set, state Secret does not exist, we do not have permissions to create it", + cfg: &settings{ + AuthKey: "foo", + KubeSecret: "foo", + }, + kc: &kube.FakeClient{ + CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) { + return false, false, nil + }, + GetSecretImpl: func(context.Context, string) (*kube.Secret, error) { + return nil, &kube.Status{Code: 404} + }, + }, + wantCfg: &settings{ + AuthKey: "foo", + KubeSecret: "foo", + }, + wantErr: true, + }, + { + name: "TS_AUTHKEY set, we encounter a non-404 error when trying to retrieve the state Secret", + cfg: &settings{ + AuthKey: "foo", + KubeSecret: "foo", + }, + kc: &kube.FakeClient{ + CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) { + return false, false, nil + }, + GetSecretImpl: func(context.Context, string) (*kube.Secret, error) { + return nil, &kube.Status{Code: 403} + }, + }, + wantCfg: &settings{ + AuthKey: "foo", + KubeSecret: "foo", + }, + wantErr: true, + }, + { + name: "TS_AUTHKEY set, we encounter a non-404 error when trying to check Secret permissions", + cfg: &settings{ + AuthKey: "foo", + KubeSecret: "foo", + }, + wantCfg: &settings{ + AuthKey: "foo", + KubeSecret: "foo", + }, + kc: &kube.FakeClient{ + CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) { + return false, false, errors.New("broken") + }, + }, + wantErr: true, + }, + { + // Interactive login using URL in Pod logs + name: "TS_AUTHKEY not set, state Secret does not exist, we have permissions to create it", + cfg: &settings{ + KubeSecret: "foo", + }, + wantCfg: &settings{ + KubeSecret: "foo", + }, + kc: &kube.FakeClient{ + CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) { + return false, true, nil + }, + GetSecretImpl: func(context.Context, string) (*kube.Secret, error) { + return nil, &kube.Status{Code: 404} + }, + }, + }, + { + // Interactive login using URL in Pod logs + name: "TS_AUTHKEY not set, state Secret exists, but does not contain auth key", + cfg: &settings{ + KubeSecret: "foo", + }, + wantCfg: &settings{ + KubeSecret: "foo", + }, + kc: &kube.FakeClient{ + CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) { + return false, false, nil + }, + GetSecretImpl: func(context.Context, string) (*kube.Secret, error) { + return &kube.Secret{}, nil + }, + }, + }, + { + name: "TS_AUTHKEY not set, state Secret contains auth key, we do not have RBAC to patch it", + cfg: &settings{ + KubeSecret: "foo", + }, + kc: &kube.FakeClient{ + CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) { + return false, false, nil + }, + GetSecretImpl: func(context.Context, string) (*kube.Secret, error) { + return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil + }, + }, + wantCfg: &settings{ + KubeSecret: "foo", + }, + wantErr: true, + }, + { + name: "TS_AUTHKEY not set, state Secret contains auth key, we have RBAC to patch it", + cfg: &settings{ + KubeSecret: "foo", + }, + kc: &kube.FakeClient{ + CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) { + return true, false, nil + }, + GetSecretImpl: func(context.Context, string) (*kube.Secret, error) { + return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil + }, + }, + wantCfg: &settings{ + KubeSecret: "foo", + AuthKey: "foo", + KubernetesCanPatch: true, + }, + }, + } + + for _, tt := range tests { + kc = tt.kc + t.Run(tt.name, func(t *testing.T) { + if err := tt.cfg.setupKube(context.Background()); (err != nil) != tt.wantErr { + t.Errorf("settings.setupKube() error = %v, wantErr %v", err, tt.wantErr) + } + if diff := cmp.Diff(*tt.cfg, *tt.wantCfg); diff != "" { + t.Errorf("unexpected contents of settings after running settings.setupKube()\n(-got +want):\n%s", diff) + } + }) + } +} diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index f39811738..80033e138 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -171,44 +171,16 @@ func main() { } } - if cfg.InKubernetes { - initKube(cfg.Root) - } - // Context is used for all setup stuff until we're in steady // state, so that if something is hanging we eventually time out // and crashloop the container. bootCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - if cfg.InKubernetes && cfg.KubeSecret != "" { - canPatch, err := kc.CheckSecretPermissions(bootCtx, cfg.KubeSecret) - if err != nil { - log.Fatalf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err) - } - cfg.KubernetesCanPatch = canPatch - - if cfg.AuthKey == "" && !isOneStepConfig(cfg) { - key, err := findKeyInKubeSecret(bootCtx, cfg.KubeSecret) - if err != nil { - log.Fatalf("Getting authkey from kube secret: %v", err) - } - if key != "" { - // This behavior of pulling authkeys from kube secrets was added - // at the same time as the patch permission, so we can enforce - // that we must be able to patch out the authkey after - // authenticating if you want to use this feature. This avoids - // us having to deal with the case where we might leave behind - // an unnecessary reusable authkey in a secret, like a rake in - // the grass. - if !cfg.KubernetesCanPatch { - log.Fatalf("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.") - } - log.Print("Using authkey found in kube secret") - cfg.AuthKey = key - } else { - log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.") - } + if cfg.InKubernetes { + initKubeClient(cfg.Root) + if err := cfg.setupKube(bootCtx); err != nil { + log.Fatalf("error setting up for running on Kubernetes: %v", err) } } diff --git a/ipn/store/kubestore/store_kube.go b/ipn/store/kubestore/store_kube.go index 3eb29898e..2da4d8f3a 100644 --- a/ipn/store/kubestore/store_kube.go +++ b/ipn/store/kubestore/store_kube.go @@ -7,6 +7,7 @@ package kubestore import ( "context" + "fmt" "net" "strings" "time" @@ -18,7 +19,7 @@ import ( // Store is an ipn.StateStore that uses a Kubernetes Secret for persistence. type Store struct { - client *kube.Client + client kube.Client canPatch bool secretName string } @@ -29,7 +30,7 @@ func New(_ logger.Logf, secretName string) (*Store, error) { if err != nil { return nil, err } - canPatch, err := c.CheckSecretPermissions(context.Background(), secretName) + canPatch, _, err := c.CheckSecretPermissions(context.Background(), secretName) if err != nil { return nil, err } @@ -83,7 +84,7 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) error { secret, err := s.client.GetSecret(ctx, s.secretName) if err != nil { - if st, ok := err.(*kube.Status); ok && st.Code == 404 { + if kube.IsNotFoundErr(err) { return s.client.CreateSecret(ctx, &kube.Secret{ TypeMeta: kube.TypeMeta{ APIVersion: "v1", @@ -100,6 +101,19 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) error { return err } if s.canPatch { + if len(secret.Data) == 0 { // if user has pre-created a blank Secret + m := []kube.JSONPatch{ + { + Op: "add", + Path: "/data", + Value: map[string][]byte{sanitizeKey(id): bs}, + }, + } + if err := s.client.JSONPatchSecret(ctx, s.secretName, m); err != nil { + return fmt.Errorf("error patching Secret %s with a /data field: %v", s.secretName, err) + } + return nil + } m := []kube.JSONPatch{ { Op: "add", @@ -108,7 +122,7 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) error { }, } if err := s.client.JSONPatchSecret(ctx, s.secretName, m); err != nil { - return err + return fmt.Errorf("error patching Secret %s with /data/%s field", s.secretName, sanitizeKey(id)) } return nil } diff --git a/kube/client.go b/kube/client.go index f4befd1c8..62daa366e 100644 --- a/kube/client.go +++ b/kube/client.go @@ -49,7 +49,18 @@ func readFile(n string) ([]byte, error) { // Client handles connections to Kubernetes. // It expects to be run inside a cluster. -type Client struct { +type Client interface { + GetSecret(context.Context, string) (*Secret, error) + UpdateSecret(context.Context, *Secret) error + CreateSecret(context.Context, *Secret) error + StrategicMergePatchSecret(context.Context, string, *Secret, string) error + JSONPatchSecret(context.Context, string, []JSONPatch) error + CheckSecretPermissions(context.Context, string) (bool, bool, error) + SetDialer(dialer func(context.Context, string, string) (net.Conn, error)) + SetURL(string) +} + +type client struct { mu sync.Mutex url string ns string @@ -59,7 +70,7 @@ type Client struct { } // New returns a new client -func New() (*Client, error) { +func New() (Client, error) { ns, err := readFile("namespace") if err != nil { return nil, err @@ -72,7 +83,7 @@ func New() (*Client, error) { if ok := cp.AppendCertsFromPEM(caCert); !ok { return nil, fmt.Errorf("kube: error in creating root cert pool") } - return &Client{ + return &client{ url: defaultURL, ns: string(ns), client: &http.Client{ @@ -87,23 +98,23 @@ func New() (*Client, error) { // SetURL sets the URL to use for the Kubernetes API. // This is used only for testing. -func (c *Client) SetURL(url string) { +func (c *client) SetURL(url string) { c.url = url } // SetDialer sets the dialer to use when establishing a connection // to the Kubernetes API server. -func (c *Client) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) { +func (c *client) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) { c.client.Transport.(*http.Transport).DialContext = dialer } -func (c *Client) expireToken() { +func (c *client) expireToken() { c.mu.Lock() defer c.mu.Unlock() c.tokenExpiry = time.Now() } -func (c *Client) getOrRenewToken() (string, error) { +func (c *client) getOrRenewToken() (string, error) { c.mu.Lock() defer c.mu.Unlock() tk, te := c.token, c.tokenExpiry @@ -120,7 +131,7 @@ func (c *Client) getOrRenewToken() (string, error) { return c.token, nil } -func (c *Client) secretURL(name string) string { +func (c *client) secretURL(name string) string { if name == "" { return fmt.Sprintf("%s/api/v1/namespaces/%s/secrets", c.url, c.ns) } @@ -153,7 +164,7 @@ func setHeader(key, value string) func(*http.Request) { // decoded from JSON. // If the request fails with a 401, the token is expired and a new one is // requested. -func (c *Client) doRequest(ctx context.Context, method, url string, in, out any, opts ...func(*http.Request)) error { +func (c *client) doRequest(ctx context.Context, method, url string, in, out any, opts ...func(*http.Request)) error { req, err := c.newRequest(ctx, method, url, in) if err != nil { return err @@ -178,7 +189,7 @@ func (c *Client) doRequest(ctx context.Context, method, url string, in, out any, return nil } -func (c *Client) newRequest(ctx context.Context, method, url string, in any) (*http.Request, error) { +func (c *client) newRequest(ctx context.Context, method, url string, in any) (*http.Request, error) { tk, err := c.getOrRenewToken() if err != nil { return nil, err @@ -209,7 +220,7 @@ func (c *Client) newRequest(ctx context.Context, method, url string, in any) (*h } // GetSecret fetches the secret from the Kubernetes API. -func (c *Client) GetSecret(ctx context.Context, name string) (*Secret, error) { +func (c *client) GetSecret(ctx context.Context, name string) (*Secret, error) { s := &Secret{Data: make(map[string][]byte)} if err := c.doRequest(ctx, "GET", c.secretURL(name), nil, s); err != nil { return nil, err @@ -218,18 +229,18 @@ func (c *Client) GetSecret(ctx context.Context, name string) (*Secret, error) { } // CreateSecret creates a secret in the Kubernetes API. -func (c *Client) CreateSecret(ctx context.Context, s *Secret) error { +func (c *client) CreateSecret(ctx context.Context, s *Secret) error { s.Namespace = c.ns return c.doRequest(ctx, "POST", c.secretURL(""), s, nil) } // UpdateSecret updates a secret in the Kubernetes API. -func (c *Client) UpdateSecret(ctx context.Context, s *Secret) error { +func (c *client) UpdateSecret(ctx context.Context, s *Secret) error { return c.doRequest(ctx, "PUT", c.secretURL(s.Name), s, nil) } // JSONPatch is a JSON patch operation. -// It currently (2023-03-02) only supports the "remove" operation. +// It currently (2023-03-02) only supports "add" and "remove" operations. // // https://tools.ietf.org/html/rfc6902 type JSONPatch struct { @@ -239,8 +250,8 @@ type JSONPatch struct { } // JSONPatchSecret updates a secret in the Kubernetes API using a JSON patch. -// It currently (2023-03-02) only supports the "remove" operation. -func (c *Client) JSONPatchSecret(ctx context.Context, name string, patch []JSONPatch) error { +// It currently (2023-03-02) only supports "add" and "remove" operations. +func (c *client) JSONPatchSecret(ctx context.Context, name string, patch []JSONPatch) error { for _, p := range patch { if p.Op != "remove" && p.Op != "add" { panic(fmt.Errorf("unsupported JSON patch operation: %q", p.Op)) @@ -252,7 +263,7 @@ func (c *Client) JSONPatchSecret(ctx context.Context, name string, patch []JSONP // StrategicMergePatchSecret updates a secret in the Kubernetes API using a // strategic merge patch. // If a fieldManager is provided, it will be used to track the patch. -func (c *Client) StrategicMergePatchSecret(ctx context.Context, name string, s *Secret, fieldManager string) error { +func (c *client) StrategicMergePatchSecret(ctx context.Context, name string, s *Secret, fieldManager string) error { surl := c.secretURL(name) if fieldManager != "" { uv := url.Values{ @@ -267,7 +278,7 @@ func (c *Client) StrategicMergePatchSecret(ctx context.Context, name string, s * // CheckSecretPermissions checks the secret access permissions of the current // pod. It returns an error if the basic permissions tailscale needs are -// missing, and reports whether the patch permission is additionally present. +// missing, and reports whether the patch and create permissions are additionally present. // // Errors encountered during the access checking process are logged, but ignored // so that the pod tries to fail alive if the permissions exist and there's just @@ -275,7 +286,7 @@ func (c *Client) StrategicMergePatchSecret(ctx context.Context, name string, s * // should always be able to use SSARs to assess their own permissions, but since // we didn't use to check permissions this way we'll be cautious in case some // old version of k8s deviates from the current behavior. -func (c *Client) CheckSecretPermissions(ctx context.Context, secretName string) (canPatch bool, err error) { +func (c *client) CheckSecretPermissions(ctx context.Context, secretName string) (canPatch, canCreate bool, err error) { var errs []error for _, verb := range []string{"get", "update"} { ok, err := c.checkPermission(ctx, verb, secretName) @@ -286,19 +297,24 @@ func (c *Client) CheckSecretPermissions(ctx context.Context, secretName string) } } if len(errs) > 0 { - return false, multierr.New(errs...) + return false, false, multierr.New(errs...) } - ok, err := c.checkPermission(ctx, "patch", secretName) + canPatch, err = c.checkPermission(ctx, "patch", secretName) if err != nil { log.Printf("error checking patch permission on secret %s: %v", secretName, err) - return false, nil + return false, false, nil + } + canCreate, err = c.checkPermission(ctx, "create", secretName) + if err != nil { + log.Printf("error checking create permission on secret %s: %v", secretName, err) + return false, false, nil } - return ok, nil + return canPatch, canCreate, nil } // checkPermission reports whether the current pod has permission to use the -// given verb (e.g. get, update, patch) on secretName. -func (c *Client) checkPermission(ctx context.Context, verb, secretName string) (bool, error) { +// given verb (e.g. get, update, patch, create) on secretName. +func (c *client) checkPermission(ctx context.Context, verb, secretName string) (bool, error) { sar := map[string]any{ "apiVersion": "authorization.k8s.io/v1", "kind": "SelfSubjectAccessReview", @@ -322,3 +338,10 @@ func (c *Client) checkPermission(ctx context.Context, verb, secretName string) ( } return res.Status.Allowed, nil } + +func IsNotFoundErr(err error) bool { + if st, ok := err.(*Status); ok && st.Code == 404 { + return true + } + return false +} diff --git a/kube/fake_client.go b/kube/fake_client.go new file mode 100644 index 000000000..4f9ea819c --- /dev/null +++ b/kube/fake_client.go @@ -0,0 +1,37 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package kube provides a client to interact with Kubernetes. +// This package is Tailscale-internal and not meant for external consumption. +// Further, the API should not be considered stable. +package kube + +import ( + "context" + "net" +) + +var _ Client = &FakeClient{} + +type FakeClient struct { + GetSecretImpl func(context.Context, string) (*Secret, error) + CheckSecretPermissionsImpl func(ctx context.Context, name string) (bool, bool, error) +} + +func (fc *FakeClient) CheckSecretPermissions(ctx context.Context, name string) (bool, bool, error) { + return fc.CheckSecretPermissionsImpl(ctx, name) +} +func (fc *FakeClient) GetSecret(ctx context.Context, name string) (*Secret, error) { + return fc.GetSecretImpl(ctx, name) +} +func (fc *FakeClient) SetURL(_ string) {} +func (fc *FakeClient) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) { +} +func (fc *FakeClient) StrategicMergePatchSecret(context.Context, string, *Secret, string) error { + return nil +} +func (fc *FakeClient) JSONPatchSecret(context.Context, string, []JSONPatch) error { + return nil +} +func (fc *FakeClient) UpdateSecret(context.Context, *Secret) error { return nil } +func (fc *FakeClient) CreateSecret(context.Context, *Secret) error { return nil }