From 0842e2f45b7384fff1991f0782bf0816f0578fda Mon Sep 17 00:00:00 2001 From: Maisem Ali Date: Wed, 1 Sep 2021 08:11:43 -0700 Subject: [PATCH] ipn/store: add ability to store data as k8s secrets. Signed-off-by: Maisem Ali --- cmd/tailscale/depaware.txt | 1 + cmd/tailscaled/depaware.txt | 1 + cmd/tailscaled/tailscaled.go | 2 +- docs/k8s/README.md | 20 ++++ docs/k8s/role.yaml | 10 ++ docs/k8s/rolebinding.yaml | 12 +++ docs/k8s/sa.yaml | 5 + ipn/ipnserver/server.go | 16 ++- ipn/store.go | 72 ++++++++++++++ kube/api.go | 188 +++++++++++++++++++++++++++++++++++ kube/client.go | 170 +++++++++++++++++++++++++++++++ 11 files changed, 493 insertions(+), 4 deletions(-) create mode 100644 docs/k8s/README.md create mode 100644 docs/k8s/role.yaml create mode 100644 docs/k8s/rolebinding.yaml create mode 100644 docs/k8s/sa.yaml create mode 100644 kube/api.go create mode 100644 kube/client.go diff --git a/cmd/tailscale/depaware.txt b/cmd/tailscale/depaware.txt index 180f07656..7c2b8dcc9 100644 --- a/cmd/tailscale/depaware.txt +++ b/cmd/tailscale/depaware.txt @@ -31,6 +31,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/hostinfo from tailscale.com/net/interfaces tailscale.com/ipn from tailscale.com/cmd/tailscale/cli+ tailscale.com/ipn/ipnstate from tailscale.com/cmd/tailscale/cli+ + tailscale.com/kube from tailscale.com/ipn tailscale.com/metrics from tailscale.com/derp tailscale.com/net/dnscache from tailscale.com/derp/derphttp tailscale.com/net/flowtrack from tailscale.com/wgengine/filter+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 8a8cae982..9278a8aaa 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -105,6 +105,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/ipn/ipnstate from tailscale.com/ipn+ tailscale.com/ipn/localapi from tailscale.com/ipn/ipnserver tailscale.com/ipn/policy from tailscale.com/ipn/ipnlocal + tailscale.com/kube from tailscale.com/ipn tailscale.com/log/filelogger from tailscale.com/ipn/ipnserver tailscale.com/log/logheap from tailscale.com/control/controlclient tailscale.com/logpolicy from tailscale.com/cmd/tailscaled diff --git a/cmd/tailscaled/tailscaled.go b/cmd/tailscaled/tailscaled.go index 8e1e19945..9d0db3f1b 100644 --- a/cmd/tailscaled/tailscaled.go +++ b/cmd/tailscaled/tailscaled.go @@ -111,7 +111,7 @@ func main() { flag.StringVar(&args.socksAddr, "socks5-server", "", `optional [ip]:port to run a SOCK5 server (e.g. "localhost:1080")`) flag.StringVar(&args.tunname, "tun", defaultTunName(), `tunnel interface name; use "userspace-networking" (beta) to not use TUN`) flag.Var(flagtype.PortValue(&args.port, 0), "port", "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select") - flag.StringVar(&args.statepath, "state", paths.DefaultTailscaledStateFile(), "path of state file") + flag.StringVar(&args.statepath, "state", paths.DefaultTailscaledStateFile(), "path of state file; use 'kube:' to use Kubernetes secrets") flag.StringVar(&args.socketpath, "socket", paths.DefaultTailscaledSocket(), "path of the service unix socket") flag.StringVar(&args.birdSocketPath, "bird-socket", "", "path of the bird unix socket") flag.BoolVar(&printVersion, "version", false, "print version information and exit") diff --git a/docs/k8s/README.md b/docs/k8s/README.md new file mode 100644 index 000000000..cd76b8150 --- /dev/null +++ b/docs/k8s/README.md @@ -0,0 +1,20 @@ +# Using Kubernetes Secrets as the state store for Tailscale +Tailscale supports using Kubernetes Secrets as the state store, however there is some configuration required in order for it to work. + +**Note: this only works if `tailscaled` runs inside a pod in the cluster.** + +1. Create a service account for Tailscale (optional) + ``` + kubectl create -f sa.yaml + ``` + +1. Create role and role bindings for the service account + ``` + kubectl create -f role.yaml + kubectl create -f rolebinding.yaml + ``` + +1. Launch `tailscaled` with a Kubernetes Secret as the state store. + ``` + tailscaled --state=kube:tailscale + ``` diff --git a/docs/k8s/role.yaml b/docs/k8s/role.yaml new file mode 100644 index 000000000..f31060aa6 --- /dev/null +++ b/docs/k8s/role.yaml @@ -0,0 +1,10 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: default + name: tailscale +rules: +- apiGroups: [""] # "" indicates the core API group + resourceNames: ["tailscale"] + resources: ["secrets"] + verbs: ["create", "get", "update"] diff --git a/docs/k8s/rolebinding.yaml b/docs/k8s/rolebinding.yaml new file mode 100644 index 000000000..e979d1c1a --- /dev/null +++ b/docs/k8s/rolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + namespace: default + name: tailscale +subjects: +- kind: ServiceAccount + name: tailscale +roleRef: + kind: Role + name: tailscale + apiGroup: rbac.authorization.k8s.io diff --git a/docs/k8s/sa.yaml b/docs/k8s/sa.yaml new file mode 100644 index 000000000..ffa54fad8 --- /dev/null +++ b/docs/k8s/sa.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tailscale + namespace: default diff --git a/ipn/ipnserver/server.go b/ipn/ipnserver/server.go index e9bdfaad1..e1966d70d 100644 --- a/ipn/ipnserver/server.go +++ b/ipn/ipnserver/server.go @@ -613,9 +613,19 @@ func Run(ctx context.Context, logf logger.Logf, logid string, getEngine func() ( var store ipn.StateStore if opts.StatePath != "" { - store, err = ipn.NewFileStore(opts.StatePath) - if err != nil { - return fmt.Errorf("ipn.NewFileStore(%q): %v", opts.StatePath, err) + const kubePrefix = "kube:" + switch { + case strings.HasPrefix(opts.StatePath, kubePrefix): + secretName := strings.TrimPrefix(opts.StatePath, kubePrefix) + store, err = ipn.NewKubeStore(secretName) + if err != nil { + return fmt.Errorf("ipn.NewKubeStore(%q): %v", secretName, err) + } + default: + store, err = ipn.NewFileStore(opts.StatePath) + if err != nil { + return fmt.Errorf("ipn.NewFileStore(%q): %v", opts.StatePath, err) + } } if opts.AutostartStateKey == "" { autoStartKey, err := store.ReadState(ipn.ServerModeStartKey) diff --git a/ipn/store.go b/ipn/store.go index 9d5b5829e..58f6bbc38 100644 --- a/ipn/store.go +++ b/ipn/store.go @@ -6,6 +6,7 @@ package ipn import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -14,8 +15,10 @@ import ( "os" "path/filepath" "sync" + "time" "tailscale.com/atomicfile" + "tailscale.com/kube" ) // ErrStateNotExist is returned by StateStore.ReadState when the @@ -55,6 +58,75 @@ type StateStore interface { WriteState(id StateKey, bs []byte) error } +// KubeStore is a StateStore that uses a Kubernetes Secret for persistence. +type KubeStore struct { + client *kube.Client + secretName string +} + +// NewKubeStore returns a new KubeStore that persists to the named secret. +func NewKubeStore(secretName string) (*KubeStore, error) { + c, err := kube.New() + if err != nil { + return nil, err + } + return &KubeStore{ + client: c, + secretName: secretName, + }, nil +} + +func (s *KubeStore) String() string { return "KubeStore" } + +// ReadState implements the StateStore interface. +func (s *KubeStore) ReadState(id StateKey) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + secret, err := s.client.GetSecret(ctx, s.secretName) + if err != nil { + if st, ok := err.(*kube.Status); ok && st.Code == 404 { + return nil, ErrStateNotExist + } + return nil, err + } + b, ok := secret.Data[string(id)] + if !ok { + return nil, ErrStateNotExist + } + return b, nil +} + +// WriteState implements the StateStore interface. +func (s *KubeStore) WriteState(id StateKey, bs []byte) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + secret, err := s.client.GetSecret(ctx, s.secretName) + if err != nil { + if st, ok := err.(*kube.Status); ok && st.Code == 404 { + return s.client.CreateSecret(ctx, &kube.Secret{ + TypeMeta: kube.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: kube.ObjectMeta{ + Name: s.secretName, + }, + Data: map[string][]byte{ + string(id): bs, + }, + }) + } + return err + } + secret.Data[string(id)] = bs + if err := s.client.UpdateSecret(ctx, secret); err != nil { + return err + } + return err +} + // MemoryStore is a store that keeps state in memory only. type MemoryStore struct { mu sync.Mutex diff --git a/kube/api.go b/kube/api.go new file mode 100644 index 000000000..8a2bfbc47 --- /dev/null +++ b/kube/api.go @@ -0,0 +1,188 @@ +// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package kube + +import "time" + +// Note: The API types are copied from k8s.io/api{,machinery} to not introduce a +// module dependency on the Kubernetes API as it pulls in many more dependencies. + +// TypeMeta describes an individual object in an API response or request with +// strings representing the type of the object and its API schema version. +// Structures that are versioned or persisted should inline TypeMeta. +type TypeMeta struct { + // Kind is a string value representing the REST resource this object represents. + // Servers may infer this from the endpoint the client submits requests to. + // Cannot be updated. + // In CamelCase. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + // +optional + Kind string `json:"kind,omitempty"` + + // APIVersion defines the versioned schema of this representation of an object. + // Servers should convert recognized schemas to the latest internal value, and + // may reject unrecognized values. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + // +optional + APIVersion string `json:"apiVersion,omitempty"` +} + +// ObjectMeta is metadata that all persisted resources must have, which +// includes all objects users must create. +type ObjectMeta struct { + // Name must be unique within a namespace. Is required when creating resources, although + // some resources may allow a client to request the generation of an appropriate name + // automatically. Name is primarily intended for creation idempotence and configuration + // definition. + // Cannot be updated. + // More info: http://kubernetes.io/docs/user-guide/identifiers#names + // +optional + Name string `json:"name"` + + // Namespace defines the space within which each name must be unique. An empty namespace is + // equivalent to the "default" namespace, but "default" is the canonical representation. + // Not all objects are required to be scoped to a namespace - the value of this field for + // those objects will be empty. + // + // Must be a DNS_LABEL. + // Cannot be updated. + // More info: http://kubernetes.io/docs/user-guide/namespaces + // +optional + Namespace string `json:"namespace"` + + // UID is the unique in time and space value for this object. It is typically generated by + // the server on successful creation of a resource and is not allowed to change on PUT + // operations. + // + // Populated by the system. + // Read-only. + // More info: http://kubernetes.io/docs/user-guide/identifiers#uids + // +optional + UID string `json:"uid,omitempty"` + + // An opaque value that represents the internal version of this object that can + // be used by clients to determine when objects have changed. May be used for optimistic + // concurrency, change detection, and the watch operation on a resource or set of resources. + // Clients must treat these values as opaque and passed unmodified back to the server. + // They may only be valid for a particular resource or set of resources. + // + // Populated by the system. + // Read-only. + // Value must be treated as opaque by clients and . + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + // +optional + ResourceVersion string `json:"resourceVersion,omitempty"` + + // A sequence number representing a specific generation of the desired state. + // Populated by the system. Read-only. + // +optional + Generation int64 `json:"generation,omitempty"` + + // CreationTimestamp is a timestamp representing the server time when this object was + // created. It is not guaranteed to be set in happens-before order across separate operations. + // Clients may not set this value. It is represented in RFC3339 form and is in UTC. + // + // Populated by the system. + // Read-only. + // Null for lists. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + CreationTimestamp time.Time `json:"creationTimestamp,omitempty"` + + // DeletionTimestamp is RFC 3339 date and time at which this resource will be deleted. This + // field is set by the server when a graceful deletion is requested by the user, and is not + // directly settable by a client. The resource is expected to be deleted (no longer visible + // from resource lists, and not reachable by name) after the time in this field, once the + // finalizers list is empty. As long as the finalizers list contains items, deletion is blocked. + // Once the deletionTimestamp is set, this value may not be unset or be set further into the + // future, although it may be shortened or the resource may be deleted prior to this time. + // For example, a user may request that a pod is deleted in 30 seconds. The Kubelet will react + // by sending a graceful termination signal to the containers in the pod. After that 30 seconds, + // the Kubelet will send a hard termination signal (SIGKILL) to the container and after cleanup, + // remove the pod from the API. In the presence of network partitions, this object may still + // exist after this timestamp, until an administrator or automated process can determine the + // resource is fully terminated. + // If not set, graceful deletion of the object has not been requested. + // + // Populated by the system when a graceful deletion is requested. + // Read-only. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + DeletionTimestamp *time.Time `json:"deletionTimestamp,omitempty"` + + // Number of seconds allowed for this object to gracefully terminate before + // it will be removed from the system. Only set when deletionTimestamp is also set. + // May only be shortened. + // Read-only. + // +optional + DeletionGracePeriodSeconds *int64 `json:"deletionGracePeriodSeconds,omitempty"` + + // Map of string keys and values that can be used to organize and categorize + // (scope and select) objects. May match selectors of replication controllers + // and services. + // More info: http://kubernetes.io/docs/user-guide/labels + // +optional + Labels map[string]string `json:"labels,omitempty"` + + // Annotations is an unstructured key value map stored with a resource that may be + // set by external tools to store and retrieve arbitrary metadata. They are not + // queryable and should be preserved when modifying objects. + // More info: http://kubernetes.io/docs/user-guide/annotations + // +optional + Annotations map[string]string `json:"annotations,omitempty"` +} + +// Secret holds secret data of a certain type. The total bytes of the values +// in the Data field must be less than MaxSecretSize bytes. +type Secret struct { + TypeMeta `json:",inline"` + ObjectMeta `json:"metadata"` + + // Data contains the secret data. Each key must consist of alphanumeric + // characters, '-', '_' or '.'. The serialized form of the secret data is a + // base64 encoded string, representing the arbitrary (possibly non-string) + // data value here. Described in https://tools.ietf.org/html/rfc4648#section-4 + // +optional + Data map[string][]byte `json:"data,omitempty"` +} + +// Status is a return value for calls that don't return other objects. +type Status struct { + TypeMeta `json:",inline"` + // Status of the operation. + // One of: "Success" or "Failure". + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + // +optional + Status string `json:"status,omitempty"` + + // A human-readable description of the status of this operation. + // +optional + Message string `json:"message,omitempty"` + + // A machine-readable description of why this operation is in the + // "Failure" status. If this value is empty there + // is no information available. A Reason clarifies an HTTP status + // code but does not override it. + // +optional + Reason string `json:"reason,omitempty"` + + // Extended data associated with the reason. Each reason may define its + // own extended details. This field is optional and the data returned + // is not guaranteed to conform to any schema except that defined by + // the reason type. + // +optional + Details *struct { + Name string `json:"name,omitempty"` + Kind string `json:"kind,omitempty"` + } `json:"details,omitempty"` + + // Suggested HTTP return code for this status, 0 if not set. + // +optional + Code int `json:"code,omitempty"` +} + +func (s *Status) Error() string { + return s.Message +} diff --git a/kube/client.go b/kube/client.go new file mode 100644 index 000000000..8bb8bc883 --- /dev/null +++ b/kube/client.go @@ -0,0 +1,170 @@ +// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package kube provides a client to interact with Kubernetes. +// This package is Tailscale-internal and not meant for external consumption. +// Further, the API should not be considered stable. +package kube + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sync" + "time" +) + +const ( + saPath = "/var/run/secrets/kubernetes.io/serviceaccount" + defaultURL = "https://kubernetes.default.svc" +) + +func readFile(n string) ([]byte, error) { + return os.ReadFile(filepath.Join(saPath, n)) +} + +// Client handles connections to Kubernetes. +// It expects to be run inside a cluster. +type Client struct { + mu sync.Mutex + url string + ns string + client *http.Client + token string + tokenExpiry time.Time +} + +// New returns a new client +func New() (*Client, error) { + ns, err := readFile("namespace") + if err != nil { + return nil, err + } + caCert, err := readFile("ca.crt") + if err != nil { + return nil, err + } + cp := x509.NewCertPool() + if ok := cp.AppendCertsFromPEM(caCert); !ok { + return nil, fmt.Errorf("kube: error in creating root cert pool") + } + return &Client{ + url: defaultURL, + ns: string(ns), + client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: cp, + }, + }, + }, + }, nil +} + +func (c *Client) expireToken() { + c.mu.Lock() + defer c.mu.Unlock() + c.tokenExpiry = time.Now() +} + +func (c *Client) getOrRenewToken() (string, error) { + c.mu.Lock() + defer c.mu.Unlock() + tk, te := c.token, c.tokenExpiry + if time.Now().Before(te) { + return tk, nil + } + + tkb, err := readFile("token") + if err != nil { + return "", err + } + c.token = string(tkb) + c.tokenExpiry = time.Now().Add(30 * time.Minute) + return c.token, nil +} + +func (c *Client) secretURL(name string) string { + if name == "" { + return fmt.Sprintf("%s/api/v1/namespaces/%s/secrets", c.url, c.ns) + } + return fmt.Sprintf("%s/api/v1/namespaces/%s/secrets/%s", c.url, c.ns, name) +} + +func getError(resp *http.Response) error { + if resp.StatusCode == 200 { + return nil + } + st := &Status{} + if err := json.NewDecoder(resp.Body).Decode(st); err != nil { + return err + } + return st +} + +func (c *Client) doRequest(ctx context.Context, method, url string, in, out interface{}) error { + tk, err := c.getOrRenewToken() + if err != nil { + return err + } + var body io.Reader + if in != nil { + var b bytes.Buffer + if err := json.NewEncoder(&b).Encode(in); err != nil { + return err + } + body = &b + } + req, err := http.NewRequestWithContext(ctx, method, url, body) + if err != nil { + return err + } + if body != nil { + req.Header.Add("Content-Type", "application/json") + } + req.Header.Add("Accept", "application/json") + req.Header.Add("Authorization", "Bearer "+tk) + resp, err := c.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if err := getError(resp); err != nil { + if st, ok := err.(*Status); ok && st.Code == 401 { + c.expireToken() + } + return err + } + if out != nil { + return json.NewDecoder(resp.Body).Decode(out) + } + return nil +} + +// GetSecret fetches the secret from the Kubernetes API. +func (c *Client) GetSecret(ctx context.Context, name string) (*Secret, error) { + s := &Secret{Data: make(map[string][]byte)} + if err := c.doRequest(ctx, "GET", c.secretURL(name), nil, s); err != nil { + return nil, err + } + return s, nil +} + +// CreateSecret creates a secret in the Kubernetes API. +func (c *Client) CreateSecret(ctx context.Context, s *Secret) error { + s.Namespace = c.ns + return c.doRequest(ctx, "POST", c.secretURL(""), s, nil) +} + +// UpdateSecret updates a secret in the Kubernetes API. +func (c *Client) UpdateSecret(ctx context.Context, s *Secret) error { + return c.doRequest(ctx, "PUT", c.secretURL(s.Name), s, nil) +}