archive
/
tailscale
mirror of https://github.com/tailscale/tailscale/
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

kube/{kubeapi,kubeclient},ipn/store/kubestore,cmd/{containerboot,k8s-operator}: emit kube store Events (#14112)

Browse Source 
Adds functionality to kube client to emit Events.
Updates kube store to emit Events when tailscaled state has been loaded, updated or if any errors where
encountered during those operations.
This should help in cases where an error related to state loading/updating caused the Pod to crash in a loop-
unlike logs of the originally failed container instance, Events associated with the Pod will still be
accessible even after N restarts.

Updates tailscale/tailscale#14080

Signed-off-by: Irbe Krumina <irbe@tailscale.com>
pull/14058/merge
Irbe Krumina 3 days ago committed by GitHub
parent da70a84a4b
commit 00517c8189
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 506 additions and 102 deletions
Show Stats Download Patch File Download Diff File

4
cmd/containerboot/kube.go
Unescape Escape View File

@ -61,7 +61,7 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
Path: "/data/authkey", Path: "/data/authkey",
}, },
} }
if err := kc.JSONPatchSecret(ctx, secretName, m); err != nil { if err := kc.JSONPatchResource(ctx, secretName, kubeclient.TypeSecrets, m); err != nil {
if s, ok := err.(*kubeapi.Status); ok && s.Code == http.StatusUnprocessableEntity { if s, ok := err.(*kubeapi.Status); ok && s.Code == http.StatusUnprocessableEntity {
// This is kubernetes-ese for "the field you asked to // This is kubernetes-ese for "the field you asked to
// delete already doesn't exist", aka no-op. // delete already doesn't exist", aka no-op.
@ -81,7 +81,7 @@ func initKubeClient(root string) {
kubeclient.SetRootPathForTesting(root) kubeclient.SetRootPathForTesting(root)
} }
var err error var err error
kc, err = kubeclient.New() kc, err = kubeclient.New("tailscale-container")
if err != nil { if err != nil {
log.Fatalf("Error creating kube client: %v", err) log.Fatalf("Error creating kube client: %v", err)
} }

2
cmd/containerboot/services.go
Unescape Escape View File

@ -389,7 +389,7 @@ func (ep *egressProxy) setStatus(ctx context.Context, status *egressservices.Sta
Path: fmt.Sprintf("/data/%s", egressservices.KeyEgressServices), Path: fmt.Sprintf("/data/%s", egressservices.KeyEgressServices),
Value: bs, Value: bs,
} }
if err := ep.kc.JSONPatchSecret(ctx, ep.stateSecret, []kubeclient.JSONPatch{patch}); err != nil { if err := ep.kc.JSONPatchResource(ctx, ep.stateSecret, kubeclient.TypeSecrets, []kubeclient.JSONPatch{patch}); err != nil {
return fmt.Errorf("error patching state Secret: %w", err) return fmt.Errorf("error patching state Secret: %w", err)
} }
ep.tailnetAddrs = n.NetMap.SelfNode.Addresses().AsSlice() ep.tailnetAddrs = n.NetMap.SelfNode.Addresses().AsSlice()

3
cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml
Unescape Escape View File

@ -16,6 +16,9 @@ rules:
- apiGroups: [""] - apiGroups: [""]
resources: ["secrets"] resources: ["secrets"]
verbs: ["create","delete","deletecollection","get","list","patch","update","watch"] verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "patch", "get"]
--- ---
apiVersion: rbac.authorization.k8s.io/v1 apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding kind: RoleBinding

8
cmd/k8s-operator/deploy/manifests/operator.yaml
Unescape Escape View File

@ -4703,6 +4703,14 @@ rules:
- patch - patch
- update - update
- watch - watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- get
--- ---
apiVersion: rbac.authorization.k8s.io/v1 apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding kind: RoleBinding

8
cmd/k8s-operator/deploy/manifests/proxy.yaml
Unescape Escape View File

@ -30,6 +30,14 @@ spec:
valueFrom: valueFrom:
fieldRef: fieldRef:
fieldPath: status.podIP fieldPath: status.podIP
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_UID
valueFrom:
fieldRef:
fieldPath: metadata.uid
securityContext: securityContext:
capabilities: capabilities:
add: add:

8
cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml
Unescape Escape View File

@ -24,3 +24,11 @@ spec:
valueFrom: valueFrom:
fieldRef: fieldRef:
fieldPath: status.podIP fieldPath: status.podIP
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_UID
valueFrom:
fieldRef:
fieldPath: metadata.uid

24
cmd/k8s-operator/proxygroup_specs.go
Unescape Escape View File

@ -126,15 +126,6 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
}, },
}, },
}, },
{
Name: "POD_NAME",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
// Secret is named after the pod.
FieldPath: "metadata.name",
},
},
},
{ {
Name: "TS_KUBE_SECRET", Name: "TS_KUBE_SECRET",
Value: "$(POD_NAME)", Value: "$(POD_NAME)",
@ -147,10 +138,6 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
Value: "/etc/tsconfig/$(POD_NAME)", Value: "/etc/tsconfig/$(POD_NAME)",
}, },
{
Name: "TS_USERSPACE",
Value: "false",
},
{ {
Name: "TS_INTERNAL_APP", Name: "TS_INTERNAL_APP",
Value: kubetypes.AppProxyGroupEgress, Value: kubetypes.AppProxyGroupEgress,
@ -171,7 +158,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
}) })
} }
return envs return append(c.Env, envs...)
}() }()
return ss, nil return ss, nil
@ -215,6 +202,15 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
return secrets return secrets
}(), }(),
}, },
{
APIGroups: []string{""},
Resources: []string{"events"},
Verbs: []string{
"create",
"patch",
"get",
},
},
}, },
} }
} }

4
cmd/k8s-operator/testutils_test.go
Unescape Escape View File

@ -70,6 +70,8 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
Env: []corev1.EnvVar{ Env: []corev1.EnvVar{
{Name: "TS_USERSPACE", Value: "false"}, {Name: "TS_USERSPACE", Value: "false"},
{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, {Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "TS_KUBE_SECRET", Value: opts.secretName}, {Name: "TS_KUBE_SECRET", Value: opts.secretName},
{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"}, {Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
}, },
@ -228,6 +230,8 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
Env: []corev1.EnvVar{ Env: []corev1.EnvVar{
{Name: "TS_USERSPACE", Value: "true"}, {Name: "TS_USERSPACE", Value: "true"},
{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, {Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "TS_KUBE_SECRET", Value: opts.secretName}, {Name: "TS_KUBE_SECRET", Value: opts.secretName},
{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"}, {Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"}, {Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"},

44
ipn/store/kubestore/store_kube.go
Unescape Escape View File

@ -7,6 +7,7 @@ package kubestore
import ( import (
"context" "context"
"fmt" "fmt"
"log"
"net" "net"
"os" "os"
"strings" "strings"
@ -19,8 +20,18 @@ import (
"tailscale.com/types/logger" "tailscale.com/types/logger"
) )
// TODO(irbekrm): should we bump this? should we have retries? See tailscale/tailscale#13024 const (
const timeout = 5 * time.Second // timeout is the timeout for a single state update that includes calls to the API server to write or read a
// state Secret and emit an Event.
timeout = 30 * time.Second
reasonTailscaleStateUpdated = "TailscaledStateUpdated"
reasonTailscaleStateLoaded = "TailscaleStateLoaded"
reasonTailscaleStateUpdateFailed = "TailscaleStateUpdateFailed"
reasonTailscaleStateLoadFailed = "TailscaleStateLoadFailed"
eventTypeWarning = "Warning"
eventTypeNormal = "Normal"
)
// Store is an ipn.StateStore that uses a Kubernetes Secret for persistence. // Store is an ipn.StateStore that uses a Kubernetes Secret for persistence.
type Store struct { type Store struct {
@ -35,7 +46,7 @@ type Store struct {
// New returns a new Store that persists to the named Secret. // New returns a new Store that persists to the named Secret.
func New(_ logger.Logf, secretName string) (*Store, error) { func New(_ logger.Logf, secretName string) (*Store, error) {
c, err := kubeclient.New() c, err := kubeclient.New("tailscale-state-store")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -72,13 +83,22 @@ func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) {
// WriteState implements the StateStore interface. // WriteState implements the StateStore interface.
func (s *Store) WriteState(id ipn.StateKey, bs []byte) (err error) { func (s *Store) WriteState(id ipn.StateKey, bs []byte) (err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer func() { defer func() {
if err == nil { if err == nil {
s.memory.WriteState(ipn.StateKey(sanitizeKey(id)), bs) s.memory.WriteState(ipn.StateKey(sanitizeKey(id)), bs)
} }
if err != nil {
if err := s.client.Event(ctx, eventTypeWarning, reasonTailscaleStateUpdateFailed, err.Error()); err != nil {
log.Printf("kubestore: error creating tailscaled state update Event: %v", err)
}
} else {
if err := s.client.Event(ctx, eventTypeNormal, reasonTailscaleStateUpdated, "Successfully updated tailscaled state Secret"); err != nil {
log.Printf("kubestore: error creating tailscaled state Event: %v", err)
}
}
cancel()
}() }()
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
secret, err := s.client.GetSecret(ctx, s.secretName) secret, err := s.client.GetSecret(ctx, s.secretName)
if err != nil { if err != nil {
@ -107,7 +127,7 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) (err error) {
Value: map[string][]byte{sanitizeKey(id): bs}, Value: map[string][]byte{sanitizeKey(id): bs},
}, },
} }
if err := s.client.JSONPatchSecret(ctx, s.secretName, m); err != nil { if err := s.client.JSONPatchResource(ctx, s.secretName, kubeclient.TypeSecrets, m); err != nil {
return fmt.Errorf("error patching Secret %s with a /data field: %v", s.secretName, err) return fmt.Errorf("error patching Secret %s with a /data field: %v", s.secretName, err)
} }
return nil return nil
@ -119,8 +139,8 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) (err error) {
Value: bs, Value: bs,
}, },
} }
if err := s.client.JSONPatchSecret(ctx, s.secretName, m); err != nil { if err := s.client.JSONPatchResource(ctx, s.secretName, kubeclient.TypeSecrets, m); err != nil {
return fmt.Errorf("error patching Secret %s with /data/%s field", s.secretName, sanitizeKey(id)) return fmt.Errorf("error patching Secret %s with /data/%s field: %v", s.secretName, sanitizeKey(id), err)
} }
return nil return nil
} }
@ -131,7 +151,7 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) (err error) {
return err return err
} }
func (s *Store) loadState() error { func (s *Store) loadState() (err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout) ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel() defer cancel()
@ -140,8 +160,14 @@ func (s *Store) loadState() error {
if st, ok := err.(*kubeapi.Status); ok && st.Code == 404 { if st, ok := err.(*kubeapi.Status); ok && st.Code == 404 {
return ipn.ErrStateNotExist return ipn.ErrStateNotExist
} }
if err := s.client.Event(ctx, eventTypeWarning, reasonTailscaleStateLoadFailed, err.Error()); err != nil {
log.Printf("kubestore: error creating Event: %v", err)
}
return err return err
} }
if err := s.client.Event(ctx, eventTypeNormal, reasonTailscaleStateLoaded, "Successfully loaded tailscaled state from Secret"); err != nil {
log.Printf("kubestore: error creating Event: %v", err)
}
s.memory.LoadFromMap(secret.Data) s.memory.LoadFromMap(secret.Data)
return nil return nil
} }

57
kube/kubeapi/api.go
Unescape Escape View File

@ -7,7 +7,9 @@
// dependency size for those consumers when adding anything new here. // dependency size for those consumers when adding anything new here.
package kubeapi package kubeapi
import "time" import (
"time"
)
// Note: The API types are copied from k8s.io/api{,machinery} to not introduce a // Note: The API types are copied from k8s.io/api{,machinery} to not introduce a
// module dependency on the Kubernetes API as it pulls in many more dependencies. // module dependency on the Kubernetes API as it pulls in many more dependencies.
@ -151,6 +153,57 @@ type Secret struct {
Data map[string][]byte `json:"data,omitempty"` Data map[string][]byte `json:"data,omitempty"`
} }
// Event contains a subset of fields from corev1.Event.
// https://github.com/kubernetes/api/blob/6cc44b8953ae704d6d9ec2adf32e7ae19199ea9f/core/v1/types.go#L7034
// It is copied here to avoid having to import kube libraries.
type Event struct {
TypeMeta `json:",inline"`
ObjectMeta `json:"metadata"`
Message string `json:"message,omitempty"`
Reason string `json:"reason,omitempty"`
Source EventSource `json:"source,omitempty"` // who is emitting this Event
Type string `json:"type,omitempty"` // Normal or Warning
// InvolvedObject is the subject of the Event. `kubectl describe` will, for most object types, display any
// currently present cluster Events matching the object (but you probably want to set UID for this to work).
InvolvedObject ObjectReference `json:"involvedObject"`
Count int32 `json:"count,omitempty"` // how many times Event was observed
FirstTimestamp time.Time `json:"firstTimestamp,omitempty"`
LastTimestamp time.Time `json:"lastTimestamp,omitempty"`
}
// EventSource includes a subset of fields from corev1.EventSource.
// https://github.com/kubernetes/api/blob/6cc44b8953ae704d6d9ec2adf32e7ae19199ea9f/core/v1/types.go#L7007
// It is copied here to avoid having to import kube libraries.
type EventSource struct {
// Component is the name of the component that is emitting the Event.
Component string `json:"component,omitempty"`
}
// ObjectReference contains a subset of fields from corev1.ObjectReference.
// https://github.com/kubernetes/api/blob/6cc44b8953ae704d6d9ec2adf32e7ae19199ea9f/core/v1/types.go#L6902
// It is copied here to avoid having to import kube libraries.
type ObjectReference struct {
// Kind of the referent.
// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
// +optional
Kind string `json:"kind,omitempty"`
// Namespace of the referent.
// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/
// +optional
Namespace string `json:"namespace,omitempty"`
// Name of the referent.
// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
// +optional
Name string `json:"name,omitempty"`
// UID of the referent.
// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids
// +optional
UID string `json:"uid,omitempty"`
// API version of the referent.
// +optional
APIVersion string `json:"apiVersion,omitempty"`
}
// Status is a return value for calls that don't return other objects. // Status is a return value for calls that don't return other objects.
type Status struct { type Status struct {
TypeMeta `json:",inline"` TypeMeta `json:",inline"`
@ -186,6 +239,6 @@ type Status struct {
Code int `json:"code,omitempty"` Code int `json:"code,omitempty"`
} }
func (s *Status) Error() string { func (s Status) Error() string {
return s.Message return s.Message
} }

289
kube/kubeclient/client.go
Unescape Escape View File

@ -23,16 +23,21 @@ import (
"net/url" "net/url"
"os" "os"
"path/filepath" "path/filepath"
"strings"
"sync" "sync"
"time" "time"
"tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeapi"
"tailscale.com/tstime"
"tailscale.com/util/multierr" "tailscale.com/util/multierr"
) )
const ( const (
saPath = "/var/run/secrets/kubernetes.io/serviceaccount" saPath = "/var/run/secrets/kubernetes.io/serviceaccount"
defaultURL = "https://kubernetes.default.svc" defaultURL = "https://kubernetes.default.svc"
TypeSecrets = "secrets"
typeEvents = "events"
) )
// rootPathForTests is set by tests to override the root path to the // rootPathForTests is set by tests to override the root path to the
@ -57,8 +62,13 @@ type Client interface {
GetSecret(context.Context, string) (*kubeapi.Secret, error) GetSecret(context.Context, string) (*kubeapi.Secret, error)
UpdateSecret(context.Context, *kubeapi.Secret) error UpdateSecret(context.Context, *kubeapi.Secret) error
CreateSecret(context.Context, *kubeapi.Secret) error CreateSecret(context.Context, *kubeapi.Secret) error
// Event attempts to ensure an event with the specified options associated with the Pod in which we are
// currently running. This is best effort - if the client is not able to create events, this operation will be a
// no-op. If there is already an Event with the given reason for the current Pod, it will get updated (only
// count and timestamp are expected to change), else a new event will be created.
Event(_ context.Context, typ, reason, msg string) error
StrategicMergePatchSecret(context.Context, string, *kubeapi.Secret, string) error StrategicMergePatchSecret(context.Context, string, *kubeapi.Secret, string) error
JSONPatchSecret(context.Context, string, []JSONPatch) error JSONPatchResource(_ context.Context, resourceName string, resourceType string, patches []JSONPatch) error
CheckSecretPermissions(context.Context, string) (bool, bool, error) CheckSecretPermissions(context.Context, string) (bool, bool, error)
SetDialer(dialer func(context.Context, string, string) (net.Conn, error)) SetDialer(dialer func(context.Context, string, string) (net.Conn, error))
SetURL(string) SetURL(string)
@ -66,15 +76,24 @@ type Client interface {
type client struct { type client struct {
mu sync.Mutex mu sync.Mutex
name string
url string url string
ns string podName string
podUID string
ns string // Pod namespace
client *http.Client client *http.Client
token string token string
tokenExpiry time.Time tokenExpiry time.Time
cl tstime.Clock
// hasEventsPerms is true if client can emit Events for the Pod in which it runs. If it is set to false any
// calls to Events() will be a no-op.
hasEventsPerms bool
// kubeAPIRequest sends a request to the kube API server. It can set to a fake in tests.
kubeAPIRequest kubeAPIRequestFunc
} }
// New returns a new client // New returns a new client
func New() (Client, error) { func New(name string) (Client, error) {
ns, err := readFile("namespace") ns, err := readFile("namespace")
if err != nil { if err != nil {
return nil, err return nil, err
@ -87,9 +106,11 @@ func New() (Client, error) {
if ok := cp.AppendCertsFromPEM(caCert); !ok { if ok := cp.AppendCertsFromPEM(caCert); !ok {
return nil, fmt.Errorf("kube: error in creating root cert pool") return nil, fmt.Errorf("kube: error in creating root cert pool")
} }
return &client{ c := &client{
url: defaultURL, url: defaultURL,
ns: string(ns), ns: string(ns),
name: name,
cl: tstime.DefaultClock{},
client: &http.Client{ client: &http.Client{
Transport: &http.Transport{ Transport: &http.Transport{
TLSClientConfig: &tls.Config{ TLSClientConfig: &tls.Config{
@ -97,7 +118,10 @@ func New() (Client, error) {
}, },
}, },
}, },
}, nil }
c.kubeAPIRequest = newKubeAPIRequest(c)
c.setEventPerms()
return c, nil
} }
// SetURL sets the URL to use for the Kubernetes API. // SetURL sets the URL to use for the Kubernetes API.
@ -115,14 +139,14 @@ func (c *client) SetDialer(dialer func(ctx context.Context, network, addr string
func (c *client) expireToken() { func (c *client) expireToken() {
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
c.tokenExpiry = time.Now() c.tokenExpiry = c.cl.Now()
} }
func (c *client) getOrRenewToken() (string, error) { func (c *client) getOrRenewToken() (string, error) {
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
tk, te := c.token, c.tokenExpiry tk, te := c.token, c.tokenExpiry
if time.Now().Before(te) { if c.cl.Now().Before(te) {
return tk, nil return tk, nil
} }
@ -131,17 +155,10 @@ func (c *client) getOrRenewToken() (string, error) {
return "", err return "", err
} }
c.token = string(tkb) c.token = string(tkb)
c.tokenExpiry = time.Now().Add(30 * time.Minute) c.tokenExpiry = c.cl.Now().Add(30 * time.Minute)
return c.token, nil return c.token, nil
} }
func (c *client) secretURL(name string) string {
if name == "" {
return fmt.Sprintf("%s/api/v1/namespaces/%s/secrets", c.url, c.ns)
}
return fmt.Sprintf("%s/api/v1/namespaces/%s/secrets/%s", c.url, c.ns, name)
}
func getError(resp *http.Response) error { func getError(resp *http.Response) error {
if resp.StatusCode == 200 || resp.StatusCode == 201 { if resp.StatusCode == 200 || resp.StatusCode == 201 {
// These are the only success codes returned by the Kubernetes API. // These are the only success codes returned by the Kubernetes API.
@ -161,36 +178,41 @@ func setHeader(key, value string) func(*http.Request) {
} }
} }
// doRequest performs an HTTP request to the Kubernetes API. type kubeAPIRequestFunc func(ctx context.Context, method, url string, in, out any, opts ...func(*http.Request)) error
// If in is not nil, it is expected to be a JSON-encodable object and will be
// sent as the request body. // newKubeAPIRequest returns a function that can perform an HTTP request to the Kubernetes API.
// If out is not nil, it is expected to be a pointer to an object that can be func newKubeAPIRequest(c *client) kubeAPIRequestFunc {
// decoded from JSON. // If in is not nil, it is expected to be a JSON-encodable object and will be
// If the request fails with a 401, the token is expired and a new one is // sent as the request body.
// requested. // If out is not nil, it is expected to be a pointer to an object that can be
func (c *client) doRequest(ctx context.Context, method, url string, in, out any, opts ...func(*http.Request)) error { // decoded from JSON.
req, err := c.newRequest(ctx, method, url, in) // If the request fails with a 401, the token is expired and a new one is
if err != nil { // requested.
return err f := func(ctx context.Context, method, url string, in, out any, opts ...func(*http.Request)) error {
} req, err := c.newRequest(ctx, method, url, in)
for _, opt := range opts { if err != nil {
opt(req) return err
}
resp, err := c.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if err := getError(resp); err != nil {
if st, ok := err.(*kubeapi.Status); ok && st.Code == 401 {
c.expireToken()
} }
return err for _, opt := range opts {
} opt(req)
if out != nil { }
return json.NewDecoder(resp.Body).Decode(out) resp, err := c.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if err := getError(resp); err != nil {
if st, ok := err.(*kubeapi.Status); ok && st.Code == 401 {
c.expireToken()
}
return err
}
if out != nil {
return json.NewDecoder(resp.Body).Decode(out)
}
return nil
} }
return nil return f
} }
func (c *client) newRequest(ctx context.Context, method, url string, in any) (*http.Request, error) { func (c *client) newRequest(ctx context.Context, method, url string, in any) (*http.Request, error) {
@ -226,7 +248,7 @@ func (c *client) newRequest(ctx context.Context, method, url string, in any) (*h
// GetSecret fetches the secret from the Kubernetes API. // GetSecret fetches the secret from the Kubernetes API.
func (c *client) GetSecret(ctx context.Context, name string) (*kubeapi.Secret, error) { func (c *client) GetSecret(ctx context.Context, name string) (*kubeapi.Secret, error) {
s := &kubeapi.Secret{Data: make(map[string][]byte)} s := &kubeapi.Secret{Data: make(map[string][]byte)}
if err := c.doRequest(ctx, "GET", c.secretURL(name), nil, s); err != nil { if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, TypeSecrets), nil, s); err != nil {
return nil, err return nil, err
} }
return s, nil return s, nil
@ -235,16 +257,16 @@ func (c *client) GetSecret(ctx context.Context, name string) (*kubeapi.Secret, e
// CreateSecret creates a secret in the Kubernetes API. // CreateSecret creates a secret in the Kubernetes API.
func (c *client) CreateSecret(ctx context.Context, s *kubeapi.Secret) error { func (c *client) CreateSecret(ctx context.Context, s *kubeapi.Secret) error {
s.Namespace = c.ns s.Namespace = c.ns
return c.doRequest(ctx, "POST", c.secretURL(""), s, nil) return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", TypeSecrets), s, nil)
} }
// UpdateSecret updates a secret in the Kubernetes API. // UpdateSecret updates a secret in the Kubernetes API.
func (c *client) UpdateSecret(ctx context.Context, s *kubeapi.Secret) error { func (c *client) UpdateSecret(ctx context.Context, s *kubeapi.Secret) error {
return c.doRequest(ctx, "PUT", c.secretURL(s.Name), s, nil) return c.kubeAPIRequest(ctx, "PUT", c.resourceURL(s.Name, TypeSecrets), s, nil)
} }
// JSONPatch is a JSON patch operation. // JSONPatch is a JSON patch operation.
// It currently (2023-03-02) only supports "add" and "remove" operations. // It currently (2024-11-15) only supports "add", "remove" and "replace" operations.
// //
// https://tools.ietf.org/html/rfc6902 // https://tools.ietf.org/html/rfc6902
type JSONPatch struct { type JSONPatch struct {
@ -253,22 +275,22 @@ type JSONPatch struct {
Value any `json:"value,omitempty"` Value any `json:"value,omitempty"`
} }
// JSONPatchSecret updates a secret in the Kubernetes API using a JSON patch. // JSONPatchResource updates a resource in the Kubernetes API using a JSON patch.
// It currently (2023-03-02) only supports "add" and "remove" operations. // It currently (2024-11-15) only supports "add", "remove" and "replace" operations.
func (c *client) JSONPatchSecret(ctx context.Context, name string, patch []JSONPatch) error { func (c *client) JSONPatchResource(ctx context.Context, name, typ string, patches []JSONPatch) error {
for _, p := range patch { for _, p := range patches {
if p.Op != "remove" && p.Op != "add" && p.Op != "replace" { if p.Op != "remove" && p.Op != "add" && p.Op != "replace" {
return fmt.Errorf("unsupported JSON patch operation: %q", p.Op) return fmt.Errorf("unsupported JSON patch operation: %q", p.Op)
} }
} }
return c.doRequest(ctx, "PATCH", c.secretURL(name), patch, nil, setHeader("Content-Type", "application/json-patch+json")) return c.kubeAPIRequest(ctx, "PATCH", c.resourceURL(name, typ), patches, nil, setHeader("Content-Type", "application/json-patch+json"))
} }
// StrategicMergePatchSecret updates a secret in the Kubernetes API using a // StrategicMergePatchSecret updates a secret in the Kubernetes API using a
// strategic merge patch. // strategic merge patch.
// If a fieldManager is provided, it will be used to track the patch. // If a fieldManager is provided, it will be used to track the patch.
func (c *client) StrategicMergePatchSecret(ctx context.Context, name string, s *kubeapi.Secret, fieldManager string) error { func (c *client) StrategicMergePatchSecret(ctx context.Context, name string, s *kubeapi.Secret, fieldManager string) error {
surl := c.secretURL(name) surl := c.resourceURL(name, TypeSecrets)
if fieldManager != "" { if fieldManager != "" {
uv := url.Values{ uv := url.Values{
"fieldManager": {fieldManager}, "fieldManager": {fieldManager},
@ -277,7 +299,66 @@ func (c *client) StrategicMergePatchSecret(ctx context.Context, name string, s *
} }
s.Namespace = c.ns s.Namespace = c.ns
s.Name = name s.Name = name
return c.doRequest(ctx, "PATCH", surl, s, nil, setHeader("Content-Type", "application/strategic-merge-patch+json")) return c.kubeAPIRequest(ctx, "PATCH", surl, s, nil, setHeader("Content-Type", "application/strategic-merge-patch+json"))
}
// Event tries to ensure an Event associated with the Pod in which we are running. It is best effort - the event will be
// created if the kube client on startup was able to determine the name and UID of this Pod from POD_NAME,POD_UID env
// vars and if permissions check for event creation succeeded. Events are keyed on opts.Reason- if an Event for the
// current Pod with that reason already exists, its count and first timestamp will be updated, else a new Event will be
// created.
func (c *client) Event(ctx context.Context, typ, reason, msg string) error {
if !c.hasEventsPerms {
return nil
}
name := c.nameForEvent(reason)
ev, err := c.getEvent(ctx, name)
now := c.cl.Now()
if err != nil {
if !IsNotFoundErr(err) {
return err
}
// Event not found - create it
ev := kubeapi.Event{
ObjectMeta: kubeapi.ObjectMeta{
Name: name,
Namespace: c.ns,
},
Type: typ,
Reason: reason,
Message: msg,
Source: kubeapi.EventSource{
Component: c.name,
},
InvolvedObject: kubeapi.ObjectReference{
Name: c.podName,
Namespace: c.ns,
UID: c.podUID,
Kind: "Pod",
APIVersion: "v1",
},
FirstTimestamp: now,
LastTimestamp: now,
Count: 1,
}
return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", typeEvents), &ev, nil)
}
// If the Event already exists, we patch its count and last timestamp. This ensures that when users run 'kubectl
// describe pod...', they see the event just once (but with a message of how many times it has appeared over
// last timestamp - first timestamp period of time).
count := ev.Count + 1
countPatch := JSONPatch{
Op: "replace",
Value: count,
Path: "/count",
}
tsPatch := JSONPatch{
Op: "replace",
Value: now,
Path: "/lastTimestamp",
}
return c.JSONPatchResource(ctx, name, typeEvents, []JSONPatch{countPatch, tsPatch})
} }
// CheckSecretPermissions checks the secret access permissions of the current // CheckSecretPermissions checks the secret access permissions of the current
@ -293,7 +374,7 @@ func (c *client) StrategicMergePatchSecret(ctx context.Context, name string, s *
func (c *client) CheckSecretPermissions(ctx context.Context, secretName string) (canPatch, canCreate bool, err error) { func (c *client) CheckSecretPermissions(ctx context.Context, secretName string) (canPatch, canCreate bool, err error) {
var errs []error var errs []error
for _, verb := range []string{"get", "update"} { for _, verb := range []string{"get", "update"} {
ok, err := c.checkPermission(ctx, verb, secretName) ok, err := c.checkPermission(ctx, verb, TypeSecrets, secretName)
if err != nil { if err != nil {
log.Printf("error checking %s permission on secret %s: %v", verb, secretName, err) log.Printf("error checking %s permission on secret %s: %v", verb, secretName, err)
} else if !ok { } else if !ok {
@ -303,12 +384,12 @@ func (c *client) CheckSecretPermissions(ctx context.Context, secretName string)
if len(errs) > 0 { if len(errs) > 0 {
return false, false, multierr.New(errs...) return false, false, multierr.New(errs...)
} }
canPatch, err = c.checkPermission(ctx, "patch", secretName) canPatch, err = c.checkPermission(ctx, "patch", TypeSecrets, secretName)
if err != nil { if err != nil {
log.Printf("error checking patch permission on secret %s: %v", secretName, err) log.Printf("error checking patch permission on secret %s: %v", secretName, err)
return false, false, nil return false, false, nil
} }
canCreate, err = c.checkPermission(ctx, "create", secretName) canCreate, err = c.checkPermission(ctx, "create", TypeSecrets, secretName)
if err != nil { if err != nil {
log.Printf("error checking create permission on secret %s: %v", secretName, err) log.Printf("error checking create permission on secret %s: %v", secretName, err)
return false, false, nil return false, false, nil
@ -316,19 +397,64 @@ func (c *client) CheckSecretPermissions(ctx context.Context, secretName string)
return canPatch, canCreate, nil return canPatch, canCreate, nil
} }
// checkPermission reports whether the current pod has permission to use the func IsNotFoundErr(err error) bool {
// given verb (e.g. get, update, patch, create) on secretName. if st, ok := err.(*kubeapi.Status); ok && st.Code == 404 {
func (c *client) checkPermission(ctx context.Context, verb, secretName string) (bool, error) { return true
}
return false
}
// setEventPerms checks whether this client will be able to write tailscaled Events to its Pod and updates the state
// accordingly. If it determines that the client can not write Events, any subsequent calls to client.Event will be a
// no-op.
func (c *client) setEventPerms() {
name := os.Getenv("POD_NAME")
uid := os.Getenv("POD_UID")
hasPerms := false
defer func() {
c.podName = name
c.podUID = uid
c.hasEventsPerms = hasPerms
if !hasPerms {
log.Printf(`kubeclient: this client is not able to write tailscaled Events to the Pod in which it is running.
To help with future debugging you can make it able write Events by giving it get,create,patch permissions for Events in the Pod namespace
and setting POD_NAME, POD_UID env vars for the Pod.`)
}
}()
if name == "" || uid == "" {
return
}
for _, verb := range []string{"get", "create", "patch"} {
can, err := c.checkPermission(context.Background(), verb, typeEvents, "")
if err != nil {
log.Printf("kubeclient: error checking Events permissions: %v", err)
return
}
if !can {
return
}
}
hasPerms = true
return
}
// checkPermission reports whether the current pod has permission to use the given verb (e.g. get, update, patch,
// create) on the given resource type. If name is not an empty string, will check the check will be for resource with
// the given name only.
func (c *client) checkPermission(ctx context.Context, verb, typ, name string) (bool, error) {
ra := map[string]any{
"namespace": c.ns,
"verb": verb,
"resource": typ,
}
if name != "" {
ra["name"] = name
}
sar := map[string]any{ sar := map[string]any{
"apiVersion": "authorization.k8s.io/v1", "apiVersion": "authorization.k8s.io/v1",
"kind": "SelfSubjectAccessReview", "kind": "SelfSubjectAccessReview",
"spec": map[string]any{ "spec": map[string]any{
"resourceAttributes": map[string]any{ "resourceAttributes": ra,
"namespace": c.ns,
"verb": verb,
"resource": "secrets",
"name": secretName,
},
}, },
} }
var res struct { var res struct {
@ -337,15 +463,32 @@ func (c *client) checkPermission(ctx context.Context, verb, secretName string) (
} `json:"status"` } `json:"status"`
} }
url := c.url + "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews" url := c.url + "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews"
if err := c.doRequest(ctx, "POST", url, sar, &res); err != nil { if err := c.kubeAPIRequest(ctx, "POST", url, sar, &res); err != nil {
return false, err return false, err
} }
return res.Status.Allowed, nil return res.Status.Allowed, nil
} }
func IsNotFoundErr(err error) bool { // resourceURL returns a URL that can be used to interact with the given resource type and, if name is not empty string,
if st, ok := err.(*kubeapi.Status); ok && st.Code == 404 { // the named resource of that type.
return true // Note that this only works for core/v1 resource types.
func (c *client) resourceURL(name, typ string) string {
if name == "" {
return fmt.Sprintf("%s/api/v1/namespaces/%s/%s", c.url, c.ns, typ)
} }
return false return fmt.Sprintf("%s/api/v1/namespaces/%s/%s/%s", c.url, c.ns, typ, name)
}
// nameForEvent returns a name for the Event that uniquely identifies Event with that reason for the current Pod.
func (c *client) nameForEvent(reason string) string {
return fmt.Sprintf("%s.%s.%s", c.podName, c.podUID, strings.ToLower(reason))
}
// getEvent fetches the event from the Kubernetes API.
func (c *client) getEvent(ctx context.Context, name string) (*kubeapi.Event, error) {
e := &kubeapi.Event{}
if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, typeEvents), nil, e); err != nil {
return nil, err
}
return e, nil
} }

151
kube/kubeclient/client_test.go
Unescape Escape View File

@ -0,0 +1,151 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package kubeclient
import (
"context"
"encoding/json"
"net/http"
"testing"
"github.com/google/go-cmp/cmp"
"tailscale.com/kube/kubeapi"
"tailscale.com/tstest"
)
func Test_client_Event(t *testing.T) {
cl := &tstest.Clock{}
tests := []struct {
name string
typ string
reason string
msg string
argSets []args
wantErr bool
}{
{
name: "new_event_gets_created",
typ: "Normal",
reason: "TestReason",
msg: "TestMessage",
argSets: []args{
{ // request to GET event returns not found
wantsMethod: "GET",
wantsURL: "test-apiserver/api/v1/namespaces/test-ns/events/test-pod.test-uid.testreason",
setErr: &kubeapi.Status{Code: 404},
},
{ // sends POST request to create event
wantsMethod: "POST",
wantsURL: "test-apiserver/api/v1/namespaces/test-ns/events",
wantsIn: &kubeapi.Event{
ObjectMeta: kubeapi.ObjectMeta{
Name: "test-pod.test-uid.testreason",
Namespace: "test-ns",
},
Type: "Normal",
Reason: "TestReason",
Message: "TestMessage",
Source: kubeapi.EventSource{
Component: "test-client",
},
InvolvedObject: kubeapi.ObjectReference{
Name: "test-pod",
UID: "test-uid",
Namespace: "test-ns",
APIVersion: "v1",
Kind: "Pod",
},
FirstTimestamp: cl.Now(),
LastTimestamp: cl.Now(),
Count: 1,
},
},
},
},
{
name: "existing_event_gets_patched",
typ: "Warning",
reason: "TestReason",
msg: "TestMsg",
argSets: []args{
{ // request to GET event does not error - this is enough to assume that event exists
wantsMethod: "GET",
wantsURL: "test-apiserver/api/v1/namespaces/test-ns/events/test-pod.test-uid.testreason",
setOut: []byte(`{"count":2}`),
},
{ // sends PATCH request to update the event
wantsMethod: "PATCH",
wantsURL: "test-apiserver/api/v1/namespaces/test-ns/events/test-pod.test-uid.testreason",
wantsIn: []JSONPatch{
{Op: "replace", Path: "/count", Value: int32(3)},
{Op: "replace", Path: "/lastTimestamp", Value: cl.Now()},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := &client{
cl: cl,
name: "test-client",
podName: "test-pod",
podUID: "test-uid",
url: "test-apiserver",
ns: "test-ns",
kubeAPIRequest: fakeKubeAPIRequest(t, tt.argSets),
hasEventsPerms: true,
}
if err := c.Event(context.Background(), tt.typ, tt.reason, tt.msg); (err != nil) != tt.wantErr {
t.Errorf("client.Event() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
// args is a set of values for testing a single call to client.kubeAPIRequest.
type args struct {
// wantsMethod is the expected value of 'method' arg.
wantsMethod string
// wantsURL is the expected value of 'url' arg.
wantsURL string
// wantsIn is the expected value of 'in' arg.
wantsIn any
// setOut can be set to a byte slice representing valid JSON. If set 'out' arg will get set to the unmarshalled
// JSON object.
setOut []byte
// setErr is the error that kubeAPIRequest will return.
setErr error
}
// fakeKubeAPIRequest can be used to test that a series of calls to client.kubeAPIRequest gets called with expected
// values and to set these calls to return preconfigured values. 'argSets' should be set to a slice of expected
// arguments and should-be return values of a series of kubeAPIRequest calls.
func fakeKubeAPIRequest(t *testing.T, argSets []args) kubeAPIRequestFunc {
count := 0
f := func(ctx context.Context, gotMethod, gotUrl string, gotIn, gotOut any, opts ...func(*http.Request)) error {
t.Helper()
if count >= len(argSets) {
t.Fatalf("unexpected call to client.kubeAPIRequest, expected %d calls, but got a %dth call", len(argSets), count+1)
}
a := argSets[count]
if gotMethod != a.wantsMethod {
t.Errorf("[%d] got method %q, wants method %q", count, gotMethod, a.wantsMethod)
}
if gotUrl != a.wantsURL {
t.Errorf("[%d] got URL %q, wants URL %q", count, gotMethod, a.wantsMethod)
}
if d := cmp.Diff(gotIn, a.wantsIn); d != "" {
t.Errorf("[%d] unexpected payload (-want + got):\n%s", count, d)
}
if len(a.setOut) != 0 {
if err := json.Unmarshal(a.setOut, gotOut); err != nil {
t.Fatalf("[%d] error unmarshalling output: %v", count, err)
}
}
count++
return a.setErr
}
return f
}

6
kube/kubeclient/fake_client.go
Unescape Escape View File

@ -29,7 +29,11 @@ func (fc *FakeClient) SetDialer(dialer func(ctx context.Context, network, addr s
func (fc *FakeClient) StrategicMergePatchSecret(context.Context, string, *kubeapi.Secret, string) error { func (fc *FakeClient) StrategicMergePatchSecret(context.Context, string, *kubeapi.Secret, string) error {
return nil return nil
} }
func (fc *FakeClient) JSONPatchSecret(context.Context, string, []JSONPatch) error { func (fc *FakeClient) Event(context.Context, string, string, string) error {
return nil
}
func (fc *FakeClient) JSONPatchResource(context.Context, string, string, []JSONPatch) error {
return nil return nil
} }
func (fc *FakeClient) UpdateSecret(context.Context, *kubeapi.Secret) error { return nil } func (fc *FakeClient) UpdateSecret(context.Context, *kubeapi.Secret) error { return nil }

Write Preview
Loading…
Cancel
Save