archive
/
tailscale
mirror of https://github.com/tailscale/tailscale/
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

cmd/{containerboot,k8s-operator}: use state Secret for checking device auth (#16328)

Browse Source 
Previously, the operator checked the ProxyGroup status fields for
information on how many of the proxies had successfully authed. Use
their state Secrets instead as a more reliable source of truth.

containerboot has written device_fqdn and device_ips keys to the
state Secret since inception, and pod_uid since 1.78.0, so there's
no need to use the API for that data. Read it from the state Secret
for consistency. However, to ensure we don't read data from a
previous run of containerboot, make sure we reset containerboot's
state keys on startup.

One other knock-on effect of that is ProxyGroups can briefly be
marked not Ready while a Pod is restarting. Introduce a new
ProxyGroupAvailable condition to more accurately reflect
when downstream controllers can implement flows that rely on a
ProxyGroup having at least 1 proxy Pod running.

Fixes #16327

Change-Id: I026c18e9d23e87109a471a87b8e4fb6271716a66

Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>
pull/16409/head
Tom Proctor 5 months ago committed by GitHub
parent f81baa2d56
commit 711698f5a9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 373 additions and 202 deletions
Show Stats Download Patch File Download Diff File

42
cmd/containerboot/kube.go
Unescape Escape View File

@ -18,12 +18,15 @@ import (
"time" "time"
"tailscale.com/ipn" "tailscale.com/ipn"
"tailscale.com/kube/egressservices"
"tailscale.com/kube/ingressservices"
"tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeapi"
"tailscale.com/kube/kubeclient" "tailscale.com/kube/kubeclient"
"tailscale.com/kube/kubetypes" "tailscale.com/kube/kubetypes"
"tailscale.com/logtail/backoff" "tailscale.com/logtail/backoff"
"tailscale.com/tailcfg" "tailscale.com/tailcfg"
"tailscale.com/types/logger" "tailscale.com/types/logger"
"tailscale.com/util/set"
) )
// kubeClient is a wrapper around Tailscale's internal kube client that knows how to talk to the kube API server. We use // kubeClient is a wrapper around Tailscale's internal kube client that knows how to talk to the kube API server. We use
@ -117,20 +120,39 @@ func (kc *kubeClient) deleteAuthKey(ctx context.Context) error {
return nil return nil
} }
// storeCapVerUID stores the current capability version of tailscale and, if provided, UID of the Pod in the tailscale // resetContainerbootState resets state from previous runs of containerboot to
// state Secret. // ensure the operator doesn't use stale state when a Pod is first recreated.
// These two fields are used by the Kubernetes Operator to observe the current capability version of tailscaled running in this container. func (kc *kubeClient) resetContainerbootState(ctx context.Context, podUID string) error {
func (kc *kubeClient) storeCapVerUID(ctx context.Context, podUID string) error { existingSecret, err := kc.GetSecret(ctx, kc.stateSecret)
capVerS := fmt.Sprintf("%d", tailcfg.CurrentCapabilityVersion) if err != nil {
d := map[string][]byte{ return fmt.Errorf("failed to read state Secret %q to reset state: %w", kc.stateSecret, err)
kubetypes.KeyCapVer: []byte(capVerS), }
s := &kubeapi.Secret{
Data: map[string][]byte{
kubetypes.KeyCapVer: fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion),
},
} }
if podUID != "" { if podUID != "" {
d[kubetypes.KeyPodUID] = []byte(podUID) s.Data[kubetypes.KeyPodUID] = []byte(podUID)
} }
s := &kubeapi.Secret{
Data: d, toClear := set.SetOf([]string{
kubetypes.KeyDeviceID,
kubetypes.KeyDeviceFQDN,
kubetypes.KeyDeviceIPs,
kubetypes.KeyHTTPSEndpoint,
egressservices.KeyEgressServices,
ingressservices.IngressConfigKey,
})
for key := range existingSecret.Data {
if toClear.Contains(key) {
// It's fine to leave the key in place as a debugging breadcrumb,
// it should get a new value soon.
s.Data[key] = nil
}
} }
return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container") return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
} }

80
cmd/containerboot/kube_test.go
Unescape Escape View File

@ -8,13 +8,18 @@ package main
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"testing" "testing"
"time" "time"
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"tailscale.com/ipn" "tailscale.com/ipn"
"tailscale.com/kube/egressservices"
"tailscale.com/kube/ingressservices"
"tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeapi"
"tailscale.com/kube/kubeclient" "tailscale.com/kube/kubeclient"
"tailscale.com/kube/kubetypes"
"tailscale.com/tailcfg"
) )
func TestSetupKube(t *testing.T) { func TestSetupKube(t *testing.T) {
@ -238,3 +243,78 @@ func TestWaitForConsistentState(t *testing.T) {
t.Fatalf("expected nil, got %v", err) t.Fatalf("expected nil, got %v", err)
} }
} }
func TestResetContainerbootState(t *testing.T) {
capver := fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion)
for name, tc := range map[string]struct {
podUID string
initial map[string][]byte
expected map[string][]byte
}{
"empty_initial": {
podUID: "1234",
initial: map[string][]byte{},
expected: map[string][]byte{
kubetypes.KeyCapVer: capver,
kubetypes.KeyPodUID: []byte("1234"),
},
},
"empty_initial_no_pod_uid": {
initial: map[string][]byte{},
expected: map[string][]byte{
kubetypes.KeyCapVer: capver,
},
},
"only_relevant_keys_updated": {
podUID: "1234",
initial: map[string][]byte{
kubetypes.KeyCapVer: []byte("1"),
kubetypes.KeyPodUID: []byte("5678"),
kubetypes.KeyDeviceID: []byte("device-id"),
kubetypes.KeyDeviceFQDN: []byte("device-fqdn"),
kubetypes.KeyDeviceIPs: []byte(`["192.0.2.1"]`),
kubetypes.KeyHTTPSEndpoint: []byte("https://example.com"),
egressservices.KeyEgressServices: []byte("egress-services"),
ingressservices.IngressConfigKey: []byte("ingress-config"),
"_current-profile": []byte("current-profile"),
"_machinekey": []byte("machine-key"),
"_profiles": []byte("profiles"),
"_serve_e0ce": []byte("serve-e0ce"),
"profile-e0ce": []byte("profile-e0ce"),
},
expected: map[string][]byte{
kubetypes.KeyCapVer: capver,
kubetypes.KeyPodUID: []byte("1234"),
// Cleared keys.
kubetypes.KeyDeviceID: nil,
kubetypes.KeyDeviceFQDN: nil,
kubetypes.KeyDeviceIPs: nil,
kubetypes.KeyHTTPSEndpoint: nil,
egressservices.KeyEgressServices: nil,
ingressservices.IngressConfigKey: nil,
// Tailscaled keys not included in patch.
},
},
} {
t.Run(name, func(t *testing.T) {
var actual map[string][]byte
kc := &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
return &kubeapi.Secret{
Data: tc.initial,
}, nil
},
StrategicMergePatchSecretImpl: func(ctx context.Context, name string, secret *kubeapi.Secret, _ string) error {
actual = secret.Data
return nil
},
}}
if err := kc.resetContainerbootState(context.Background(), tc.podUID); err != nil {
t.Fatalf("resetContainerbootState() error = %v", err)
}
if diff := cmp.Diff(tc.expected, actual); diff != "" {
t.Errorf("resetContainerbootState() mismatch (-want +got):\n%s", diff)
}
})
}
}

19
cmd/containerboot/main.go
Unescape Escape View File

@ -188,6 +188,14 @@ func run() error {
if err := cfg.setupKube(bootCtx, kc); err != nil { if err := cfg.setupKube(bootCtx, kc); err != nil {
return fmt.Errorf("error setting up for running on Kubernetes: %w", err) return fmt.Errorf("error setting up for running on Kubernetes: %w", err)
} }
// Clear out any state from previous runs of containerboot. Check
// hasKubeStateStore because although we know we're in kube, that
// doesn't guarantee the state store is properly configured.
if hasKubeStateStore(cfg) {
if err := kc.resetContainerbootState(bootCtx, cfg.PodUID); err != nil {
return fmt.Errorf("error clearing previous state from Secret: %w", err)
}
}
} }
client, daemonProcess, err := startTailscaled(bootCtx, cfg) client, daemonProcess, err := startTailscaled(bootCtx, cfg)
@ -367,11 +375,6 @@ authLoop:
if err := client.SetServeConfig(ctx, new(ipn.ServeConfig)); err != nil { if err := client.SetServeConfig(ctx, new(ipn.ServeConfig)); err != nil {
return fmt.Errorf("failed to unset serve config: %w", err) return fmt.Errorf("failed to unset serve config: %w", err)
} }
if hasKubeStateStore(cfg) {
if err := kc.storeHTTPSEndpoint(ctx, ""); err != nil {
return fmt.Errorf("failed to update HTTPS endpoint in tailscale state: %w", err)
}
}
} }
if hasKubeStateStore(cfg) && isTwoStepConfigAuthOnce(cfg) { if hasKubeStateStore(cfg) && isTwoStepConfigAuthOnce(cfg) {
@ -384,12 +387,6 @@ authLoop:
} }
} }
if hasKubeStateStore(cfg) {
if err := kc.storeCapVerUID(ctx, cfg.PodUID); err != nil {
return fmt.Errorf("storing capability version and UID: %w", err)
}
}
w, err = client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState) w, err = client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState)
if err != nil { if err != nil {
return fmt.Errorf("rewatching tailscaled for updates after auth: %w", err) return fmt.Errorf("rewatching tailscaled for updates after auth: %w", err)

132
cmd/containerboot/main_test.go
Unescape Escape View File

@ -460,6 +460,7 @@ func TestContainerBoot(t *testing.T) {
Env: map[string]string{ Env: map[string]string{
"KUBERNETES_SERVICE_HOST": env.kube.Host, "KUBERNETES_SERVICE_HOST": env.kube.Host,
"KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port,
"POD_UID": "some-pod-uid",
}, },
KubeSecret: map[string]string{ KubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
@ -471,17 +472,20 @@ func TestContainerBoot(t *testing.T) {
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
}, },
WantKubeSecret: map[string]string{ WantKubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
kubetypes.KeyCapVer: capver,
kubetypes.KeyPodUID: "some-pod-uid",
}, },
}, },
{ {
Notify: runningNotify, Notify: runningNotify,
WantKubeSecret: map[string]string{ WantKubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
"device_fqdn": "test-node.test.ts.net", "device_fqdn": "test-node.test.ts.net",
"device_id": "myID", "device_id": "myID",
"device_ips": `["100.64.0.1"]`, "device_ips": `["100.64.0.1"]`,
"tailscale_capver": capver, kubetypes.KeyCapVer: capver,
kubetypes.KeyPodUID: "some-pod-uid",
}, },
}, },
}, },
@ -554,7 +558,8 @@ func TestContainerBoot(t *testing.T) {
"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking", "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
}, },
WantKubeSecret: map[string]string{ WantKubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
kubetypes.KeyCapVer: capver,
}, },
}, },
{ {
@ -565,7 +570,8 @@ func TestContainerBoot(t *testing.T) {
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
}, },
WantKubeSecret: map[string]string{ WantKubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
kubetypes.KeyCapVer: capver,
}, },
}, },
{ {
@ -574,10 +580,10 @@ func TestContainerBoot(t *testing.T) {
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
}, },
WantKubeSecret: map[string]string{ WantKubeSecret: map[string]string{
"device_fqdn": "test-node.test.ts.net", "device_fqdn": "test-node.test.ts.net",
"device_id": "myID", "device_id": "myID",
"device_ips": `["100.64.0.1"]`, "device_ips": `["100.64.0.1"]`,
"tailscale_capver": capver, kubetypes.KeyCapVer: capver,
}, },
}, },
}, },
@ -599,17 +605,18 @@ func TestContainerBoot(t *testing.T) {
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
}, },
WantKubeSecret: map[string]string{ WantKubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
kubetypes.KeyCapVer: capver,
}, },
}, },
{ {
Notify: runningNotify, Notify: runningNotify,
WantKubeSecret: map[string]string{ WantKubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
"device_fqdn": "test-node.test.ts.net", "device_fqdn": "test-node.test.ts.net",
"device_id": "myID", "device_id": "myID",
"device_ips": `["100.64.0.1"]`, "device_ips": `["100.64.0.1"]`,
"tailscale_capver": capver, kubetypes.KeyCapVer: capver,
}, },
}, },
{ {
@ -624,11 +631,11 @@ func TestContainerBoot(t *testing.T) {
}, },
}, },
WantKubeSecret: map[string]string{ WantKubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
"device_fqdn": "new-name.test.ts.net", "device_fqdn": "new-name.test.ts.net",
"device_id": "newID", "device_id": "newID",
"device_ips": `["100.64.0.1"]`, "device_ips": `["100.64.0.1"]`,
"tailscale_capver": capver, kubetypes.KeyCapVer: capver,
}, },
}, },
}, },
@ -912,18 +919,19 @@ func TestContainerBoot(t *testing.T) {
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
}, },
WantKubeSecret: map[string]string{ WantKubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
kubetypes.KeyCapVer: capver,
}, },
}, },
{ {
Notify: runningNotify, Notify: runningNotify,
WantKubeSecret: map[string]string{ WantKubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
"device_fqdn": "test-node.test.ts.net", "device_fqdn": "test-node.test.ts.net",
"device_id": "myID", "device_id": "myID",
"device_ips": `["100.64.0.1"]`, "device_ips": `["100.64.0.1"]`,
"https_endpoint": "no-https", "https_endpoint": "no-https",
"tailscale_capver": capver, kubetypes.KeyCapVer: capver,
}, },
}, },
}, },
@ -947,7 +955,8 @@ func TestContainerBoot(t *testing.T) {
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
}, },
WantKubeSecret: map[string]string{ WantKubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
kubetypes.KeyCapVer: capver,
}, },
EndpointStatuses: map[string]int{ EndpointStatuses: map[string]int{
egressSvcTerminateURL(env.localAddrPort): 200, egressSvcTerminateURL(env.localAddrPort): 200,
@ -956,12 +965,12 @@ func TestContainerBoot(t *testing.T) {
{ {
Notify: runningNotify, Notify: runningNotify,
WantKubeSecret: map[string]string{ WantKubeSecret: map[string]string{
"egress-services": mustBase64(t, egressStatus), "egress-services": string(mustJSON(t, egressStatus)),
"authkey": "tskey-key", "authkey": "tskey-key",
"device_fqdn": "test-node.test.ts.net", "device_fqdn": "test-node.test.ts.net",
"device_id": "myID", "device_id": "myID",
"device_ips": `["100.64.0.1"]`, "device_ips": `["100.64.0.1"]`,
"tailscale_capver": capver, kubetypes.KeyCapVer: capver,
}, },
EndpointStatuses: map[string]int{ EndpointStatuses: map[string]int{
egressSvcTerminateURL(env.localAddrPort): 200, egressSvcTerminateURL(env.localAddrPort): 200,
@ -1002,7 +1011,8 @@ func TestContainerBoot(t *testing.T) {
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
}, },
WantKubeSecret: map[string]string{ WantKubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
kubetypes.KeyCapVer: capver,
}, },
}, },
{ {
@ -1016,10 +1026,11 @@ func TestContainerBoot(t *testing.T) {
// Missing "_current-profile" key. // Missing "_current-profile" key.
}, },
WantKubeSecret: map[string]string{ WantKubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
"_machinekey": "foo", "_machinekey": "foo",
"_profiles": "foo", "_profiles": "foo",
"profile-baff": "foo", "profile-baff": "foo",
kubetypes.KeyCapVer: capver,
}, },
WantLog: "Waiting for tailscaled to finish writing state to Secret \"tailscale\"", WantLog: "Waiting for tailscaled to finish writing state to Secret \"tailscale\"",
}, },
@ -1029,11 +1040,12 @@ func TestContainerBoot(t *testing.T) {
"_current-profile": "foo", "_current-profile": "foo",
}, },
WantKubeSecret: map[string]string{ WantKubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
"_machinekey": "foo", "_machinekey": "foo",
"_profiles": "foo", "_profiles": "foo",
"profile-baff": "foo", "profile-baff": "foo",
"_current-profile": "foo", "_current-profile": "foo",
kubetypes.KeyCapVer: capver,
}, },
WantLog: "HTTP server at [::]:9002 closed", WantLog: "HTTP server at [::]:9002 closed",
WantExitCode: ptr.To(0), WantExitCode: ptr.To(0),
@ -1061,7 +1073,7 @@ func TestContainerBoot(t *testing.T) {
fmt.Sprintf("TS_TEST_SOCKET=%s", env.lapi.Path), fmt.Sprintf("TS_TEST_SOCKET=%s", env.lapi.Path),
fmt.Sprintf("TS_SOCKET=%s", env.runningSockPath), fmt.Sprintf("TS_SOCKET=%s", env.runningSockPath),
fmt.Sprintf("TS_TEST_ONLY_ROOT=%s", env.d), fmt.Sprintf("TS_TEST_ONLY_ROOT=%s", env.d),
fmt.Sprint("TS_TEST_FAKE_NETFILTER=true"), "TS_TEST_FAKE_NETFILTER=true",
} }
for k, v := range tc.Env { for k, v := range tc.Env {
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
@ -1489,10 +1501,7 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) {
} }
switch r.Header.Get("Content-Type") { switch r.Header.Get("Content-Type") {
case "application/json-patch+json": case "application/json-patch+json":
req := []struct { req := []kubeclient.JSONPatch{}
Op string `json:"op"`
Path string `json:"path"`
}{}
if err := json.Unmarshal(bs, &req); err != nil { if err := json.Unmarshal(bs, &req); err != nil {
panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs))) panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs)))
} }
@ -1503,23 +1512,20 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) {
panic(fmt.Sprintf("unsupported json-patch path %q", op.Path)) panic(fmt.Sprintf("unsupported json-patch path %q", op.Path))
} }
delete(k.secret, strings.TrimPrefix(op.Path, "/data/")) delete(k.secret, strings.TrimPrefix(op.Path, "/data/"))
case "replace": case "add", "replace":
path, ok := strings.CutPrefix(op.Path, "/data/") path, ok := strings.CutPrefix(op.Path, "/data/")
if !ok { if !ok {
panic(fmt.Sprintf("unsupported json-patch path %q", op.Path)) panic(fmt.Sprintf("unsupported json-patch path %q", op.Path))
} }
req := make([]kubeclient.JSONPatch, 0) val, ok := op.Value.(string)
if err := json.Unmarshal(bs, &req); err != nil { if !ok {
panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs))) panic(fmt.Sprintf("unsupported json patch value %v: cannot be converted to string", op.Value))
} }
v, err := base64.StdEncoding.DecodeString(val)
for _, patch := range req { if err != nil {
val, ok := patch.Value.(string) panic(fmt.Sprintf("json patch value %q is not base64 encoded: %v", val, err))
if !ok {
panic(fmt.Sprintf("unsupported json patch value %v: cannot be converted to string", patch.Value))
}
k.secret[path] = val
} }
k.secret[path] = string(v)
default: default:
panic(fmt.Sprintf("unsupported json-patch op %q", op.Op)) panic(fmt.Sprintf("unsupported json-patch op %q", op.Op))
} }

2
cmd/k8s-operator/egress-services-readiness.go
Unescape Escape View File

@ -102,7 +102,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re
msg = err.Error() msg = err.Error()
return res, err return res, err
} }
if !tsoperator.ProxyGroupIsReady(pg) { if !tsoperator.ProxyGroupAvailable(pg) {
l.Infof("ProxyGroup for Service is not ready, waiting...") l.Infof("ProxyGroup for Service is not ready, waiting...")
reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
st = metav1.ConditionFalse st = metav1.ConditionFalse

2
cmd/k8s-operator/egress-services-readiness_test.go
Unescape Escape View File

@ -137,7 +137,7 @@ func setReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger, replic
} }
func setPGReady(pg *tsapi.ProxyGroup, cl tstime.Clock, l *zap.SugaredLogger) { func setPGReady(pg *tsapi.ProxyGroup, cl tstime.Clock, l *zap.SugaredLogger) {
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, "foo", "foo", pg.Generation, cl, l) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, "foo", "foo", pg.Generation, cl, l)
} }
func setEndpointForReplica(pg *tsapi.ProxyGroup, ordinal int32, eps *discoveryv1.EndpointSlice) { func setEndpointForReplica(pg *tsapi.ProxyGroup, ordinal int32, eps *discoveryv1.EndpointSlice) {

2
cmd/k8s-operator/egress-services.go
Unescape Escape View File

@ -531,7 +531,7 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
return false, nil return false, nil
} }
if !tsoperator.ProxyGroupIsReady(pg) { if !tsoperator.ProxyGroupAvailable(pg) {
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l) tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
} }

6
cmd/k8s-operator/ingress-for-pg.go
Unescape Escape View File

@ -182,7 +182,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
} }
return false, fmt.Errorf("getting ProxyGroup %q: %w", pgName, err) return false, fmt.Errorf("getting ProxyGroup %q: %w", pgName, err)
} }
if !tsoperator.ProxyGroupIsReady(pg) { if !tsoperator.ProxyGroupAvailable(pg) {
logger.Infof("ProxyGroup is not (yet) ready") logger.Infof("ProxyGroup is not (yet) ready")
return false, nil return false, nil
} }
@ -666,7 +666,7 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki
} }
// Validate TLS configuration // Validate TLS configuration
if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && (len(ing.Spec.TLS) > 1 || len(ing.Spec.TLS[0].Hosts) > 1) { if len(ing.Spec.TLS) > 0 && (len(ing.Spec.TLS) > 1 || len(ing.Spec.TLS[0].Hosts) > 1) {
errs = append(errs, fmt.Errorf("Ingress contains invalid TLS block %v: only a single TLS entry with a single host is allowed", ing.Spec.TLS)) errs = append(errs, fmt.Errorf("Ingress contains invalid TLS block %v: only a single TLS entry with a single host is allowed", ing.Spec.TLS))
} }
@ -683,7 +683,7 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki
} }
// Validate ProxyGroup readiness // Validate ProxyGroup readiness
if !tsoperator.ProxyGroupIsReady(pg) { if !tsoperator.ProxyGroupAvailable(pg) {
errs = append(errs, fmt.Errorf("ProxyGroup %q is not ready", pg.Name)) errs = append(errs, fmt.Errorf("ProxyGroup %q is not ready", pg.Name))
} }

10
cmd/k8s-operator/ingress-for-pg_test.go
Unescape Escape View File

@ -305,7 +305,7 @@ func TestValidateIngress(t *testing.T) {
Status: tsapi.ProxyGroupStatus{ Status: tsapi.ProxyGroupStatus{
Conditions: []metav1.Condition{ Conditions: []metav1.Condition{
{ {
Type: string(tsapi.ProxyGroupReady), Type: string(tsapi.ProxyGroupAvailable),
Status: metav1.ConditionTrue, Status: metav1.ConditionTrue,
ObservedGeneration: 1, ObservedGeneration: 1,
}, },
@ -399,7 +399,7 @@ func TestValidateIngress(t *testing.T) {
Status: tsapi.ProxyGroupStatus{ Status: tsapi.ProxyGroupStatus{
Conditions: []metav1.Condition{ Conditions: []metav1.Condition{
{ {
Type: string(tsapi.ProxyGroupReady), Type: string(tsapi.ProxyGroupAvailable),
Status: metav1.ConditionFalse, Status: metav1.ConditionFalse,
ObservedGeneration: 1, ObservedGeneration: 1,
}, },
@ -755,7 +755,7 @@ func verifyTailscaledConfig(t *testing.T, fc client.Client, pgName string, expec
Labels: pgSecretLabels(pgName, "config"), Labels: pgSecretLabels(pgName, "config"),
}, },
Data: map[string][]byte{ Data: map[string][]byte{
tsoperator.TailscaledConfigFileName(106): []byte(fmt.Sprintf(`{"Version":""%s}`, expected)), tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte(fmt.Sprintf(`{"Version":""%s}`, expected)),
}, },
}) })
} }
@ -794,13 +794,13 @@ func createPGResources(t *testing.T, fc client.Client, pgName string) {
Labels: pgSecretLabels(pgName, "config"), Labels: pgSecretLabels(pgName, "config"),
}, },
Data: map[string][]byte{ Data: map[string][]byte{
tsoperator.TailscaledConfigFileName(106): []byte("{}"), tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte("{}"),
}, },
} }
mustCreate(t, fc, pgCfgSecret) mustCreate(t, fc, pgCfgSecret)
pg.Status.Conditions = []metav1.Condition{ pg.Status.Conditions = []metav1.Condition{
{ {
Type: string(tsapi.ProxyGroupReady), Type: string(tsapi.ProxyGroupAvailable),
Status: metav1.ConditionTrue, Status: metav1.ConditionTrue,
ObservedGeneration: 1, ObservedGeneration: 1,
}, },

167
cmd/k8s-operator/proxygroup.go
Unescape Escape View File

@ -52,6 +52,17 @@ const (
// Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c // Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c
optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
staticEndpointsMaxAddrs = 2 staticEndpointsMaxAddrs = 2
// The minimum tailcfg.CapabilityVersion that deployed clients are expected
// to support to be compatible with the current ProxyGroup controller.
// If the controller needs to depend on newer client behaviour, it should
// maintain backwards compatible logic for older capability versions for 3
// stable releases, as per documentation on supported version drift:
// https://tailscale.com/kb/1236/kubernetes-operator#supported-versions
//
// tailcfg.CurrentCapabilityVersion was 106 when the ProxyGroup controller was
// first introduced.
pgMinCapabilityVersion = 106
) )
var ( var (
@ -204,14 +215,27 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
} }
desiredReplicas := int(pgReplicas(pg)) desiredReplicas := int(pgReplicas(pg))
// Set ProxyGroupAvailable condition.
status := metav1.ConditionFalse
reason := reasonProxyGroupCreating
message := fmt.Sprintf("%d/%d ProxyGroup pods running", len(pg.Status.Devices), desiredReplicas)
if len(pg.Status.Devices) > 0 {
status = metav1.ConditionTrue
if len(pg.Status.Devices) == desiredReplicas {
reason = reasonProxyGroupReady
}
}
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, status, reason, message, pg.Generation, r.clock, logger)
// Set ProxyGroupReady condition.
if len(pg.Status.Devices) < desiredReplicas { if len(pg.Status.Devices) < desiredReplicas {
message := fmt.Sprintf("%d/%d ProxyGroup pods running", len(pg.Status.Devices), desiredReplicas)
logger.Debug(message) logger.Debug(message)
return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message) return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message)
} }
if len(pg.Status.Devices) > desiredReplicas { if len(pg.Status.Devices) > desiredReplicas {
message := fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(pg.Status.Devices)-desiredReplicas) message = fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(pg.Status.Devices)-desiredReplicas)
logger.Debug(message) logger.Debug(message)
return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message) return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message)
} }
@ -524,17 +548,13 @@ func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg
if err := r.deleteTailnetDevice(ctx, m.tsID, logger); err != nil { if err := r.deleteTailnetDevice(ctx, m.tsID, logger); err != nil {
return err return err
} }
if err := r.Delete(ctx, m.stateSecret); err != nil { if err := r.Delete(ctx, m.stateSecret); err != nil && !apierrors.IsNotFound(err) {
if !apierrors.IsNotFound(err) { return fmt.Errorf("error deleting state Secret %q: %w", m.stateSecret.Name, err)
return fmt.Errorf("error deleting state Secret %s: %w", m.stateSecret.Name, err)
}
} }
configSecret := m.stateSecret.DeepCopy() configSecret := m.stateSecret.DeepCopy()
configSecret.Name += "-config" configSecret.Name += "-config"
if err := r.Delete(ctx, configSecret); err != nil { if err := r.Delete(ctx, configSecret); err != nil && !apierrors.IsNotFound(err) {
if !apierrors.IsNotFound(err) { return fmt.Errorf("error deleting config Secret %q: %w", configSecret.Name, err)
return fmt.Errorf("error deleting config Secret %s: %w", configSecret.Name, err)
}
} }
// NOTE(ChaosInTheCRD): we shouldn't need to get the service first, checking for a not found error should be enough // NOTE(ChaosInTheCRD): we shouldn't need to get the service first, checking for a not found error should be enough
svc := &corev1.Service{ svc := &corev1.Service{
@ -635,17 +655,38 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
return nil, err return nil, err
} }
var authKey string var authKey *string
if existingCfgSecret == nil { if existingCfgSecret == nil {
logger.Debugf("Creating authkey for new ProxyGroup proxy") logger.Debugf("Creating authkey for new ProxyGroup proxy")
tags := pg.Spec.Tags.Stringify() tags := pg.Spec.Tags.Stringify()
if len(tags) == 0 { if len(tags) == 0 {
tags = r.defaultTags tags = r.defaultTags
} }
authKey, err = newAuthKey(ctx, r.tsClient, tags) key, err := newAuthKey(ctx, r.tsClient, tags)
if err != nil { if err != nil {
return nil, err return nil, err
} }
authKey = &key
}
if authKey == nil {
// Get state Secret to check if it's already authed.
stateSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: pgStateSecretName(pg.Name, i),
Namespace: r.tsNamespace,
},
}
if err := r.Get(ctx, client.ObjectKeyFromObject(stateSecret), stateSecret); err != nil && !apierrors.IsNotFound(err) {
return nil, err
}
if shouldRetainAuthKey(stateSecret) && existingCfgSecret != nil {
authKey, err = authKeyFromSecret(existingCfgSecret)
if err != nil {
return nil, fmt.Errorf("error retrieving auth key from existing config Secret: %w", err)
}
}
} }
replicaName := pgNodePortServiceName(pg.Name, i) replicaName := pgNodePortServiceName(pg.Name, i)
@ -661,7 +702,14 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
} }
} }
configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, existingCfgSecret, endpoints[replicaName]) // AdvertiseServices config is set by ingress-pg-reconciler, so make sure we
// don't overwrite it if already set.
existingAdvertiseServices, err := extractAdvertiseServicesConfig(existingCfgSecret)
if err != nil {
return nil, err
}
configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[replicaName], existingAdvertiseServices)
if err != nil { if err != nil {
return nil, fmt.Errorf("error creating tailscaled config: %w", err) return nil, fmt.Errorf("error creating tailscaled config: %w", err)
} }
@ -811,20 +859,22 @@ func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.Pro
gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len()))
} }
func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret, staticEndpoints []netip.AddrPort) (tailscaledConfigs, error) { func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, authKey *string, staticEndpoints []netip.AddrPort, oldAdvertiseServices []string) (tailscaledConfigs, error) {
conf := &ipn.ConfigVAlpha{ conf := &ipn.ConfigVAlpha{
Version: "alpha0", Version: "alpha0",
AcceptDNS: "false", AcceptDNS: "false",
AcceptRoutes: "false", // AcceptRoutes defaults to true AcceptRoutes: "false", // AcceptRoutes defaults to true
Locked: "false", Locked: "false",
Hostname: ptr.To(fmt.Sprintf("%s-%d", pg.Name, idx)), Hostname: ptr.To(fmt.Sprintf("%s-%d", pg.Name, idx)),
AdvertiseServices: oldAdvertiseServices,
AuthKey: authKey,
} }
if pg.Spec.HostnamePrefix != "" { if pg.Spec.HostnamePrefix != "" {
conf.Hostname = ptr.To(fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, idx)) conf.Hostname = ptr.To(fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, idx))
} }
if shouldAcceptRoutes(class) { if shouldAcceptRoutes(pc) {
conf.AcceptRoutes = "true" conf.AcceptRoutes = "true"
} }
@ -832,51 +882,26 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32
conf.StaticEndpoints = staticEndpoints conf.StaticEndpoints = staticEndpoints
} }
deviceAuthed := false return map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha{
for _, d := range pg.Status.Devices { pgMinCapabilityVersion: *conf,
if d.Hostname == *conf.Hostname { }, nil
deviceAuthed = true
break
}
}
if authKey != "" {
conf.AuthKey = &authKey
} else if !deviceAuthed {
key, err := authKeyFromSecret(oldSecret)
if err != nil {
return nil, fmt.Errorf("error retrieving auth key from Secret: %w", err)
}
conf.AuthKey = key
}
capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha)
// AdvertiseServices config is set by ingress-pg-reconciler, so make sure we
// don't overwrite it here.
if err := copyAdvertiseServicesConfig(conf, oldSecret, 106); err != nil {
return nil, err
}
capVerConfigs[106] = *conf
return capVerConfigs, nil
} }
func copyAdvertiseServicesConfig(conf *ipn.ConfigVAlpha, oldSecret *corev1.Secret, capVer tailcfg.CapabilityVersion) error { func extractAdvertiseServicesConfig(cfgSecret *corev1.Secret) ([]string, error) {
if oldSecret == nil { if cfgSecret == nil {
return nil return nil, nil
} }
oldConfB := oldSecret.Data[tsoperator.TailscaledConfigFileName(capVer)] conf, err := latestConfigFromSecret(cfgSecret)
if len(oldConfB) == 0 { if err != nil {
return nil return nil, err
} }
var oldConf ipn.ConfigVAlpha if conf == nil {
if err := json.Unmarshal(oldConfB, &oldConf); err != nil { return nil, nil
return fmt.Errorf("error unmarshalling existing config: %w", err)
} }
conf.AdvertiseServices = oldConf.AdvertiseServices
return nil return conf.AdvertiseServices, nil
} }
func (r *ProxyGroupReconciler) validate(_ *tsapi.ProxyGroup) error { func (r *ProxyGroupReconciler) validate(_ *tsapi.ProxyGroup) error {
@ -914,7 +939,7 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr
dnsName: prefs.Config.UserProfile.LoginName, dnsName: prefs.Config.UserProfile.LoginName,
} }
pod := &corev1.Pod{} pod := &corev1.Pod{}
if err := r.Get(ctx, client.ObjectKey{Namespace: r.tsNamespace, Name: secret.Name}, pod); err != nil && !apierrors.IsNotFound(err) { if err := r.Get(ctx, client.ObjectKey{Namespace: r.tsNamespace, Name: fmt.Sprintf("%s-%d", pg.Name, ordinal)}, pod); err != nil && !apierrors.IsNotFound(err) {
return nil, err return nil, err
} else if err == nil { } else if err == nil {
nm.podUID = string(pod.UID) nm.podUID = string(pod.UID)
@ -932,17 +957,23 @@ func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, staticEndpoint
} }
for _, m := range metadata { for _, m := range metadata {
device, ok, err := getDeviceInfo(ctx, r.tsClient, m.stateSecret) if !strings.EqualFold(string(m.stateSecret.Data[kubetypes.KeyPodUID]), m.podUID) {
if err != nil { // Current Pod has not yet written its UID to the state Secret, data may
return nil, err // be stale.
}
if !ok {
continue continue
} }
dev := tsapi.TailnetDevice{ device := tsapi.TailnetDevice{}
Hostname: device.Hostname, if ipsB := m.stateSecret.Data[kubetypes.KeyDeviceIPs]; len(ipsB) > 0 {
TailnetIPs: device.TailnetIPs, ips := []string{}
if err := json.Unmarshal(ipsB, &ips); err != nil {
return nil, fmt.Errorf("failed to extract device IPs from state Secret %q: %w", m.stateSecret.Name, err)
}
device.TailnetIPs = ips
}
if hostname, _, ok := strings.Cut(string(m.stateSecret.Data[kubetypes.KeyDeviceFQDN]), "."); ok {
device.Hostname = hostname
} }
if ep, ok := staticEndpoints[device.Hostname]; ok && len(ep) > 0 { if ep, ok := staticEndpoints[device.Hostname]; ok && len(ep) > 0 {
@ -950,10 +981,10 @@ func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, staticEndpoint
for _, e := range ep { for _, e := range ep {
eps = append(eps, e.String()) eps = append(eps, e.String())
} }
dev.StaticEndpoints = eps device.StaticEndpoints = eps
} }
devices = append(devices, dev) devices = append(devices, device)
} }
return devices, nil return devices, nil

6
cmd/k8s-operator/proxygroup_specs.go
Unescape Escape View File

@ -351,7 +351,7 @@ func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.S
for i := range pgReplicas(pg) { for i := range pgReplicas(pg) {
secrets = append(secrets, &corev1.Secret{ secrets = append(secrets, &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%d", pg.Name, i), Name: pgStateSecretName(pg.Name, i),
Namespace: namespace, Namespace: namespace,
Labels: pgSecretLabels(pg.Name, "state"), Labels: pgSecretLabels(pg.Name, "state"),
OwnerReferences: pgOwnerReference(pg), OwnerReferences: pgOwnerReference(pg),
@ -422,6 +422,10 @@ func pgConfigSecretName(pgName string, i int32) string {
return fmt.Sprintf("%s-%d-config", pgName, i) return fmt.Sprintf("%s-%d-config", pgName, i)
} }
func pgStateSecretName(pgName string, i int32) string {
return fmt.Sprintf("%s-%d", pgName, i)
}
func pgEgressCMName(pg string) string { func pgEgressCMName(pg string) string {
return fmt.Sprintf("%s-egress-config", pg) return fmt.Sprintf("%s-egress-config", pg)
} }

18
cmd/k8s-operator/proxygroup_test.go
Unescape Escape View File

@ -877,6 +877,7 @@ func TestProxyGroup(t *testing.T) {
expectReconciled(t, reconciler, "", pg.Name) expectReconciled(t, reconciler, "", pg.Name)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar())
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar())
expectEqual(t, fc, pg) expectEqual(t, fc, pg)
expectProxyGroupResources(t, fc, pg, true, pc) expectProxyGroupResources(t, fc, pg, true, pc)
if expected := 1; reconciler.egressProxyGroups.Len() != expected { if expected := 1; reconciler.egressProxyGroups.Len() != expected {
@ -913,6 +914,7 @@ func TestProxyGroup(t *testing.T) {
}, },
} }
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar()) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar())
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupReady, "2/2 ProxyGroup pods running", 0, cl, zl.Sugar())
expectEqual(t, fc, pg) expectEqual(t, fc, pg)
expectProxyGroupResources(t, fc, pg, true, pc) expectProxyGroupResources(t, fc, pg, true, pc)
}) })
@ -924,12 +926,14 @@ func TestProxyGroup(t *testing.T) {
}) })
expectReconciled(t, reconciler, "", pg.Name) expectReconciled(t, reconciler, "", pg.Name)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar()) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar())
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar())
expectEqual(t, fc, pg) expectEqual(t, fc, pg)
expectProxyGroupResources(t, fc, pg, true, pc) expectProxyGroupResources(t, fc, pg, true, pc)
addNodeIDToStateSecrets(t, fc, pg) addNodeIDToStateSecrets(t, fc, pg)
expectReconciled(t, reconciler, "", pg.Name) expectReconciled(t, reconciler, "", pg.Name)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar()) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar())
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupReady, "3/3 ProxyGroup pods running", 0, cl, zl.Sugar())
pg.Status.Devices = append(pg.Status.Devices, tsapi.TailnetDevice{ pg.Status.Devices = append(pg.Status.Devices, tsapi.TailnetDevice{
Hostname: "hostname-nodeid-2", Hostname: "hostname-nodeid-2",
TailnetIPs: []string{"1.2.3.4", "::1"}, TailnetIPs: []string{"1.2.3.4", "::1"},
@ -947,6 +951,7 @@ func TestProxyGroup(t *testing.T) {
expectReconciled(t, reconciler, "", pg.Name) expectReconciled(t, reconciler, "", pg.Name)
pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device. pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device.
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupReady, "1/1 ProxyGroup pods running", 0, cl, zl.Sugar())
expectEqual(t, fc, pg) expectEqual(t, fc, pg)
expectProxyGroupResources(t, fc, pg, true, pc) expectProxyGroupResources(t, fc, pg, true, pc)
}) })
@ -1224,7 +1229,7 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) {
Namespace: tsNamespace, Namespace: tsNamespace,
}, },
Data: map[string][]byte{ Data: map[string][]byte{
tsoperator.TailscaledConfigFileName(106): existingConfigBytes, tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): existingConfigBytes,
}, },
}) })
@ -1261,7 +1266,7 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) {
ResourceVersion: "2", ResourceVersion: "2",
}, },
Data: map[string][]byte{ Data: map[string][]byte{
tsoperator.TailscaledConfigFileName(106): expectedConfigBytes, tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): expectedConfigBytes,
}, },
}) })
} }
@ -1421,8 +1426,13 @@ func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyG
mustUpdate(t, fc, tsNamespace, fmt.Sprintf("test-%d", i), func(s *corev1.Secret) { mustUpdate(t, fc, tsNamespace, fmt.Sprintf("test-%d", i), func(s *corev1.Secret) {
s.Data = map[string][]byte{ s.Data = map[string][]byte{
currentProfileKey: []byte(key), currentProfileKey: []byte(key),
key: bytes, key: bytes,
kubetypes.KeyDeviceIPs: []byte(`["1.2.3.4", "::1"]`),
kubetypes.KeyDeviceFQDN: []byte(fmt.Sprintf("hostname-nodeid-%d.tails-scales.ts.net", i)),
// TODO(tomhjp): We have two different mechanisms to retrieve device IDs.
// Consolidate on this one.
kubetypes.KeyDeviceID: []byte(fmt.Sprintf("nodeid-%d", i)),
} }
}) })
} }

36
cmd/k8s-operator/sts.go
Unescape Escape View File

@ -897,14 +897,6 @@ func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) {
} }
} }
func readAuthKey(secret *corev1.Secret, key string) (*string, error) {
origConf := &ipn.ConfigVAlpha{}
if err := json.Unmarshal([]byte(secret.Data[key]), origConf); err != nil {
return nil, fmt.Errorf("error unmarshaling previous tailscaled config in %q: %w", key, err)
}
return origConf.AuthKey, nil
}
// tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy // tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy
// state and auth key and returns tailscaled config files for currently supported proxy versions. // state and auth key and returns tailscaled config files for currently supported proxy versions.
func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) { func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
@ -951,7 +943,10 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
return capVerConfigs, nil return capVerConfigs, nil
} }
func authKeyFromSecret(s *corev1.Secret) (key *string, err error) { // latestConfigFromSecret returns the ipn.ConfigVAlpha with the highest capver
// as found in the Secret's key names, e.g. "cap-107.hujson" has capver 107.
// If no config is found, it returns nil.
func latestConfigFromSecret(s *corev1.Secret) (*ipn.ConfigVAlpha, error) {
latest := tailcfg.CapabilityVersion(-1) latest := tailcfg.CapabilityVersion(-1)
latestStr := "" latestStr := ""
for k, data := range s.Data { for k, data := range s.Data {
@ -968,12 +963,31 @@ func authKeyFromSecret(s *corev1.Secret) (key *string, err error) {
latest = v latest = v
} }
} }
var conf *ipn.ConfigVAlpha
if latestStr != "" {
conf = &ipn.ConfigVAlpha{}
if err := json.Unmarshal([]byte(s.Data[latestStr]), conf); err != nil {
return nil, fmt.Errorf("error unmarshaling tailscaled config from Secret %q in field %q: %w", s.Name, latestStr, err)
}
}
return conf, nil
}
func authKeyFromSecret(s *corev1.Secret) (key *string, err error) {
conf, err := latestConfigFromSecret(s)
if err != nil {
return nil, err
}
// Allow for configs that don't contain an auth key. Perhaps // Allow for configs that don't contain an auth key. Perhaps
// users have some mechanisms to delete them. Auth key is // users have some mechanisms to delete them. Auth key is
// normally not needed after the initial login. // normally not needed after the initial login.
if latestStr != "" { if conf != nil {
return readAuthKey(s, latestStr) key = conf.AuthKey
} }
return key, nil return key, nil
} }

2
cmd/k8s-operator/svc-for-pg.go
Unescape Escape View File

@ -164,7 +164,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin
} }
return false, fmt.Errorf("getting ProxyGroup %q: %w", pgName, err) return false, fmt.Errorf("getting ProxyGroup %q: %w", pgName, err)
} }
if !tsoperator.ProxyGroupIsReady(pg) { if !tsoperator.ProxyGroupAvailable(pg) {
logger.Infof("ProxyGroup is not (yet) ready") logger.Infof("ProxyGroup is not (yet) ready")
return false, nil return false, nil
} }

4
cmd/k8s-operator/svc-for-pg_test.go
Unescape Escape View File

@ -142,7 +142,7 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien
Labels: pgSecretLabels("test-pg", "config"), Labels: pgSecretLabels("test-pg", "config"),
}, },
Data: map[string][]byte{ Data: map[string][]byte{
tsoperator.TailscaledConfigFileName(106): []byte(`{"Version":""}`), tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte(`{"Version":""}`),
}, },
} }
@ -179,7 +179,7 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien
// Set ProxyGroup status to ready // Set ProxyGroup status to ready
pg.Status.Conditions = []metav1.Condition{ pg.Status.Conditions = []metav1.Condition{
{ {
Type: string(tsapi.ProxyGroupReady), Type: string(tsapi.ProxyGroupAvailable),
Status: metav1.ConditionTrue, Status: metav1.ConditionTrue,
ObservedGeneration: 1, ObservedGeneration: 1,
}, },

9
cmd/k8s-operator/tsrecorder.go
Unescape Escape View File

@ -446,18 +446,15 @@ func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string)
return tsapi.RecorderTailnetDevice{}, false, err return tsapi.RecorderTailnetDevice{}, false, err
} }
return getDeviceInfo(ctx, r.tsClient, secret)
}
func getDeviceInfo(ctx context.Context, tsClient tsClient, secret *corev1.Secret) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
prefs, ok, err := getDevicePrefs(secret) prefs, ok, err := getDevicePrefs(secret)
if !ok || err != nil { if !ok || err != nil {
return tsapi.RecorderTailnetDevice{}, false, err return tsapi.RecorderTailnetDevice{}, false, err
} }
// TODO(tomhjp): The profile info doesn't include addresses, which is why we // TODO(tomhjp): The profile info doesn't include addresses, which is why we
// need the API. Should we instead update the profile to include addresses? // need the API. Should maybe update tsrecorder to write IPs to the state
device, err := tsClient.Device(ctx, string(prefs.Config.NodeID), nil) // Secret like containerboot does.
device, err := r.tsClient.Device(ctx, string(prefs.Config.NodeID), nil)
if err != nil { if err != nil {
return tsapi.RecorderTailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err) return tsapi.RecorderTailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err)
} }

11
k8s-operator/apis/v1alpha1/types_connector.go
Unescape Escape View File

@ -205,11 +205,12 @@ type ConnectorStatus struct {
type ConditionType string type ConditionType string
const ( const (
ConnectorReady ConditionType = `ConnectorReady` ConnectorReady ConditionType = `ConnectorReady`
ProxyClassReady ConditionType = `ProxyClassReady` ProxyClassReady ConditionType = `ProxyClassReady`
ProxyGroupReady ConditionType = `ProxyGroupReady` ProxyGroupReady ConditionType = `ProxyGroupReady` // All proxy Pods running.
ProxyReady ConditionType = `TailscaleProxyReady` // a Tailscale-specific condition type for corev1.Service ProxyGroupAvailable ConditionType = `ProxyGroupAvailable` // At least one proxy Pod running.
RecorderReady ConditionType = `RecorderReady` ProxyReady ConditionType = `TailscaleProxyReady` // a Tailscale-specific condition type for corev1.Service
RecorderReady ConditionType = `RecorderReady`
// EgressSvcValid gets set on a user configured ExternalName Service that defines a tailnet target to be exposed // EgressSvcValid gets set on a user configured ExternalName Service that defines a tailnet target to be exposed
// on a ProxyGroup. // on a ProxyGroup.
// Set to true if the user provided configuration is valid. // Set to true if the user provided configuration is valid.

10
k8s-operator/conditions.go
Unescape Escape View File

@ -137,8 +137,16 @@ func ProxyClassIsReady(pc *tsapi.ProxyClass) bool {
} }
func ProxyGroupIsReady(pg *tsapi.ProxyGroup) bool { func ProxyGroupIsReady(pg *tsapi.ProxyGroup) bool {
return proxyGroupCondition(pg, tsapi.ProxyGroupReady)
}
func ProxyGroupAvailable(pg *tsapi.ProxyGroup) bool {
return proxyGroupCondition(pg, tsapi.ProxyGroupAvailable)
}
func proxyGroupCondition(pg *tsapi.ProxyGroup, condType tsapi.ConditionType) bool {
idx := xslices.IndexFunc(pg.Status.Conditions, func(cond metav1.Condition) bool { idx := xslices.IndexFunc(pg.Status.Conditions, func(cond metav1.Condition) bool {
return cond.Type == string(tsapi.ProxyGroupReady) return cond.Type == string(condType)
}) })
if idx == -1 { if idx == -1 {
return false return false

17
kube/kubeclient/fake_client.go
Unescape Escape View File

@ -13,12 +13,13 @@ import (
var _ Client = &FakeClient{} var _ Client = &FakeClient{}
type FakeClient struct { type FakeClient struct {
GetSecretImpl func(context.Context, string) (*kubeapi.Secret, error) GetSecretImpl func(context.Context, string) (*kubeapi.Secret, error)
CheckSecretPermissionsImpl func(ctx context.Context, name string) (bool, bool, error) CheckSecretPermissionsImpl func(ctx context.Context, name string) (bool, bool, error)
CreateSecretImpl func(context.Context, *kubeapi.Secret) error CreateSecretImpl func(context.Context, *kubeapi.Secret) error
UpdateSecretImpl func(context.Context, *kubeapi.Secret) error UpdateSecretImpl func(context.Context, *kubeapi.Secret) error
JSONPatchResourceImpl func(context.Context, string, string, []JSONPatch) error JSONPatchResourceImpl func(context.Context, string, string, []JSONPatch) error
ListSecretsImpl func(context.Context, map[string]string) (*kubeapi.SecretList, error) ListSecretsImpl func(context.Context, map[string]string) (*kubeapi.SecretList, error)
StrategicMergePatchSecretImpl func(context.Context, string, *kubeapi.Secret, string) error
} }
func (fc *FakeClient) CheckSecretPermissions(ctx context.Context, name string) (bool, bool, error) { func (fc *FakeClient) CheckSecretPermissions(ctx context.Context, name string) (bool, bool, error) {
@ -30,8 +31,8 @@ func (fc *FakeClient) GetSecret(ctx context.Context, name string) (*kubeapi.Secr
func (fc *FakeClient) SetURL(_ string) {} func (fc *FakeClient) SetURL(_ string) {}
func (fc *FakeClient) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) { func (fc *FakeClient) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) {
} }
func (fc *FakeClient) StrategicMergePatchSecret(context.Context, string, *kubeapi.Secret, string) error { func (fc *FakeClient) StrategicMergePatchSecret(ctx context.Context, name string, s *kubeapi.Secret, fieldManager string) error {
return nil return fc.StrategicMergePatchSecretImpl(ctx, name, s, fieldManager)
} }
func (fc *FakeClient) Event(context.Context, string, string, string) error { func (fc *FakeClient) Event(context.Context, string, string, string) error {
return nil return nil

Write Preview
Loading…
Cancel
Save