From 3451b89e5f01fd6c4845bcbf06f1bece9f39cdfc Mon Sep 17 00:00:00 2001 From: Mike Beaumont Date: Wed, 9 Aug 2023 01:03:08 +0200 Subject: [PATCH] cmd/k8s-operator: put Tailscale IPs in Service ingress status Updates #502 Signed-off-by: Mike Beaumont --- cmd/containerboot/kube.go | 14 +++++++++++++- cmd/containerboot/main.go | 2 +- cmd/containerboot/main_test.go | 16 ++++++++++------ cmd/k8s-operator/ingress.go | 2 +- cmd/k8s-operator/operator_test.go | 21 +++++++++++++++++++++ cmd/k8s-operator/sts.go | 20 +++++++++++++------- cmd/k8s-operator/svc.go | 14 ++++++++------ 7 files changed, 67 insertions(+), 22 deletions(-) diff --git a/cmd/containerboot/kube.go b/cmd/containerboot/kube.go index ab097a7c5..c7c86f472 100644 --- a/cmd/containerboot/kube.go +++ b/cmd/containerboot/kube.go @@ -7,9 +7,11 @@ package main import ( "context" + "encoding/json" "fmt" "log" "net/http" + "net/netip" "os" "tailscale.com/kube" @@ -32,7 +34,7 @@ func findKeyInKubeSecret(ctx context.Context, secretName string) (string, error) // storeDeviceInfo writes deviceID into the "device_id" data field of the kube // secret secretName. -func storeDeviceInfo(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID, fqdn string) error { +func storeDeviceInfo(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID, fqdn string, addresses []netip.Prefix) error { // First check if the secret exists at all. Even if running on // kubernetes, we do not necessarily store state in a k8s secret. if _, err := kc.GetSecret(ctx, secretName); err != nil { @@ -46,10 +48,20 @@ func storeDeviceInfo(ctx context.Context, secretName string, deviceID tailcfg.St return err } + var ips []string + for _, addr := range addresses { + ips = append(ips, addr.Addr().String()) + } + deviceIPs, err := json.Marshal(ips) + if err != nil { + return err + } + m := &kube.Secret{ Data: map[string][]byte{ "device_id": []byte(deviceID), "device_fqdn": []byte(fqdn), + "device_ips": deviceIPs, }, } return kc.StrategicMergePatchSecret(ctx, secretName, m, "tailscale-container") diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 8c7f25fcc..03748d221 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -314,7 +314,7 @@ authLoop: } deviceInfo := []any{n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name()} if cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != "" && deephash.Update(¤tDeviceInfo, &deviceInfo) { - if err := storeDeviceInfo(ctx, cfg.KubeSecret, n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name()); err != nil { + if err := storeDeviceInfo(ctx, cfg.KubeSecret, n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses().AsSlice()); err != nil { log.Fatalf("storing device ID in kube secret: %v", err) } } diff --git a/cmd/containerboot/main_test.go b/cmd/containerboot/main_test.go index 842c9ccdb..5b0222711 100644 --- a/cmd/containerboot/main_test.go +++ b/cmd/containerboot/main_test.go @@ -113,10 +113,10 @@ func TestContainerBoot(t *testing.T) { State: ptr.To(ipn.Running), NetMap: &netmap.NetworkMap{ SelfNode: (&tailcfg.Node{ - StableID: tailcfg.StableNodeID("myID"), - Name: "test-node.test.ts.net", + StableID: tailcfg.StableNodeID("myID"), + Name: "test-node.test.ts.net", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, }).View(), - Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, }, } tests := []struct { @@ -359,6 +359,7 @@ func TestContainerBoot(t *testing.T) { "authkey": "tskey-key", "device_fqdn": "test-node.test.ts.net", "device_id": "myID", + "device_ips": `["100.64.0.1"]`, }, }, }, @@ -447,6 +448,7 @@ func TestContainerBoot(t *testing.T) { WantKubeSecret: map[string]string{ "device_fqdn": "test-node.test.ts.net", "device_id": "myID", + "device_ips": `["100.64.0.1"]`, }, }, }, @@ -476,6 +478,7 @@ func TestContainerBoot(t *testing.T) { "authkey": "tskey-key", "device_fqdn": "test-node.test.ts.net", "device_id": "myID", + "device_ips": `["100.64.0.1"]`, }, }, { @@ -483,16 +486,17 @@ func TestContainerBoot(t *testing.T) { State: ptr.To(ipn.Running), NetMap: &netmap.NetworkMap{ SelfNode: (&tailcfg.Node{ - StableID: tailcfg.StableNodeID("newID"), - Name: "new-name.test.ts.net", + StableID: tailcfg.StableNodeID("newID"), + Name: "new-name.test.ts.net", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, }).View(), - Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, }, }, WantKubeSecret: map[string]string{ "authkey": "tskey-key", "device_fqdn": "new-name.test.ts.net", "device_id": "newID", + "device_ips": `["100.64.0.1"]`, }, }, }, diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index f3537b617..dd52299e0 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -194,7 +194,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga return fmt.Errorf("failed to provision: %w", err) } - _, tsHost, err := a.ssr.DeviceInfo(ctx, crl) + _, tsHost, _, err := a.ssr.DeviceInfo(ctx, crl) if err != nil { return fmt.Errorf("failed to get device ID: %w", err) } diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index bcd0bbd50..cdbe668b2 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -80,6 +80,7 @@ func TestLoadBalancerClass(t *testing.T) { } s.Data["device_id"] = []byte("ts-id-1234") s.Data["device_fqdn"] = []byte("tailscale.device.name.") + s.Data["device_ips"] = []byte(`["100.99.98.97", "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf"]`) }) expectReconciled(t, sr, "default", "test") want := &corev1.Service{ @@ -104,6 +105,12 @@ func TestLoadBalancerClass(t *testing.T) { { Hostname: "tailscale.device.name", }, + { + IP: "100.99.98.97", + }, + { + IP: "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf", + }, }, }, }, @@ -306,6 +313,7 @@ func TestAnnotationIntoLB(t *testing.T) { } s.Data["device_id"] = []byte("ts-id-1234") s.Data["device_fqdn"] = []byte("tailscale.device.name.") + s.Data["device_ips"] = []byte(`["100.99.98.97", "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf"]`) }) expectReconciled(t, sr, "default", "test") want := &corev1.Service{ @@ -364,6 +372,12 @@ func TestAnnotationIntoLB(t *testing.T) { { Hostname: "tailscale.device.name", }, + { + IP: "100.99.98.97", + }, + { + IP: "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf", + }, }, }, }, @@ -425,6 +439,7 @@ func TestLBIntoAnnotation(t *testing.T) { } s.Data["device_id"] = []byte("ts-id-1234") s.Data["device_fqdn"] = []byte("tailscale.device.name.") + s.Data["device_ips"] = []byte(`["100.99.98.97", "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf"]`) }) expectReconciled(t, sr, "default", "test") want := &corev1.Service{ @@ -449,6 +464,12 @@ func TestLBIntoAnnotation(t *testing.T) { { Hostname: "tailscale.device.name", }, + { + IP: "100.99.98.97", + }, + { + IP: "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf", + }, }, }, }, diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 4969002ad..131443e9b 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -122,7 +122,7 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare return false, nil } - id, _, err := a.DeviceInfo(ctx, labels) + id, _, _, err := a.DeviceInfo(ctx, labels) if err != nil { return false, fmt.Errorf("getting device info: %w", err) } @@ -232,25 +232,31 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * // DeviceInfo returns the device ID and hostname for the Tailscale device // associated with the given labels. -func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string) (id tailcfg.StableNodeID, hostname string, err error) { +func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string) (id tailcfg.StableNodeID, hostname string, ips []string, err error) { sec, err := getSingleObject[corev1.Secret](ctx, a.Client, a.operatorNamespace, childLabels) if err != nil { - return "", "", err + return "", "", nil, err } if sec == nil { - return "", "", nil + return "", "", nil, nil } id = tailcfg.StableNodeID(sec.Data["device_id"]) if id == "" { - return "", "", nil + return "", "", nil, nil } // Kubernetes chokes on well-formed FQDNs with the trailing dot, so we have // to remove it. hostname = strings.TrimSuffix(string(sec.Data["device_fqdn"]), ".") if hostname == "" { - return "", "", nil + return "", "", nil, nil } - return id, hostname, nil + if rawDeviceIPs, ok := sec.Data["device_ips"]; ok { + if err := json.Unmarshal(rawDeviceIPs, &ips); err != nil { + return "", "", nil, err + } + } + + return id, hostname, ips, nil } func (a *tailscaleSTSReconciler) newAuthKey(ctx context.Context, tags []string) (string, error) { diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index 8b777965d..954b825a8 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -139,7 +139,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga return nil } - _, tsHost, err := a.ssr.DeviceInfo(ctx, crl) + _, tsHost, tsIPs, err := a.ssr.DeviceInfo(ctx, crl) if err != nil { return fmt.Errorf("failed to get device ID: %w", err) } @@ -153,12 +153,14 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga return nil } - logger.Debugf("setting ingress hostname to %q", tsHost) - svc.Status.LoadBalancer.Ingress = []corev1.LoadBalancerIngress{ - { - Hostname: tsHost, - }, + logger.Debugf("setting ingress to %q, %s", tsHost, strings.Join(tsIPs, ", ")) + ingress := []corev1.LoadBalancerIngress{ + {Hostname: tsHost}, } + for _, ip := range tsIPs { + ingress = append(ingress, corev1.LoadBalancerIngress{IP: ip}) + } + svc.Status.LoadBalancer.Ingress = ingress if err := a.Status().Update(ctx, svc); err != nil { return fmt.Errorf("failed to update service status: %w", err) }