cmd/k8s-operator: add support for control plane assigned groups

Previously we would use the Impersonate-Group header to pass through
tags to the k8s api server. However, we would do nothing for non-tagged
nodes. Now that we have a way to specify these via peerCaps respect those
and send down groups for non-tagged nodes as well.

For tagged nodes, it defaults to sending down the tags as groups to retain
legacy behavior if there are no caps set. Otherwise, the tags are omitted.

Updates #5055

Signed-off-by: Maisem Ali <maisem@tailscale.com>

cmd/k8s-operator/proxy.go

@@ -16,12 +16,26 @@ import (
 
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/set"
 )
 
 type whoIsKey struct{}
 
+// whoIsFromRequest returns the WhoIsResponse previously stashed by a call to
+// addWhoIsToRequest.
+func whoIsFromRequest(r *http.Request) *apitype.WhoIsResponse {
+	return r.Context().Value(whoIsKey{}).(*apitype.WhoIsResponse)
+}
+
+// addWhoIsToRequest stashes who in r's context, retrievable by a call to
+// whoIsFromRequest.
+func addWhoIsToRequest(r *http.Request, who *apitype.WhoIsResponse) *http.Request {
+	return r.WithContext(context.WithValue(r.Context(), whoIsKey{}, who))
+}
+
 // authProxy is an http.Handler that authenticates requests using the Tailscale
 // LocalAPI and then proxies them to the Kubernetes API.
 type authProxy struct {
@@ -37,8 +51,7 @@ func (h *authProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "failed to authenticate caller", http.StatusInternalServerError)
 		return
 	}
-	r = r.WithContext(context.WithValue(r.Context(), whoIsKey{}, who))
+	h.rp.ServeHTTP(w, addWhoIsToRequest(r, who))
-	h.rp.ServeHTTP(w, r)
 }
 
 // runAuthProxy runs an HTTP server that authenticates requests using the
@@ -67,6 +80,10 @@ func runAuthProxy(s *tsnet.Server, rt http.RoundTripper, logf logger.Logf) {
 		lc:   lc,
 		rp: &httputil.ReverseProxy{
 			Director: func(r *http.Request) {
+				// Replace the URL with the Kubernetes APIServer.
+				r.URL.Scheme = u.Scheme
+				r.URL.Host = u.Host
+
 				// We want to proxy to the Kubernetes API, but we want to use
 				// the caller's identity to do so. We do this by impersonating
 				// the caller using the Kubernetes User Impersonation feature:
@@ -85,21 +102,9 @@ func runAuthProxy(s *tsnet.Server, rt http.RoundTripper, logf logger.Logf) {
 				}
 
 				// Now add the impersonation headers that we want.
-				who := r.Context().Value(whoIsKey{}).(*apitype.WhoIsResponse)
+				if err := addImpersonationHeaders(r); err != nil {
-				if who.Node.IsTagged() {
+					panic("failed to add impersonation headers: " + err.Error())
-					// Use the nodes FQDN as the username, and the nodes tags as the groups.
-					// "Impersonate-Group" requires "Impersonate-User" to be set.
-					r.Header.Set("Impersonate-User", strings.TrimSuffix(who.Node.Name, "."))
-					for _, tag := range who.Node.Tags {
-						r.Header.Add("Impersonate-Group", tag)
-					}
-				} else {
-					r.Header.Set("Impersonate-User", who.UserProfile.LoginName)
 				}
-
-				// Replace the URL with the Kubernetes APIServer.
-				r.URL.Scheme = u.Scheme
-				r.URL.Host = u.Host
 			},
 			Transport: rt,
 		},
@@ -118,3 +123,58 @@ func runAuthProxy(s *tsnet.Server, rt http.RoundTripper, logf logger.Logf) {
 		log.Fatalf("runAuthProxy: failed to serve %v", err)
 	}
 }
+
+const capabilityName = "https://tailscale.com/cap/kubernetes"
+
+type capRule struct {
+	// Impersonate is a list of rules that specify how to impersonate the caller
+	// when proxying to the Kubernetes API.
+	Impersonate *impersonateRule `json:"impersonate,omitempty"`
+}
+
+// TODO(maisem): move this to some well-known location so that it can be shared
+// with control.
+type impersonateRule struct {
+	Groups []string `json:"groups,omitempty"`
+}
+
+// addImpersonationHeaders adds the appropriate headers to r to impersonate the
+// caller when proxying to the Kubernetes API. It uses the WhoIsResponse stashed
+// in the context by the authProxy.
+func addImpersonationHeaders(r *http.Request) error {
+	who := whoIsFromRequest(r)
+	rules, err := tailcfg.UnmarshalCapJSON[capRule](who.CapMap, capabilityName)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal capability: %v", err)
+	}
+
+	var groupsAdded set.Slice[string]
+	for _, rule := range rules {
+		if rule.Impersonate == nil {
+			continue
+		}
+		for _, group := range rule.Impersonate.Groups {
+			if groupsAdded.Contains(group) {
+				continue
+			}
+			r.Header.Add("Impersonate-Group", group)
+			groupsAdded.Add(group)
+		}
+	}
+
+	if !who.Node.IsTagged() {
+		r.Header.Set("Impersonate-User", who.UserProfile.LoginName)
+		return nil
+	}
+	// "Impersonate-Group" requires "Impersonate-User" to be set, so we set it
+	// to the node FQDN for tagged nodes.
+	r.Header.Set("Impersonate-User", strings.TrimSuffix(who.Node.Name, "."))
+
+	// For legacy behavior (before caps), set the groups to the nodes tags.
+	if groupsAdded.Slice().Len() == 0 {
+		for _, tag := range who.Node.Tags {
+			r.Header.Add("Impersonate-Group", tag)
+		}
+	}
+	return nil
+}

cmd/k8s-operator/proxy_test.go

@@ -0,0 +1,107 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/tailcfg"
+	"tailscale.com/util/must"
+)
+
+func TestImpersonationHeaders(t *testing.T) {
+	tests := []struct {
+		name     string
+		emailish string
+		tags     []string
+		capMap   tailcfg.PeerCapMap
+
+		wantHeaders http.Header
+	}{
+		{
+			name:     "user",
+			emailish: "foo@example.com",
+			wantHeaders: http.Header{
+				"Impersonate-User": {"foo@example.com"},
+			},
+		},
+		{
+			name:     "tagged",
+			emailish: "tagged-device",
+			tags:     []string{"tag:foo", "tag:bar"},
+			wantHeaders: http.Header{
+				"Impersonate-User":  {"node.ts.net"},
+				"Impersonate-Group": {"tag:foo", "tag:bar"},
+			},
+		},
+		{
+			name:     "user-with-cap",
+			emailish: "foo@example.com",
+			capMap: tailcfg.PeerCapMap{
+				capabilityName: {
+					[]byte(`{"impersonate":{"groups":["group1","group2"]}}`),
+					[]byte(`{"impersonate":{"groups":["group1","group3"]}}`), // One group is duplicated.
+					[]byte(`{"impersonate":{"groups":["group4"]}}`),
+					[]byte(`{"impersonate":{"groups":["group2"]}}`), // duplicate
+
+					// These should be ignored, but should parse correctly.
+					[]byte(`{}`),
+					[]byte(`{"impersonate":{}}`),
+					[]byte(`{"impersonate":{"groups":[]}}`),
+				},
+			},
+			wantHeaders: http.Header{
+				"Impersonate-Group": {"group1", "group2", "group3", "group4"},
+				"Impersonate-User":  {"foo@example.com"},
+			},
+		},
+		{
+			name:     "tagged-with-cap",
+			emailish: "tagged-device",
+			tags:     []string{"tag:foo", "tag:bar"},
+			capMap: tailcfg.PeerCapMap{
+				capabilityName: {
+					[]byte(`{"impersonate":{"groups":["group1"]}}`),
+				},
+			},
+			wantHeaders: http.Header{
+				"Impersonate-Group": {"group1"},
+				"Impersonate-User":  {"node.ts.net"},
+			},
+		},
+		{
+			name:     "bad-cap",
+			emailish: "tagged-device",
+			tags:     []string{"tag:foo", "tag:bar"},
+			capMap: tailcfg.PeerCapMap{
+				capabilityName: {
+					[]byte(`[]`),
+				},
+			},
+			wantHeaders: http.Header{},
+		},
+	}
+
+	for _, tc := range tests {
+		r := must.Get(http.NewRequest("GET", "https://op.ts.net/api/foo", nil))
+		r = addWhoIsToRequest(r, &apitype.WhoIsResponse{
+			Node: &tailcfg.Node{
+				Name: "node.ts.net",
+				Tags: tc.tags,
+			},
+			UserProfile: &tailcfg.UserProfile{
+				LoginName: tc.emailish,
+			},
+			CapMap: tc.capMap,
+		})
+		addImpersonationHeaders(r)
+
+		if d := cmp.Diff(tc.wantHeaders, r.Header); d != "" {
+			t.Errorf("unexpected header (-want +got):\n%s", d)
+		}
+	}
+}