k8s-operator/api-proxy: put kube api server events behind environment variable (#17550)

This commit modifies the k8s-operator's api proxy implementation to only
enable forwarding of api requests to tsrecorder when an environment
variable is set.

This new environment variable is named `TS_EXPERIMENTAL_KUBE_API_EVENTS`.

Updates https://github.com/tailscale/corp/issues/32448

Signed-off-by: David Bond <davidsbond93@gmail.com>

k8s-operator/api-proxy/proxy.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/client-go/transport"
 	"tailscale.com/client/local"
 	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/envknob"
 	ksr "tailscale.com/k8s-operator/sessionrecording"
 	"tailscale.com/kube/kubetypes"
 	"tailscale.com/net/netx"
@@ -96,6 +97,7 @@ func NewAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, ts *tsn
 		upstreamURL:   u,
 		ts:            ts,
 		sendEventFunc: sessionrecording.SendEvent,
+		eventsEnabled: envknob.Bool("TS_EXPERIMENTAL_KUBE_API_EVENTS"),
 	}
 	ap.rp = &httputil.ReverseProxy{
 		Rewrite: func(pr *httputil.ProxyRequest) {
@@ -192,6 +194,9 @@ type APIServerProxy struct {
 	upstreamURL *url.URL
 
 	sendEventFunc func(ap netip.AddrPort, event io.Reader, dial netx.DialFunc) error
+
+	// Flag used to enable sending API requests as events to tsrecorder.
+	eventsEnabled bool
 }
 
 // serveDefault is the default handler for Kubernetes API server requests.
@@ -310,6 +315,10 @@ func (ap *APIServerProxy) sessionForProto(w http.ResponseWriter, r *http.Request
 }
 
 func (ap *APIServerProxy) recordRequestAsEvent(req *http.Request, who *apitype.WhoIsResponse) error {
+	if !ap.eventsEnabled {
+		return nil
+	}
+
 	failOpen, addrs, err := determineRecorderConfig(who)
 	if err != nil {
 		return fmt.Errorf("error trying to determine whether the kubernetes api request needs to be recorded: %w", err)

k8s-operator/api-proxy/proxy_events_test.go

@@ -61,6 +61,7 @@ func TestRecordRequestAsEvent(t *testing.T) {
 		log:           zl.Sugar(),
 		ts:            &tsnet.Server{},
 		sendEventFunc: sender.Send,
+		eventsEnabled: true,
 	}
 
 	defaultWho := &apitype.WhoIsResponse{