cmd/k8s-proxy,k8s-operator: fix serve config for userspace mode (#16919)

The serve code leaves it up to the system's DNS resolver and netstack to
figure out how to reach the proxy destination. Combined with k8s-proxy
running in userspace mode, this means we can't rely on MagicDNS being
available or tailnet IPs being routable. I'd like to implement that as a
feature for serve in userspace mode, but for now the safer fix to get
kube-apiserver ProxyGroups consistently working in all environments is to
switch to using localhost as the proxy target instead.

This has a small knock-on in the code that does WhoIs lookups, which now
needs to check the X-Forwarded-For header that serve populates to get
the correct tailnet IP to look up, because the request's remote address
will be loopback.

Fixes #16920

Change-Id: I869ddcaf93102da50e66071bb00114cc1acc1288

Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>

cmd/k8s-proxy/k8s-proxy.go

@@ -453,7 +453,7 @@ func setServeConfig(ctx context.Context, lc *local.Client, cm *certs.CertManager
 					serviceHostPort: {
 						Handlers: map[string]*ipn.HTTPHandler{
 							"/": {
-								Proxy: fmt.Sprintf("http://%s:80", strings.TrimSuffix(status.Self.DNSName, ".")),
+								Proxy: "http://localhost:80",
 							},
 						},
 					},

k8s-operator/api-proxy/proxy.go

@@ -123,11 +123,11 @@ func (ap *APIServerProxy) Run(ctx context.Context) error {
 	if ap.authMode {
 		mode = "auth"
 	}
-	var tsLn net.Listener
+	var proxyLn net.Listener
 	var serve func(ln net.Listener) error
 	if ap.https {
 		var err error
-		tsLn, err = ap.ts.Listen("tcp", ":443")
+		proxyLn, err = ap.ts.Listen("tcp", ":443")
 		if err != nil {
 			return fmt.Errorf("could not listen on :443: %w", err)
 		}
@@ -143,7 +143,7 @@ func (ap *APIServerProxy) Run(ctx context.Context) error {
 		}
 	} else {
 		var err error
-		tsLn, err = ap.ts.Listen("tcp", ":80")
+		proxyLn, err = net.Listen("tcp", "localhost:80")
 		if err != nil {
 			return fmt.Errorf("could not listen on :80: %w", err)
 		}
@@ -152,8 +152,8 @@ func (ap *APIServerProxy) Run(ctx context.Context) error {
 
 	errs := make(chan error)
 	go func() {
-		ap.log.Infof("API server proxy in %s mode is listening on tailnet addresses %s", mode, tsLn.Addr())
+		ap.log.Infof("API server proxy in %s mode is listening on %s", mode, proxyLn.Addr())
-		if err := serve(tsLn); err != nil && err != http.ErrServerClosed {
+		if err := serve(proxyLn); err != nil && err != http.ErrServerClosed {
 			errs <- fmt.Errorf("error serving: %w", err)
 		}
 	}()
@@ -179,7 +179,7 @@ type APIServerProxy struct {
 	rp  *httputil.ReverseProxy
 
 	authMode    bool // Whether to run with impersonation using caller's tailnet identity.
-	https       bool // Whether to serve on https for the device hostname; true for k8s-operator, false for k8s-proxy.
+	https       bool // Whether to serve on https for the device hostname; true for k8s-operator, false (and localhost) for k8s-proxy.
 	ts          *tsnet.Server
 	hs          *http.Server
 	upstreamURL *url.URL
@@ -317,7 +317,23 @@ func (ap *APIServerProxy) addImpersonationHeadersAsRequired(r *http.Request) {
 }
 
 func (ap *APIServerProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) {
-	return ap.lc.WhoIs(r.Context(), r.RemoteAddr)
+	who, remoteErr := ap.lc.WhoIs(r.Context(), r.RemoteAddr)
+	if remoteErr == nil {
+		ap.log.Debugf("WhoIs from remote addr: %s", r.RemoteAddr)
+		return who, nil
+	}
+
+	var fwdErr error
+	fwdFor := r.Header.Get("X-Forwarded-For")
+	if fwdFor != "" && !ap.https {
+		who, fwdErr = ap.lc.WhoIs(r.Context(), fwdFor)
+		if fwdErr == nil {
+			ap.log.Debugf("WhoIs from X-Forwarded-For header: %s", fwdFor)
+			return who, nil
+		}
+	}
+
+	return nil, errors.Join(remoteErr, fwdErr)
 }
 
 func (ap *APIServerProxy) authError(w http.ResponseWriter, err error) {