cmd/natc,wgengine/netstack: tune buffer size and segment lifetime in natc

Some natc instances have been observed with excessive memory growth,
dominant in gvisor buffers. It is likely that the connection buffers are
sticking around for too long due to the default long segment time, and
uptuned buffer size applied by default in wgengine/netstack. Apply
configurations in natc specifically which are a better match for the
natc use case, most notably a 5s maximum segment lifetime.

Updates tailscale/corp#25169

Signed-off-by: James Tucker <james@tailscale.com>

cmd/natc/natc.go

@@ -26,6 +26,8 @@ import (
 	"github.com/inetaf/tcpproxy"
 	"github.com/peterbourgon/ff/v3"
 	"golang.org/x/net/dns/dnsmessage"
+	"gvisor.dev/gvisor/pkg/tcpip"
+	"gvisor.dev/gvisor/pkg/tcpip/transport/tcp"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/envknob"
 	"tailscale.com/hostinfo"
@@ -37,6 +39,7 @@ import (
 	"tailscale.com/tsweb"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/util/mak"
+	"tailscale.com/wgengine/netstack"
 )
 
 func main() {
@@ -112,6 +115,7 @@ func main() {
 		ts.Port = uint16(*wgPort)
 	}
 	defer ts.Close()
+
 	if *verboseTSNet {
 		ts.Logf = log.Printf
 	}
@@ -129,6 +133,33 @@ func main() {
 			log.Fatalf("debug serve: %v", http.Serve(dln, mux))
 		}()
 	}
+
+	if err := ts.Start(); err != nil {
+		log.Fatalf("ts.Start: %v", err)
+	}
+	// TODO(raggi): this is not a public interface or guarantee.
+	ns := ts.Sys().Netstack.Get().(*netstack.Impl)
+	tcpRXBufOpt := tcpip.TCPReceiveBufferSizeRangeOption{
+		Min:     tcp.MinBufferSize,
+		Default: tcp.DefaultReceiveBufferSize,
+		Max:     tcp.MaxBufferSize,
+	}
+	if err := ns.SetTransportProtocolOption(tcp.ProtocolNumber, &tcpRXBufOpt); err != nil {
+		log.Fatalf("could not set TCP RX buf size: %v", err)
+	}
+	tcpTXBufOpt := tcpip.TCPSendBufferSizeRangeOption{
+		Min:     tcp.MinBufferSize,
+		Default: tcp.DefaultSendBufferSize,
+		Max:     tcp.MaxBufferSize,
+	}
+	if err := ns.SetTransportProtocolOption(tcp.ProtocolNumber, &tcpTXBufOpt); err != nil {
+		log.Fatalf("could not set TCP TX buf size: %v", err)
+	}
+	mslOpt := tcpip.TCPTimeWaitTimeoutOption(5 * time.Second)
+	if err := ns.SetTransportProtocolOption(tcp.ProtocolNumber, &mslOpt); err != nil {
+		log.Fatalf("could not set TCP MSL: %v", err)
+	}
+
 	lc, err := ts.LocalClient()
 	if err != nil {
 		log.Fatalf("LocalClient() failed: %v", err)

wgengine/netstack/netstack.go

@@ -405,6 +405,14 @@ func (ns *Impl) Close() error {
 	return nil
 }
 
+// SetTransportProtocolOption forwards to the underlying
+// [stack.Stack.SetTransportProtocolOption]. Callers are responsible for
+// ensuring that the options are valid, compatible and appropriate for their use
+// case. Compatibility may change at any version.
+func (ns *Impl) SetTransportProtocolOption(transport tcpip.TransportProtocolNumber, option tcpip.SettableTransportProtocolOption) tcpip.Error {
+	return ns.ipstack.SetTransportProtocolOption(transport, option)
+}
+
 // A single process might have several netstacks running at the same time.
 // Exported clientmetric counters will have a sum of counters of all of them.
 var stacksForMetrics syncs.Map[*Impl, struct{}]