net/sockopts,wgengine/magicsock: export socket buffer sizing logic (#16909)

For eventual use by net/udprelay.Server

Updates tailscale/corp#31164

Signed-off-by: Jordan Whited <jordan@tailscale.com>

cmd/k8s-operator/depaware.txt

@@ -867,6 +867,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/net/portmapper                                 from tailscale.com/ipn/localapi+
         tailscale.com/net/proxymux                                   from tailscale.com/tsnet
         tailscale.com/net/routetable                                 from tailscale.com/doctor/routetable
+        tailscale.com/net/sockopts                                   from tailscale.com/wgengine/magicsock
         tailscale.com/net/socks5                                     from tailscale.com/tsnet
         tailscale.com/net/sockstats                                  from tailscale.com/control/controlclient+
         tailscale.com/net/stun                                       from tailscale.com/ipn/localapi+

cmd/tailscaled/depaware.txt

@@ -339,6 +339,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/net/portmapper                                 from tailscale.com/ipn/localapi+
         tailscale.com/net/proxymux                                   from tailscale.com/cmd/tailscaled
         tailscale.com/net/routetable                                 from tailscale.com/doctor/routetable
+        tailscale.com/net/sockopts                                   from tailscale.com/wgengine/magicsock
         tailscale.com/net/socks5                                     from tailscale.com/cmd/tailscaled
         tailscale.com/net/sockstats                                  from tailscale.com/control/controlclient+
         tailscale.com/net/stun                                       from tailscale.com/ipn/localapi+

cmd/tsidp/depaware.txt

@@ -297,6 +297,7 @@ tailscale.com/cmd/tsidp dependencies: (generated by github.com/tailscale/depawar
         tailscale.com/net/portmapper                                 from tailscale.com/ipn/localapi+
         tailscale.com/net/proxymux                                   from tailscale.com/tsnet
         tailscale.com/net/routetable                                 from tailscale.com/doctor/routetable
+        tailscale.com/net/sockopts                                   from tailscale.com/wgengine/magicsock
         tailscale.com/net/socks5                                     from tailscale.com/tsnet
         tailscale.com/net/sockstats                                  from tailscale.com/control/controlclient+
         tailscale.com/net/stun                                       from tailscale.com/ipn/localapi+

net/sockopts/sockopts.go

@@ -0,0 +1,37 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package sockopts contains logic for applying socket options.
+package sockopts
+
+import (
+	"net"
+	"runtime"
+
+	"tailscale.com/types/nettype"
+)
+
+// BufferDirection represents either the read/receive or write/send direction
+// of a socket buffer.
+type BufferDirection string
+
+const (
+	ReadDirection  BufferDirection = "read"
+	WriteDirection BufferDirection = "write"
+)
+
+func portableSetBufferSize(pconn nettype.PacketConn, direction BufferDirection, size int) error {
+	if runtime.GOOS == "plan9" {
+		// Not supported. Don't try. Avoid logspam.
+		return nil
+	}
+	var err error
+	if c, ok := pconn.(*net.UDPConn); ok {
+		if direction == WriteDirection {
+			err = c.SetWriteBuffer(size)
+		} else {
+			err = c.SetReadBuffer(size)
+		}
+	}
+	return err
+}

net/sockopts/sockopts_default.go

@@ -0,0 +1,21 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !linux
+
+package sockopts
+
+import (
+	"tailscale.com/types/nettype"
+)
+
+// SetBufferSize sets pconn's buffer to size for direction. size may be silently
+// capped depending on platform.
+//
+// errForce is only relevant for Linux, and will always be nil otherwise,
+// but we maintain a consistent cross-platform API.
+//
+// If pconn is not a [*net.UDPConn], then SetBufferSize is no-op.
+func SetBufferSize(pconn nettype.PacketConn, direction BufferDirection, size int) (errForce error, errPortable error) {
+	return nil, portableSetBufferSize(pconn, direction, size)
+}

net/sockopts/sockopts_linux.go

@@ -0,0 +1,40 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package sockopts
+
+import (
+	"net"
+	"syscall"
+
+	"tailscale.com/types/nettype"
+)
+
+// SetBufferSize sets pconn's buffer to size for direction. It attempts
+// (errForce) to set SO_SNDBUFFORCE or SO_RECVBUFFORCE which can overcome the
+// limit of net.core.{r,w}mem_max, but require CAP_NET_ADMIN. It falls back to
+// the portable implementation (errPortable) if that fails, which may be
+// silently capped to net.core.{r,w}mem_max.
+//
+// If pconn is not a [*net.UDPConn], then SetBufferSize is no-op.
+func SetBufferSize(pconn nettype.PacketConn, direction BufferDirection, size int) (errForce error, errPortable error) {
+	opt := syscall.SO_RCVBUFFORCE
+	if direction == WriteDirection {
+		opt = syscall.SO_SNDBUFFORCE
+	}
+	if c, ok := pconn.(*net.UDPConn); ok {
+		var rc syscall.RawConn
+		rc, errForce = c.SyscallConn()
+		if errForce == nil {
+			rc.Control(func(fd uintptr) {
+				errForce = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, opt, size)
+			})
+		}
+		if errForce != nil {
+			errPortable = portableSetBufferSize(pconn, direction, size)
+		}
+	}
+	return errForce, errPortable
+}

net/sockopts/sockopts_unix_test.go

@@ -3,7 +3,7 @@
 
 //go:build unix
 
-package magicsock
+package sockopts
 
 import (
 	"net"
@@ -13,7 +13,7 @@ import (
 	"tailscale.com/types/nettype"
 )
 
-func TestTrySetSocketBuffer(t *testing.T) {
+func TestSetBufferSize(t *testing.T) {
 	c, err := net.ListenPacket("udp", ":0")
 	if err != nil {
 		t.Fatal(err)
@@ -42,7 +42,8 @@ func TestTrySetSocketBuffer(t *testing.T) {
 
 	curRcv, curSnd := getBufs()
 
-	trySetSocketBuffer(c.(nettype.PacketConn), t.Logf)
+	SetBufferSize(c.(nettype.PacketConn), ReadDirection, 7<<20)
+	SetBufferSize(c.(nettype.PacketConn), WriteDirection, 7<<20)
 
 	newRcv, newSnd := getBufs()
 

tsnet/depaware.txt

@@ -293,6 +293,7 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware)
         tailscale.com/net/portmapper                                 from tailscale.com/ipn/localapi+
         tailscale.com/net/proxymux                                   from tailscale.com/tsnet
         tailscale.com/net/routetable                                 from tailscale.com/doctor/routetable
+        tailscale.com/net/sockopts                                   from tailscale.com/wgengine/magicsock
         tailscale.com/net/socks5                                     from tailscale.com/tsnet
         tailscale.com/net/sockstats                                  from tailscale.com/control/controlclient+
         tailscale.com/net/stun                                       from tailscale.com/ipn/localapi+

wgengine/magicsock/magicsock.go

@@ -45,6 +45,7 @@ import (
 	"tailscale.com/net/packet"
 	"tailscale.com/net/ping"
 	"tailscale.com/net/portmapper"
+	"tailscale.com/net/sockopts"
 	"tailscale.com/net/sockstats"
 	"tailscale.com/net/stun"
 	"tailscale.com/net/tstun"
@@ -3857,20 +3858,19 @@ func (c *Conn) DebugForcePreferDERP(n int) {
 	c.netChecker.SetForcePreferredDERP(n)
 }
 
-// portableTrySetSocketBuffer sets SO_SNDBUF and SO_RECVBUF on pconn to socketBufferSize,
+// trySetSocketBuffer attempts to set SO_SNDBUFFORCE and SO_RECVBUFFORCE which
-// logging an error if it occurs.
+// can overcome the limit of net.core.{r,w}mem_max, but require CAP_NET_ADMIN.
-func portableTrySetSocketBuffer(pconn nettype.PacketConn, logf logger.Logf) {
+// It falls back to the portable implementation if that fails, which may be
-	if runtime.GOOS == "plan9" {
+// silently capped to net.core.{r,w}mem_max.
-		// Not supported. Don't try. Avoid logspam.
+func trySetSocketBuffer(pconn nettype.PacketConn, logf logger.Logf) {
-		return
+	directions := []sockopts.BufferDirection{sockopts.ReadDirection, sockopts.WriteDirection}
-	}
+	for _, direction := range directions {
-	if c, ok := pconn.(*net.UDPConn); ok {
+		forceErr, portableErr := sockopts.SetBufferSize(pconn, direction, socketBufferSize)
-		// Attempt to increase the buffer size, and allow failures.
+		if forceErr != nil {
-		if err := c.SetReadBuffer(socketBufferSize); err != nil {
+			logf("magicsock: [warning] failed to force-set UDP %v buffer size to %d: %v; using kernel default values (impacts throughput only)", direction, socketBufferSize, forceErr)
-			logf("magicsock: failed to set UDP read buffer size to %d: %v", socketBufferSize, err)
 		}
-		if err := c.SetWriteBuffer(socketBufferSize); err != nil {
+		if portableErr != nil {
-			logf("magicsock: failed to set UDP write buffer size to %d: %v", socketBufferSize, err)
+			logf("magicsock: failed to set UDP %v buffer size to %d: %v", direction, socketBufferSize, portableErr)
 		}
 	}
 }

wgengine/magicsock/magicsock_default.go

@@ -9,15 +9,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
-
-	"tailscale.com/types/logger"
-	"tailscale.com/types/nettype"
 )
 
 func (c *Conn) listenRawDisco(family string) (io.Closer, error) {
 	return nil, fmt.Errorf("raw disco listening not supported on this OS: %w", errors.ErrUnsupported)
 }
-
-func trySetSocketBuffer(pconn nettype.PacketConn, logf logger.Logf) {
-	portableTrySetSocketBuffer(pconn, logf)
-}

wgengine/magicsock/magicsock_linux.go

@@ -13,7 +13,6 @@ import (
 	"net"
 	"net/netip"
 	"strings"
-	"syscall"
 	"time"
 
 	"github.com/mdlayher/socket"
@@ -28,7 +27,6 @@ import (
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
-	"tailscale.com/types/nettype"
 )
 
 const (
@@ -489,30 +487,3 @@ func printSockaddr(sa unix.Sockaddr) string {
 		return fmt.Sprintf("unknown(%T)", sa)
 	}
 }
-
-// trySetSocketBuffer attempts to set SO_SNDBUFFORCE and SO_RECVBUFFORCE which
-// can overcome the limit of net.core.{r,w}mem_max, but require CAP_NET_ADMIN.
-// It falls back to the portable implementation if that fails, which may be
-// silently capped to net.core.{r,w}mem_max.
-func trySetSocketBuffer(pconn nettype.PacketConn, logf logger.Logf) {
-	if c, ok := pconn.(*net.UDPConn); ok {
-		var errRcv, errSnd error
-		rc, err := c.SyscallConn()
-		if err == nil {
-			rc.Control(func(fd uintptr) {
-				errRcv = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUFFORCE, socketBufferSize)
-				if errRcv != nil {
-					logf("magicsock: [warning] failed to force-set UDP read buffer size to %d: %v; using kernel default values (impacts throughput only)", socketBufferSize, errRcv)
-				}
-				errSnd = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_SNDBUFFORCE, socketBufferSize)
-				if errSnd != nil {
-					logf("magicsock: [warning] failed to force-set UDP write buffer size to %d: %v; using kernel default values (impacts throughput only)", socketBufferSize, errSnd)
-				}
-			})
-		}
-
-		if err != nil || errRcv != nil || errSnd != nil {
-			portableTrySetSocketBuffer(pconn, logf)
-		}
-	}
-}