@ -11,9 +11,7 @@ import (
"net"
"net/netip"
"reflect"
"runtime"
"slices"
"sync"
"time"
"unsafe"
@ -32,7 +30,6 @@ import (
"tailscale.com/types/logger"
"tailscale.com/util/mak"
"tailscale.com/util/rands"
"tailscale.com/util/sysresources"
"tailscale.com/util/testenv"
)
@ -282,59 +279,20 @@ func (c *Conn) goDerpConnect(regionID int) {
go c . derpWriteChanForRegion ( regionID , key . NodePublic { } )
}
var (
bufferedDerpWrites int
bufferedDerpWritesOnce sync . Once
)
// bufferedDerpWritesBeforeDrop returns how many packets writes can be queued
// up the DERP client to write on the wire before we start dropping.
func bufferedDerpWritesBeforeDrop ( ) int {
// For mobile devices, always return the previous minimum value of 32;
// we can do this outside the sync.Once to avoid that overhead.
if runtime . GOOS == "ios" || runtime . GOOS == "android" {
return 32
}
bufferedDerpWritesOnce . Do ( func ( ) {
// Some rough sizing: for the previous fixed value of 32, the
// total consumed memory can be:
// = numDerpRegions * messages/region * sizeof(message)
//
// For sake of this calculation, assume 100 DERP regions; at
// time of writing (2023-04-03), we have 24.
//
// A reasonable upper bound for the worst-case average size of
// a message is a *disco.CallMeMaybe message with 16 endpoints;
// since sizeof(netip.AddrPort) = 32, that's 512 bytes. Thus:
// = 100 * 32 * 512
// = 1638400 (1.6MiB)
//
// On a reasonably-small node with 4GiB of memory that's
// connected to each region and handling a lot of load, 1.6MiB
// is about 0.04% of the total system memory.
//
// For sake of this calculation, then, let's double that memory
// usage to 0.08% and scale based on total system memory.
//
// For a 16GiB Linux box, this should buffer just over 256
// messages.
systemMemory := sysresources . TotalMemory ( )
memoryUsable := float64 ( systemMemory ) * 0.0008
const (
theoreticalDERPRegions = 100
messageMaximumSizeBytes = 512
)
bufferedDerpWrites = int ( memoryUsable / ( theoreticalDERPRegions * messageMaximumSizeBytes ) )
// Never drop below the previous minimum value.
if bufferedDerpWrites < 32 {
bufferedDerpWrites = 32
}
} )
return bufferedDerpWrites
}
// derpWriteQueueDepth is the depth of the in-process write queue to a single
// DERP region. DERP connections are TCP, and so the actual write queue depth is
// substantially larger than this suggests - often scaling into megabytes
// depending on dynamic TCP parameters and platform TCP tuning. This queue is
// excess of the TCP buffer depth, which means it's almost pure buffer bloat,
// and does not want to be deep - if there are key situations where a node can't
// keep up, either the TCP link to DERP is too slow, or there is a
// synchronization issue in the write path, fixes should be focused on those
// paths, rather than extending this queue.
// TODO(raggi): make this even shorter, ideally this should be a fairly direct
// line into a socket TCP buffer. The challenge at present is that connect and
// reconnect are in the write path and we don't want to block other write
// operations on those.
const derpWriteQueueDepth = 32
// derpWriteChanForRegion returns a channel to which to send DERP packet write
// requests. It creates a new DERP connection to regionID if necessary.
@ -429,7 +387,7 @@ func (c *Conn) derpWriteChanForRegion(regionID int, peer key.NodePublic) chan<-
dc . DNSCache = dnscache . Get ( )
ctx , cancel := context . WithCancel ( c . connCtx )
ch := make ( chan derpWriteRequest , bufferedDerpWritesBeforeDrop ( ) )
ch := make ( chan derpWriteRequest , derpWriteQueueDepth )
ad . c = dc
ad . writeCh = ch
James Tucker
3 months ago
committed by