net/tstun,wgengine/*: add support for NAT to routes

This adds support to make exit nodes and subnet routers work
when in scenarios where NAT is required.

It also updates the NATConfig to be generated from a `wgcfg.Config` as
that handles merging prefs with the netmap, so it has the required information
about whether an exit node is already configured and whether routes are accepted.

Updates tailscale/corp#8020

Signed-off-by: Maisem Ali <maisem@tailscale.com>

cmd/tailscaled/depaware.txt

@@ -256,6 +256,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/net/tsdial                                     from tailscale.com/control/controlclient+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/control/controlclient+
         tailscale.com/net/tstun                                      from tailscale.com/cmd/tailscaled+
+        tailscale.com/net/tstun/table                                from tailscale.com/net/tstun
         tailscale.com/net/wsconn                                     from tailscale.com/control/controlhttp+
         tailscale.com/paths                                          from tailscale.com/ipn/ipnlocal+
      💣 tailscale.com/portlist                                       from tailscale.com/ipn/ipnlocal
@@ -264,6 +265,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
   LD 💣 tailscale.com/ssh/tailssh                                    from tailscale.com/cmd/tailscaled
         tailscale.com/syncs                                          from tailscale.com/net/netcheck+
         tailscale.com/tailcfg                                        from tailscale.com/client/tailscale/apitype+
+     💣 tailscale.com/tempfork/device                                from tailscale.com/net/tstun/table
   LD    tailscale.com/tempfork/gliderlabs/ssh                        from tailscale.com/ssh/tailssh
         tailscale.com/tka                                            from tailscale.com/ipn/ipnlocal+
    W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces

net/tstun/wrap.go

@@ -25,17 +25,18 @@ import (
 	"tailscale.com/net/connstats"
 	"tailscale.com/net/packet"
 	"tailscale.com/net/tsaddr"
+	"tailscale.com/net/tstun/table"
 	"tailscale.com/syncs"
 	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
-	"tailscale.com/types/netmap"
 	"tailscale.com/types/views"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/mak"
 	"tailscale.com/wgengine/capture"
 	"tailscale.com/wgengine/filter"
+	"tailscale.com/wgengine/wgcfg"
 )
 
 const maxBufferSize = device.MaxMessageSize
@@ -522,10 +523,12 @@ type natV4Config struct {
 	// dstMasqAddrs is map of dst addresses to their respective MasqueradeAsIP
 	// addresses. The MasqueradeAsIP address is the address that should be used
 	// as the source address for packets to dst.
-	dstMasqAddrs views.Map[netip.Addr, netip.Addr] // dst -> masqAddr
+	dstMasqAddrs views.Map[key.NodePublic, netip.Addr] // dst -> masqAddr
 
-	// TODO(maisem/nyghtowl): add support for subnets and exit nodes and test them.
+	// dstAddrToPeerKeyMapper is the routing table used to map a given dst IP to
-	// Determine IP routing table algorithm to use - e.g. ART?
+	// the peer key responsible for that IP.
+	// It only contains peers that require a MasqueradeAsIP address.
+	dstAddrToPeerKeyMapper *table.RoutingTable
 }
 
 // mapDstIP returns the destination IP to use for a packet to dst.
@@ -550,51 +553,55 @@ func (c *natV4Config) selectSrcIP(oldSrc, dst netip.Addr) netip.Addr {
 	if oldSrc != c.nativeAddr {
 		return oldSrc
 	}
-	if eip, ok := c.dstMasqAddrs.GetOk(dst); ok {
+	p, ok := c.dstAddrToPeerKeyMapper.Lookup(dst)
+	if !ok {
+		return oldSrc
+	}
+	if eip, ok := c.dstMasqAddrs.GetOk(p); ok {
 		return eip
 	}
 	return oldSrc
 }
 
-// natConfigFromNetMap generates a natV4Config from nm.
+// natConfigFromWireGuardConfig generates a natV4Config from nm.
 // If v4 NAT is not required, it returns nil.
-func natConfigFromNetMap(nm *netmap.NetworkMap) *natV4Config {
+func natConfigFromWGConfig(wcfg *wgcfg.Config) *natV4Config {
-	if nm == nil || nm.SelfNode == nil {
+	if wcfg == nil {
 		return nil
 	}
-	nativeAddr := findV4(nm.SelfNode.Addresses)
+	nativeAddr := findV4(wcfg.Addresses)
 	if !nativeAddr.IsValid() {
 		return nil
 	}
 	var (
-		dstMasqAddrs map[netip.Addr]netip.Addr
+		rt           table.RoutingTableBuilder
+		dstMasqAddrs map[key.NodePublic]netip.Addr
 		listenAddrs  map[netip.Addr]struct{}
 	)
-	for _, p := range nm.Peers {
+	for i := range wcfg.Peers {
-		if !p.SelfNodeV4MasqAddrForThisPeer.IsValid() {
+		p := &wcfg.Peers[i]
-			continue
+		if !p.V4MasqAddr.IsValid() {
-		}
-		peerV4 := findV4(p.Addresses)
-		if !peerV4.IsValid() {
 			continue
 		}
-		mak.Set(&dstMasqAddrs, peerV4, p.SelfNodeV4MasqAddrForThisPeer)
+		rt.InsertOrReplace(p.PublicKey, p.AllowedIPs...)
-		mak.Set(&listenAddrs, p.SelfNodeV4MasqAddrForThisPeer, struct{}{})
+		mak.Set(&dstMasqAddrs, p.PublicKey, p.V4MasqAddr)
+		mak.Set(&listenAddrs, p.V4MasqAddr, struct{}{})
 	}
 	if len(listenAddrs) == 0 || len(dstMasqAddrs) == 0 {
 		return nil
 	}
 	return &natV4Config{
-		nativeAddr:   nativeAddr,
+		nativeAddr:             nativeAddr,
-		listenAddrs:  views.MapOf(listenAddrs),
+		listenAddrs:            views.MapOf(listenAddrs),
-		dstMasqAddrs: views.MapOf(dstMasqAddrs),
+		dstMasqAddrs:           views.MapOf(dstMasqAddrs),
+		dstAddrToPeerKeyMapper: rt.Build(),
 	}
 }
 
 // SetNetMap is called when a new NetworkMap is received.
 // It currently (2023-03-01) only updates the IPv4 NAT configuration.
-func (t *Wrapper) SetNetMap(nm *netmap.NetworkMap) {
+func (t *Wrapper) SetWGConfig(wcfg *wgcfg.Config) {
-	cfg := natConfigFromNetMap(nm)
+	cfg := natConfigFromWGConfig(wcfg)
 	old := t.natV4Config.Swap(cfg)
 	if !reflect.DeepEqual(old, cfg) {
 		t.logf("nat config: %+v", cfg)

net/tstun/wrap_test.go

@@ -25,16 +25,15 @@ import (
 	"tailscale.com/net/connstats"
 	"tailscale.com/net/netaddr"
 	"tailscale.com/net/packet"
-	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netlogtype"
-	"tailscale.com/types/netmap"
 	"tailscale.com/util/must"
 	"tailscale.com/wgengine/filter"
+	"tailscale.com/wgengine/wgcfg"
 )
 
 func udp4(src, dst string, sport, dport uint16) []byte {
@@ -597,13 +596,16 @@ func TestFilterDiscoLoop(t *testing.T) {
 }
 
 func TestNATCfg(t *testing.T) {
-	node := func(ip, eip netip.Addr) *tailcfg.Node {
+	node := func(ip, eip netip.Addr, otherAllowedIPs ...netip.Prefix) wgcfg.Peer {
-		return &tailcfg.Node{
+		p := wgcfg.Peer{
-			Addresses: []netip.Prefix{
+			PublicKey: key.NewNode().Public(),
+			AllowedIPs: []netip.Prefix{
 				netip.PrefixFrom(ip, ip.BitLen()),
 			},
-			SelfNodeV4MasqAddrForThisPeer: eip,
+			V4MasqAddr: eip,
 		}
+		p.AllowedIPs = append(p.AllowedIPs, otherAllowedIPs...)
+		return p
 	}
 	var (
 		noIP netip.Addr
@@ -615,20 +617,20 @@ func TestNATCfg(t *testing.T) {
 		peer1IP = netip.MustParseAddr("100.64.0.2")
 		peer2IP = netip.MustParseAddr("100.64.0.3")
 
-		// subnets should not be impacted.
-		// TODO(maisem/nyghtowl): add support for subnets and exit nodes and test them.
 		subnet = netip.MustParseAddr("192.168.0.1")
+
+		selfAddrs = []netip.Prefix{netip.PrefixFrom(selfNativeIP, selfNativeIP.BitLen())}
 	)
 
 	tests := []struct {
 		name    string
-		nm      *netmap.NetworkMap
+		wcfg    *wgcfg.Config
 		snatMap map[netip.Addr]netip.Addr // dst -> src
 		dnatMap map[netip.Addr]netip.Addr
 	}{
 		{
-			name: "no-netmap",
+			name: "no-cfg",
-			nm:   nil,
+			wcfg: nil,
 			snatMap: map[netip.Addr]netip.Addr{
 				peer1IP: selfNativeIP,
 				peer2IP: selfNativeIP,
@@ -642,9 +644,9 @@ func TestNATCfg(t *testing.T) {
 		},
 		{
 			name: "single-peer-requires-nat",
-			nm: &netmap.NetworkMap{
+			wcfg: &wgcfg.Config{
-				SelfNode: node(selfNativeIP, noIP),
+				Addresses: selfAddrs,
-				Peers: []*tailcfg.Node{
+				Peers: []wgcfg.Peer{
 					node(peer1IP, noIP),
 					node(peer2IP, selfEIP1),
 				},
@@ -663,9 +665,9 @@ func TestNATCfg(t *testing.T) {
 		},
 		{
 			name: "multiple-peers-require-nat",
-			nm: &netmap.NetworkMap{
+			wcfg: &wgcfg.Config{
-				SelfNode: node(selfNativeIP, noIP),
+				Addresses: selfAddrs,
-				Peers: []*tailcfg.Node{
+				Peers: []wgcfg.Peer{
 					node(peer1IP, selfEIP1),
 					node(peer2IP, selfEIP2),
 				},
@@ -682,11 +684,53 @@ func TestNATCfg(t *testing.T) {
 				subnet:       subnet,
 			},
 		},
+		{
+			name: "multiple-peers-require-nat-with-subnet",
+			wcfg: &wgcfg.Config{
+				Addresses: selfAddrs,
+				Peers: []wgcfg.Peer{
+					node(peer1IP, selfEIP1),
+					node(peer2IP, selfEIP2, netip.MustParsePrefix("192.168.0.0/24")),
+				},
+			},
+			snatMap: map[netip.Addr]netip.Addr{
+				peer1IP: selfEIP1,
+				peer2IP: selfEIP2,
+				subnet:  selfEIP2,
+			},
+			dnatMap: map[netip.Addr]netip.Addr{
+				selfNativeIP: selfNativeIP,
+				selfEIP1:     selfNativeIP,
+				selfEIP2:     selfNativeIP,
+				subnet:       subnet,
+			},
+		},
+		{
+			name: "multiple-peers-require-nat-with-default-route",
+			wcfg: &wgcfg.Config{
+				Addresses: selfAddrs,
+				Peers: []wgcfg.Peer{
+					node(peer1IP, selfEIP1),
+					node(peer2IP, selfEIP2, netip.MustParsePrefix("0.0.0.0/0")),
+				},
+			},
+			snatMap: map[netip.Addr]netip.Addr{
+				peer1IP:                        selfEIP1,
+				peer2IP:                        selfEIP2,
+				netip.MustParseAddr("8.8.8.8"): selfEIP2,
+			},
+			dnatMap: map[netip.Addr]netip.Addr{
+				selfNativeIP: selfNativeIP,
+				selfEIP1:     selfNativeIP,
+				selfEIP2:     selfNativeIP,
+				subnet:       subnet,
+			},
+		},
 		{
 			name: "no-nat",
-			nm: &netmap.NetworkMap{
+			wcfg: &wgcfg.Config{
-				SelfNode: node(selfNativeIP, noIP),
+				Addresses: selfAddrs,
-				Peers: []*tailcfg.Node{
+				Peers: []wgcfg.Peer{
 					node(peer1IP, noIP),
 					node(peer2IP, noIP),
 				},
@@ -707,15 +751,15 @@ func TestNATCfg(t *testing.T) {
 
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
-			ncfg := natConfigFromNetMap(tc.nm)
+			ncfg := natConfigFromWGConfig(tc.wcfg)
 			for peer, want := range tc.snatMap {
 				if got := ncfg.selectSrcIP(selfNativeIP, peer); got != want {
-					t.Errorf("selectSrcIP: got %v; want %v", got, want)
+					t.Errorf("selectSrcIP[%v]: got %v; want %v", peer, got, want)
 				}
 			}
 			for dstIP, want := range tc.dnatMap {
 				if got := ncfg.mapDstIP(dstIP); got != want {
-					t.Errorf("mapDstIP: got %v; want %v", got, want)
+					t.Errorf("mapDstIP[%v]: got %v; want %v", dstIP, got, want)
 				}
 			}
 		})

wgengine/userspace.go

@@ -807,6 +807,7 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config,
 
 	e.wgLock.Lock()
 	defer e.wgLock.Unlock()
+	e.tundev.SetWGConfig(cfg)
 	e.lastDNSConfig = dnsCfg
 
 	peerSet := make(map[key.NodePublic]struct{}, len(cfg.Peers))
@@ -1205,7 +1206,6 @@ func (e *userspaceEngine) SetNetworkMap(nm *netmap.NetworkMap) {
 	e.magicConn.SetNetworkMap(nm)
 	e.mu.Lock()
 	e.netMap = nm
-	e.tundev.SetNetMap(nm)
 	callbacks := make([]NetworkMapCallback, 0, 4)
 	for _, fn := range e.networkMapCallbacks {
 		callbacks = append(callbacks, fn)

wgengine/wgcfg/config.go

@@ -37,6 +37,7 @@ type Peer struct {
 	PublicKey           key.NodePublic
 	DiscoKey            key.DiscoPublic // present only so we can handle restarts within wgengine, not passed to WireGuard
 	AllowedIPs          []netip.Prefix
+	V4MasqAddr          netip.Addr // if non-zero, masquerade IPv4 traffic to this peer using this address
 	PersistentKeepalive uint16
 	// wireguard-go's endpoint for this peer. It should always equal Peer.PublicKey.
 	// We represent it explicitly so that we can detect if they diverge and recover.

wgengine/wgcfg/nmcfg/nmcfg.go

@@ -101,6 +101,7 @@ func WGCfg(nm *netmap.NetworkMap, logf logger.Logf, flags netmap.WGConfigFlags,
 		}
 
 		didExitNodeWarn := false
+		cpeer.V4MasqAddr = peer.SelfNodeV4MasqAddrForThisPeer
 		for _, allowedIP := range peer.AllowedIPs {
 			if allowedIP.Bits() == 0 && peer.StableID != exitNode {
 				if didExitNodeWarn {

wgengine/wgcfg/wgcfg_clone.go

@@ -62,6 +62,7 @@ var _PeerCloneNeedsRegeneration = Peer(struct {
 	PublicKey           key.NodePublic
 	DiscoKey            key.DiscoPublic
 	AllowedIPs          []netip.Prefix
+	V4MasqAddr          netip.Addr
 	PersistentKeepalive uint16
 	WGEndpoint          key.NodePublic
 }{})