wgengine/netstack: add local tailscale service IPs to route and terminate locally

This commit adds the tailscales service IPs served locally to OS routes, and
make interception to packets so that the traffic terminates locally without
making affects to the HA traffics.

Fixes tailscale/corp#34048

Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com>

ipn/ipnlocal/local.go

@@ -917,6 +917,18 @@ func (b *LocalBackend) setStateLocked(state ipn.State) {
 	}
 }
 
+func (b *LocalBackend) IPServiceMappings() netmap.IPServiceMappings {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.ipVIPServiceMap
+}
+
+func (b *LocalBackend) SetIPServiceMappingsForTesting(m netmap.IPServiceMappings) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.ipVIPServiceMap = m
+}
+
 // setConfigLocked uses the provided config to update the backend's prefs
 // and other state.
 func (b *LocalBackend) setConfigLocked(conf *conffile.Config) error {
@@ -5120,7 +5132,7 @@ func (b *LocalBackend) authReconfigLocked() {
 	}
 
 	oneCGNATRoute := shouldUseOneCGNATRoute(b.logf, b.sys.NetMon.Get(), b.sys.ControlKnobs(), version.OS())
-	rcfg := b.routerConfigLocked(cfg, prefs, oneCGNATRoute)
+	rcfg := b.routerConfigLocked(cfg, prefs, nm, oneCGNATRoute)
 
 	err = b.e.Reconfig(cfg, rcfg, dcfg)
 	if err == wgengine.ErrNoChanges {
@@ -5445,7 +5457,7 @@ func peerRoutes(logf logger.Logf, peers []wgcfg.Peer, cgnatThreshold int, routeA
 // routerConfig produces a router.Config from a wireguard config and IPN prefs.
 //
 // b.mu must be held.
-func (b *LocalBackend) routerConfigLocked(cfg *wgcfg.Config, prefs ipn.PrefsView, oneCGNATRoute bool) *router.Config {
+func (b *LocalBackend) routerConfigLocked(cfg *wgcfg.Config, prefs ipn.PrefsView, nm *netmap.NetworkMap, oneCGNATRoute bool) *router.Config {
 	singleRouteThreshold := 10_000
 	if oneCGNATRoute {
 		singleRouteThreshold = 1
@@ -5530,11 +5542,25 @@ func (b *LocalBackend) routerConfigLocked(cfg *wgcfg.Config, prefs ipn.PrefsView
 		}
 	}
 
+	// Get the VIPs for VIP services this node hosts. We will add all locally served VIPs to routes then
+	// we terminate these connection locally in netstack instead of routing to peer.
+	VIPServiceIPs := nm.GetIPVIPServiceMap()
+
 	if slices.ContainsFunc(rs.LocalAddrs, tsaddr.PrefixIs4) {
 		rs.Routes = append(rs.Routes, netip.PrefixFrom(tsaddr.TailscaleServiceIP(), 32))
+		for vip := range VIPServiceIPs {
+			if vip.Is4() {
+				rs.Routes = append(rs.Routes, netip.PrefixFrom(vip, 32))
+			}
+		}
 	}
 	if slices.ContainsFunc(rs.LocalAddrs, tsaddr.PrefixIs6) {
 		rs.Routes = append(rs.Routes, netip.PrefixFrom(tsaddr.TailscaleServiceIPv6(), 128))
+		for vip := range VIPServiceIPs {
+			if vip.Is6() {
+				rs.Routes = append(rs.Routes, netip.PrefixFrom(vip, 128))
+			}
+		}
 	}
 
 	return rs

wgengine/netstack/netstack.go

@@ -771,6 +771,11 @@ func (ns *Impl) handleLocalPackets(p *packet.Parsed, t *tstun.Wrapper, gro *gro.
 
 	// Determine if we care about this local packet.
 	dst := p.Dst.Addr()
+	var IPServiceMappings netmap.IPServiceMappings
+	if ns.lb != nil {
+		IPServiceMappings = ns.lb.IPServiceMappings()
+	}
+	serviceName, isVIPServiceIP := IPServiceMappings[dst]
 	switch {
 	case dst == serviceIP || dst == serviceIPv6:
 		// We want to intercept some traffic to the "service IP" (e.g.
@@ -787,6 +792,30 @@ func (ns *Impl) handleLocalPackets(p *packet.Parsed, t *tstun.Wrapper, gro *gro.
 				return filter.Accept, gro
 			}
 		}
+	case isVIPServiceIP:
+		if p.IPProto != ipproto.TCP {
+			return filter.Accept, gro
+		}
+		// returns all configured VIP services, since the IPServiceMappings contains
+		// inactive service IPs when node hosts the service, we need to check the
+		// service is active or not before dropping the packet.
+		VIPServices := ns.lb.VIPServices()
+		serviceActive := false
+		for _, svc := range VIPServices {
+			// Even though control only send service IP down when there is a config
+			// for the service, we want to still check that the config still exists
+			// before passing the packet to netstack.
+			if svc.Name == serviceName {
+				serviceActive = svc.Active
+			}
+		}
+		if !serviceActive {
+			return filter.Accept, gro
+		}
+		if debugNetstack() {
+			ns.logf("netstack: intercepting local VIP service packet: proto=%v dst=%v src=%v",
+				p.IPProto, p.Dst, p.Src)
+		}
 	case viaRange.Contains(dst):
 		// We need to handle 4via6 packets leaving the host if the via
 		// route is for this host; otherwise the packet will be dropped
@@ -998,12 +1027,32 @@ func (ns *Impl) shouldSendToHost(pkt *stack.PacketBuffer) bool {
 			return true
 		}
 
+		if ns.isVIPServiceIP(srcIP) {
+			dstIP := netip.AddrFrom4(v.DestinationAddress().As4())
+			if ns.isLocalIP(dstIP) {
+				if debugNetstack() {
+					ns.logf("netstack: sending VIP service packet to host: src=%v dst=%v", srcIP, dstIP)
+				}
+				return true
+			}
+		}
+
 	case header.IPv6:
 		srcIP := netip.AddrFrom16(v.SourceAddress().As16())
 		if srcIP == serviceIPv6 {
 			return true
 		}
 
+		if ns.isVIPServiceIP(srcIP) {
+			dstIP := netip.AddrFrom16(v.DestinationAddress().As16())
+			if ns.isLocalIP(dstIP) {
+				if debugNetstack() {
+					ns.logf("netstack: sending VIP service packet to host: src=%v dst=%v", srcIP, dstIP)
+				}
+				return true
+			}
+		}
+
 		if viaRange.Contains(srcIP) {
 			// Only send to the host if this 4via6 route is
 			// something this node handles.

wgengine/netstack/netstack_test.go

@@ -31,6 +31,7 @@ import (
 	"tailscale.com/tstest"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/logid"
+	"tailscale.com/types/netmap"
 	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/filter"
 )
@@ -741,13 +742,20 @@ func TestHandleLocalPackets(t *testing.T) {
 		// fd7a:115c:a1e0:b1a:0:7:a01:100/120
 		netip.MustParsePrefix("fd7a:115c:a1e0:b1a:0:7:a01:100/120"),
 	}
+	prefs.AdvertiseServices = []string{"svc:test-service"}
 	_, err := impl.lb.EditPrefs(&ipn.MaskedPrefs{
-		Prefs:              *prefs,
-		AdvertiseRoutesSet: true,
+		Prefs:                *prefs,
+		AdvertiseRoutesSet:   true,
+		AdvertiseServicesSet: true,
 	})
 	if err != nil {
 		t.Fatalf("EditPrefs: %v", err)
 	}
+	IPServiceMap := netmap.IPServiceMappings{
+		netip.MustParseAddr("100.99.55.111"):        "svc:test-service",
+		netip.MustParseAddr("fd7a:115c:a1e0::abcd"): "svc:test-service",
+	}
+	impl.lb.SetIPServiceMappingsForTesting(IPServiceMap)
 
 	t.Run("ShouldHandleServiceIP", func(t *testing.T) {
 		pkt := &packet.Parsed{
@@ -784,6 +792,19 @@ func TestHandleLocalPackets(t *testing.T) {
 			t.Errorf("got filter outcome %v, want filter.DropSilently", resp)
 		}
 	})
+	t.Run("ShouldHandleLocalTailscaleServices", func(t *testing.T) {
+		pkt := &packet.Parsed{
+			IPVersion: 4,
+			IPProto:   ipproto.TCP,
+			Src:       netip.MustParseAddrPort("127.0.0.1:9999"),
+			Dst:       netip.MustParseAddrPort("100.99.55.111:80"),
+			TCPFlags:  packet.TCPSyn,
+		}
+		resp, _ := impl.handleLocalPackets(pkt, impl.tundev, nil)
+		if resp != filter.DropSilently {
+			t.Errorf("got filter outcome %v, want filter.DropSilently", resp)
+		}
+	})
 	t.Run("OtherNonHandled", func(t *testing.T) {
 		pkt := &packet.Parsed{
 			IPVersion: 6,
@@ -809,8 +830,10 @@ func TestHandleLocalPackets(t *testing.T) {
 
 func TestShouldSendToHost(t *testing.T) {
 	var (
-		selfIP4 = netip.MustParseAddr("100.64.1.2")
-		selfIP6 = netip.MustParseAddr("fd7a:115c:a1e0::123")
+		selfIP4             = netip.MustParseAddr("100.64.1.2")
+		selfIP6             = netip.MustParseAddr("fd7a:115c:a1e0::123")
+		tailscaleServiceIP4 = netip.MustParseAddr("100.99.55.111")
+		tailscaleServiceIP6 = netip.MustParseAddr("fd7a:115c:a1e0::abcd")
 	)
 
 	makeTestNetstack := func(tb testing.TB) *Impl {
@@ -820,6 +843,9 @@ func TestShouldSendToHost(t *testing.T) {
 			impl.atomicIsLocalIPFunc.Store(func(addr netip.Addr) bool {
 				return addr == selfIP4 || addr == selfIP6
 			})
+			impl.atomicIsVIPServiceIPFunc.Store(func(addr netip.Addr) bool {
+				return addr == tailscaleServiceIP4 || addr == tailscaleServiceIP6
+			})
 		})
 
 		prefs := ipn.NewPrefs()
@@ -919,6 +945,33 @@ func TestShouldSendToHost(t *testing.T) {
 			dst:  netip.MustParseAddrPort("[fd7a:115:a1e0::99]:7777"),
 			want: false,
 		},
+		// After accessing the Tailscale service from host, replies from Tailscale Service IPs
+		// to the local Tailscale IPs should be sent to the host.
+		{
+			name: "from_service_ip_to_local_ip",
+			src:  netip.AddrPortFrom(tailscaleServiceIP4, 80),
+			dst:  netip.AddrPortFrom(selfIP4, 12345),
+			want: true,
+		},
+		{
+			name: "from_service_ip_to_local_ip_v6",
+			src:  netip.AddrPortFrom(tailscaleServiceIP6, 80),
+			dst:  netip.AddrPortFrom(selfIP6, 12345),
+			want: true,
+		},
+		// Traffic from remote IPs to Tailscale Service IPs should be sent over WireGuard.
+		{
+			name: "from_service_ip_to_remote",
+			src:  netip.AddrPortFrom(tailscaleServiceIP4, 80),
+			dst:  netip.MustParseAddrPort("173.201.32.56:54321"),
+			want: false,
+		},
+		{
+			name: "from_service_ip_to_remote_v6",
+			src:  netip.AddrPortFrom(tailscaleServiceIP6, 80),
+			dst:  netip.MustParseAddrPort("[2001:4860:4860::8888]:54321"),
+			want: false,
+		},
 	}
 
 	for _, tt := range testCases {