cmd/tailscaled, wgengine{,/netstack}: add netstack hybrid mode, add to Windows

For #707

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

cmd/tailscaled/tailscaled.go

@@ -230,24 +230,19 @@ func run() error {
 	}
 
 	var ns *netstack.Impl
-	if useNetstack {
+	if useNetstack || wrapNetstack {
-		tunDev, magicConn, ok := e.(wgengine.InternalsGetter).GetInternals()
+		onlySubnets := wrapNetstack && !useNetstack
-		if !ok {
+		mustStartNetstack(logf, e, onlySubnets)
-			log.Fatalf("%T is not a wgengine.InternalsGetter", e)
-		}
-		ns, err = netstack.Create(logf, tunDev, e, magicConn)
-		if err != nil {
-			log.Fatalf("netstack.Create: %v", err)
-		}
-		if err := ns.Start(); err != nil {
-			log.Fatalf("failed to start netstack: %v", err)
-		}
 	}
 
 	if socksListener != nil {
 		srv := &socks5.Server{
 			Logf: logger.WithPrefix(logf, "socks5: "),
 		}
+		// TODO: also consider wrapNetstack, where dials can go to either Tailscale
+		// or non-Tailscale targets. But that's also basically what
+		// https://github.com/tailscale/tailscale/issues/1617 is about, so do them
+		// both at the same time.
 		if useNetstack {
 			srv.Dialer = func(ctx context.Context, network, addr string) (net.Conn, error) {
 				return ns.DialContextTCP(ctx, addr)
@@ -331,6 +326,25 @@ func createEngine(logf logger.Logf, linkMon *monitor.Mon) (e wgengine.Engine, us
 	return nil, false, multierror.New(errs)
 }
 
+var wrapNetstack = shouldWrapNetstack()
+
+func shouldWrapNetstack() bool {
+	if e := os.Getenv("TS_DEBUG_WRAP_NETSTACK"); e != "" {
+		v, err := strconv.ParseBool(e)
+		if err != nil {
+			log.Fatalf("invalid TS_DEBUG_WRAP_NETSTACK value: %v", err)
+		}
+		return v
+	}
+	switch runtime.GOOS {
+	case "windows", "darwin":
+		// Enable on Windows and tailscaled-on-macOS (this doesn't
+		// affect the GUI clients).
+		return true
+	}
+	return false
+}
+
 func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.Engine, useNetstack bool, err error) {
 	conf := wgengine.Config{
 		ListenPort:  args.port,
@@ -349,8 +363,11 @@ func tryEngine(logf logger.Logf, linkMon *monitor.Mon, name string) (e wgengine.
 			dev.Close()
 			return nil, false, err
 		}
-		conf.Router = r
 		conf.DNS = dns.NewOSConfigurator(logf, devName)
+		conf.Router = r
+		if wrapNetstack {
+			conf.Router = netstack.NewSubnetRouterWrapper(conf.Router)
+		}
 	}
 	e, err = wgengine.NewUserspaceEngine(logf, conf)
 	if err != nil {
@@ -378,3 +395,17 @@ func runDebugServer(mux *http.ServeMux, addr string) {
 		log.Fatal(err)
 	}
 }
+
+func mustStartNetstack(logf logger.Logf, e wgengine.Engine, onlySubnets bool) {
+	tunDev, magicConn, ok := e.(wgengine.InternalsGetter).GetInternals()
+	if !ok {
+		log.Fatalf("%T is not a wgengine.InternalsGetter", e)
+	}
+	ns, err := netstack.Create(logf, tunDev, e, magicConn, onlySubnets)
+	if err != nil {
+		log.Fatalf("netstack.Create: %v", err)
+	}
+	if err := ns.Start(); err != nil {
+		log.Fatalf("failed to start netstack: %v", err)
+	}
+}

cmd/tailscaled/tailscaled_windows.go

@@ -36,6 +36,7 @@ import (
 	"tailscale.com/types/logger"
 	"tailscale.com/version"
 	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/netstack"
 	"tailscale.com/wgengine/router"
 )
 
@@ -171,6 +172,9 @@ func startIPNServer(ctx context.Context, logid string) error {
 			dev.Close()
 			return nil, err
 		}
+		if wrapNetstack {
+			r = netstack.NewSubnetRouterWrapper(r)
+		}
 		eng, err := wgengine.NewUserspaceEngine(logf, wgengine.Config{
 			Tun:        dev,
 			Router:     r,
@@ -182,6 +186,10 @@ func startIPNServer(ctx context.Context, logid string) error {
 			dev.Close()
 			return nil, err
 		}
+		onlySubnets := true
+		if wrapNetstack {
+			mustStartNetstack(logf, eng, onlySubnets)
+		}
 		return wgengine.NewWatchdog(eng), nil
 	}
 

ipn/ipnlocal/local.go

@@ -639,7 +639,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 
 		// Don't warn about broken Linux IP forwading when
 		// netstack is being used.
-		SkipIPForwardingCheck: wgengine.IsNetstack(b.e),
+		SkipIPForwardingCheck: wgengine.IsNetstackRouter(b.e),
 	})
 	if err != nil {
 		return err
@@ -2120,7 +2120,7 @@ func isBSD(s string) bool {
 }
 
 func (b *LocalBackend) CheckIPForwarding() error {
-	if wgengine.IsNetstack(b.e) {
+	if wgengine.IsNetstackRouter(b.e) {
 		return nil
 	}
 	if isBSD(runtime.GOOS) {

wgengine/netstack/netstack.go

@@ -47,12 +47,13 @@ const debugNetstack = false
 // and implements wgengine.FakeImpl to act as a userspace network
 // stack when Tailscale is running in fake mode.
 type Impl struct {
-	ipstack *stack.Stack
+	ipstack     *stack.Stack
-	linkEP  *channel.Endpoint
+	linkEP      *channel.Endpoint
-	tundev  *tstun.Wrapper
+	tundev      *tstun.Wrapper
-	e       wgengine.Engine
+	e           wgengine.Engine
-	mc      *magicsock.Conn
+	mc          *magicsock.Conn
-	logf    logger.Logf
+	logf        logger.Logf
+	onlySubnets bool // whether we only want to handle subnet relaying
 
 	mu  sync.Mutex
 	dns DNSMap
@@ -67,7 +68,7 @@ const nicID = 1
 const mtu = 1500
 
 // Create creates and populates a new Impl.
-func Create(logf logger.Logf, tundev *tstun.Wrapper, e wgengine.Engine, mc *magicsock.Conn) (*Impl, error) {
+func Create(logf logger.Logf, tundev *tstun.Wrapper, e wgengine.Engine, mc *magicsock.Conn, onlySubnets bool) (*Impl, error) {
 	if mc == nil {
 		return nil, errors.New("nil magicsock.Conn")
 	}
@@ -116,11 +117,13 @@ func Create(logf logger.Logf, tundev *tstun.Wrapper, e wgengine.Engine, mc *magi
 		e:                   e,
 		mc:                  mc,
 		connsOpenBySubnetIP: make(map[netaddr.IP]int),
+		onlySubnets:         onlySubnets,
 	}
 	return ns, nil
 }
 
 // Start sets up all the handlers so netstack can start working. Implements
+
 // wgengine.FakeImpl.
 func (ns *Impl) Start() error {
 	ns.e.AddNetworkMapCallback(ns.updateIPs)
@@ -223,7 +226,15 @@ func (ns *Impl) updateIPs(nm *netmap.NetworkMap) {
 		oldIPs[protocolAddr.AddressWithPrefix] = true
 	}
 	newIPs := make(map[tcpip.AddressWithPrefix]bool)
+
+	isAddr := map[netaddr.IPPrefix]bool{}
+	for _, ipp := range nm.SelfNode.Addresses {
+		isAddr[ipp] = true
+	}
 	for _, ipp := range nm.SelfNode.AllowedIPs {
+		if ns.onlySubnets && isAddr[ipp] {
+			continue
+		}
 		newIPs[ipPrefixToAddressWithPrefix(ipp)] = true
 	}
 

wgengine/netstack/subnet_router_wrapper.go

@@ -0,0 +1,36 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package netstack
+
+import (
+	"reflect"
+
+	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/router"
+)
+
+func init() {
+	wgengine.NetstackRouterType = reflect.TypeOf(&subnetRouter{})
+}
+
+type subnetRouter struct {
+	router.Router
+}
+
+// NewSubnetRouterWrapper returns a Router wrapper that prevents the
+// underlying Router r from seeing any advertised subnet routes, as
+// netstack will handle them instead.
+func NewSubnetRouterWrapper(r router.Router) router.Router {
+	return &subnetRouter{
+		Router: r,
+	}
+}
+
+func (r *subnetRouter) Set(c *router.Config) error {
+	if c != nil {
+		c.SubnetRoutes = nil // netstack will handle
+	}
+	return r.Router.Set(c)
+}

wgengine/userspace.go

@@ -14,6 +14,7 @@ import (
 	"io"
 	"net"
 	"os"
+	"reflect"
 	"runtime"
 	"strconv"
 	"strings"
@@ -169,6 +170,25 @@ func NewFakeUserspaceEngine(logf logger.Logf, listenPort uint16) (Engine, error)
 	})
 }
 
+// NetstackRouterType is a gross cross-package init-time registration
+// from netstack to here, informing this package of netstack's router
+// type.
+var NetstackRouterType reflect.Type
+
+// IsNetstackRouter reports whether e is either fully netstack based
+// (without TUN) or is at least using netstack for routing.
+func IsNetstackRouter(e Engine) bool {
+	switch e := e.(type) {
+	case *userspaceEngine:
+		if reflect.TypeOf(e.router) == NetstackRouterType {
+			return true
+		}
+	case *watchdogEngine:
+		return IsNetstackRouter(e.wrap)
+	}
+	return IsNetstack(e)
+}
+
 // IsNetstack reports whether e is a netstack-based TUN-free engine.
 func IsNetstack(e Engine) bool {
 	ig, ok := e.(InternalsGetter)

wgengine/userspace_ext_test.go

@@ -0,0 +1,92 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package wgengine_test
+
+import (
+	"testing"
+
+	"github.com/tailscale/wireguard-go/tun"
+	"tailscale.com/net/tstun"
+	"tailscale.com/types/logger"
+	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/netstack"
+	"tailscale.com/wgengine/router"
+)
+
+func TestIsNetstack(t *testing.T) {
+	e, err := wgengine.NewUserspaceEngine(t.Logf, wgengine.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer e.Close()
+	if !wgengine.IsNetstack(e) {
+		t.Errorf("IsNetstack = false; want true")
+	}
+}
+
+func TestIsNetstackRouter(t *testing.T) {
+	tests := []struct {
+		name string
+		conf wgengine.Config
+		want bool
+	}{
+		{
+			name: "no_netstack",
+			conf: wgengine.Config{
+				Tun:    newFakeOSTUN(),
+				Router: newFakeOSRouter(),
+			},
+			want: false,
+		},
+		{
+			name: "netstack",
+			conf: wgengine.Config{},
+			want: true,
+		},
+		{
+			name: "hybrid_netstack",
+			conf: wgengine.Config{
+				Tun:    newFakeOSTUN(),
+				Router: netstack.NewSubnetRouterWrapper(newFakeOSRouter()),
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			e, err := wgengine.NewUserspaceEngine(logger.Discard, tt.conf)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer e.Close()
+			if got := wgengine.IsNetstackRouter(e); got != tt.want {
+				t.Errorf("IsNetstackRouter = %v; want %v", got, tt.want)
+			}
+
+			if got := wgengine.IsNetstackRouter(wgengine.NewWatchdog(e)); got != tt.want {
+				t.Errorf("IsNetstackRouter(watchdog-wrapped) = %v; want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func newFakeOSRouter() router.Router {
+	return someRandoOSRouter{router.NewFake(logger.Discard)}
+}
+
+type someRandoOSRouter struct {
+	router.Router
+}
+
+func newFakeOSTUN() tun.Device {
+	return someRandoOSTUN{tstun.NewFake()}
+}
+
+type someRandoOSTUN struct {
+	tun.Device
+}
+
+// Name returns something that is not FakeTUN.
+func (t someRandoOSTUN) Name() (string, error) { return "some_os_tun0", nil }

wgengine/userspace_test.go

@@ -187,14 +187,3 @@ func BenchmarkGenLocalAddrFunc(b *testing.B) {
 	})
 	b.Logf("x = %v", x)
 }
-
-func TestIsNetstack(t *testing.T) {
-	e, err := NewUserspaceEngine(t.Logf, Config{})
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer e.Close()
-	if !IsNetstack(e) {
-		t.Errorf("IsNetstack = false; want true")
-	}
-}