wgengine/router: start using netlink instead of 'ip' on Linux

Converts up, down, add/del addresses, add/del routes.

Not yet done: rules.

Updates #391

Change-Id: I02554ca07046d18f838e04a626ba99bbd35266fb
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

cmd/tailscaled/depaware.txt

@@ -96,6 +96,9 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/u-root/uio/rand                                   from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/u-root/uio/ubinary                                from github.com/u-root/uio/uio
    L    github.com/u-root/uio/uio                                    from github.com/insomniacslk/dhcp/dhcpv4+
+   L 💣 github.com/vishvananda/netlink                               from tailscale.com/wgengine/router
+   L 💣 github.com/vishvananda/netlink/nl                            from github.com/vishvananda/netlink
+   L    github.com/vishvananda/netns                                 from github.com/vishvananda/netlink+
      💣 go4.org/intern                                               from inet.af/netaddr
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
         go4.org/unsafe/assume-no-moving-gc                           from go4.org/intern

go.mod

@@ -42,6 +42,7 @@ require (
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	github.com/ulikunitz/xz v0.5.10 // indirect
+	github.com/vishvananda/netlink v1.1.0
 	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174
 	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
 	golang.org/x/net v0.0.0-20211020060615-d418f374d309
@@ -191,6 +192,7 @@ require (
 	github.com/ultraware/funlen v0.0.3 // indirect
 	github.com/ultraware/whitespace v0.0.4 // indirect
 	github.com/uudashr/gocognit v1.0.1 // indirect
+	github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect
 	github.com/xanzy/ssh-agent v0.3.0 // indirect
 	go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect
 	go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect

go.sum

@@ -681,6 +681,10 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
 github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
 github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD5KW4lOuSdPKzY=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
+github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
 github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
 github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
@@ -832,6 +836,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606122018-79a91cf218c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

wgengine/router/router_linux.go

@@ -13,10 +13,13 @@ import (
 	"os/exec"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/coreos/go-iptables/iptables"
 	"github.com/go-multierror/multierror"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
 	"golang.org/x/time/rate"
 	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
@@ -81,7 +84,8 @@ const (
 	// implementation believes that table numbers are 8-bit integers, so
 	// for maximum compatibility we have to stay in the 0-255 range even
 	// though linux itself supports larger numbers.
-	tailscaleRouteTable = "52"
+	tailscaleRouteTable    = "52"
+	tailscaleRouteTableNum = 52
 )
 
 // netfilterRunner abstracts helpers to run netfilter commands. It
@@ -196,6 +200,17 @@ func useAmbientCaps() bool {
 	return v >= 7
 }
 
+// useIPCommand reports whether r should use the "ip" command (or its
+// fake commandRunner for tests) instead of netlink.
+func (r *linuxRouter) useIPCommand() bool {
+	// In the future we might need to fall back to using the "ip"
+	// command if, say, netlink is blocked somewhere but the ip
+	// command is allowed to use netlink. For now we only use the ip
+	// command runner in tests.
+	_, ok := r.cmd.(osCommandRunner)
+	return !ok
+}
+
 // onIPRuleDeleted is the callback from the link monitor for when an IP policy
 // rule is deleted. See Issue 1591.
 //
@@ -449,8 +464,18 @@ func (r *linuxRouter) addAddress(addr netaddr.IPPrefix) error {
 	if !r.v6Available && addr.IP().Is6() {
 		return nil
 	}
-	if err := r.cmd.run("ip", "addr", "add", addr.String(), "dev", r.tunname); err != nil {
+	if r.useIPCommand() {
-		return fmt.Errorf("adding address %q to tunnel interface: %w", addr, err)
+		if err := r.cmd.run("ip", "addr", "add", addr.String(), "dev", r.tunname); err != nil {
+			return fmt.Errorf("adding address %q to tunnel interface: %w", addr, err)
+		}
+	} else {
+		link, err := r.link()
+		if err != nil {
+			return fmt.Errorf("adding address %v, %w", addr, err)
+		}
+		if err := netlink.AddrReplace(link, nlAddrOfPrefix(addr)); err != nil {
+			return fmt.Errorf("adding address %v from tunnel interface: %w", addr, err)
+		}
 	}
 	if err := r.addLoopbackRule(addr.IP()); err != nil {
 		return err
@@ -468,8 +493,18 @@ func (r *linuxRouter) delAddress(addr netaddr.IPPrefix) error {
 	if err := r.delLoopbackRule(addr.IP()); err != nil {
 		return err
 	}
-	if err := r.cmd.run("ip", "addr", "del", addr.String(), "dev", r.tunname); err != nil {
+	if r.useIPCommand() {
-		return fmt.Errorf("deleting address %q from tunnel interface: %w", addr, err)
+		if err := r.cmd.run("ip", "addr", "del", addr.String(), "dev", r.tunname); err != nil {
+			return fmt.Errorf("deleting address %q from tunnel interface: %w", addr, err)
+		}
+	} else {
+		link, err := r.link()
+		if err != nil {
+			return fmt.Errorf("deleting address %v, %w", addr, err)
+		}
+		if err := netlink.AddrDel(link, nlAddrOfPrefix(addr)); err != nil {
+			return fmt.Errorf("deleting address %v from tunnel interface: %w", addr, err)
+		}
 	}
 	return nil
 }
@@ -522,7 +557,21 @@ func (r *linuxRouter) delLoopbackRule(addr netaddr.IP) error {
 // interface. Fails if the route already exists, or if adding the
 // route fails.
 func (r *linuxRouter) addRoute(cidr netaddr.IPPrefix) error {
-	return r.addRouteDef([]string{normalizeCIDR(cidr), "dev", r.tunname}, cidr)
+	if !r.v6Available && cidr.IP().Is6() {
+		return nil
+	}
+	if r.useIPCommand() {
+		return r.addRouteDef([]string{normalizeCIDR(cidr), "dev", r.tunname}, cidr)
+	}
+	linkIndex, err := r.linkIndex()
+	if err != nil {
+		return err
+	}
+	return netlink.RouteReplace(&netlink.Route{
+		LinkIndex: linkIndex,
+		Dst:       cidr.Masked().IPNet(),
+		Table:     r.routeTable(),
+	})
 }
 
 // addThrowRoute adds a throw route for the provided cidr.
@@ -533,7 +582,21 @@ func (r *linuxRouter) addThrowRoute(cidr netaddr.IPPrefix) error {
 	if !r.ipRuleAvailable {
 		return nil
 	}
-	return r.addRouteDef([]string{"throw", normalizeCIDR(cidr)}, cidr)
+	if !r.v6Available && cidr.IP().Is6() {
+		return nil
+	}
+	if r.useIPCommand() {
+		return r.addRouteDef([]string{"throw", normalizeCIDR(cidr)}, cidr)
+	}
+	err := netlink.RouteReplace(&netlink.Route{
+		Dst:   cidr.Masked().IPNet(),
+		Table: tailscaleRouteTableNum,
+		Type:  unix.RTN_THROW,
+	})
+	if err != nil {
+		r.logf("THROW ERROR adding %v: %#v", cidr, err)
+	}
+	return err
 }
 
 func (r *linuxRouter) addRouteDef(routeDef []string, cidr netaddr.IPPrefix) error {
@@ -549,11 +612,11 @@ func (r *linuxRouter) addRouteDef(routeDef []string, cidr netaddr.IPPrefix) erro
 		return nil
 	}
 
-	// TODO(bradfitz): remove this ugly hack to detect failure to
+	// This is an ugly hack to detect failure to add a route that
-	// add a route that already exists (as happens in when we're
+	// already exists (as happens in when we're racing to add
-	// racing to add kernel-maintained routes when enabling exit
+	// kernel-maintained routes when enabling exit nodes w/o Local
-	// nodes w/o Local LAN access, Issue 3060) and use netlink
+	// LAN access, Issue 3060). Fortunately in the common case we
-	// directly instead (Issue 391).
+	// use netlink directly instead and don't exercise this code.
 	if errCode(err) == 2 && strings.Contains(err.Error(), "RTNETLINK answers: File exists") {
 		r.logf("ignoring route add of %v; already exists", cidr)
 		return nil
@@ -561,11 +624,32 @@ func (r *linuxRouter) addRouteDef(routeDef []string, cidr netaddr.IPPrefix) erro
 	return err
 }
 
+var errESRCH error = syscall.ESRCH
+
 // delRoute removes the route for cidr pointing to the tunnel
 // interface. Fails if the route doesn't exist, or if removing the
 // route fails.
 func (r *linuxRouter) delRoute(cidr netaddr.IPPrefix) error {
-	return r.delRouteDef([]string{normalizeCIDR(cidr), "dev", r.tunname}, cidr)
+	if !r.v6Available && cidr.IP().Is6() {
+		return nil
+	}
+	if r.useIPCommand() {
+		return r.delRouteDef([]string{normalizeCIDR(cidr), "dev", r.tunname}, cidr)
+	}
+	linkIndex, err := r.linkIndex()
+	if err != nil {
+		return err
+	}
+	err = netlink.RouteDel(&netlink.Route{
+		LinkIndex: linkIndex,
+		Dst:       cidr.Masked().IPNet(),
+		Table:     r.routeTable(),
+	})
+	if errors.Is(err, errESRCH) {
+		// Didn't exist to begin with.
+		return nil
+	}
+	return err
 }
 
 // delThrowRoute removes the throw route for the cidr. Fails if the route
@@ -574,7 +658,22 @@ func (r *linuxRouter) delThrowRoute(cidr netaddr.IPPrefix) error {
 	if !r.ipRuleAvailable {
 		return nil
 	}
-	return r.delRouteDef([]string{"throw", normalizeCIDR(cidr)}, cidr)
+	if !r.v6Available && cidr.IP().Is6() {
+		return nil
+	}
+	if r.useIPCommand() {
+		return r.delRouteDef([]string{"throw", normalizeCIDR(cidr)}, cidr)
+	}
+	err := netlink.RouteDel(&netlink.Route{
+		Dst:   cidr.Masked().IPNet(),
+		Table: r.routeTable(),
+		Type:  unix.RTN_THROW,
+	})
+	if errors.Is(err, errESRCH) {
+		// Didn't exist to begin with.
+		return nil
+	}
+	return err
 }
 
 func (r *linuxRouter) delRouteDef(routeDef []string, cidr netaddr.IPPrefix) error {
@@ -619,14 +718,54 @@ func (r *linuxRouter) hasRoute(routeDef []string, cidr netaddr.IPPrefix) (bool,
 	return len(out) > 0, nil
 }
 
+func (r *linuxRouter) link() (netlink.Link, error) {
+	link, err := netlink.LinkByName(r.tunname)
+	if err != nil {
+		return nil, fmt.Errorf("failed to look up link %q: %w", r.tunname, err)
+	}
+	return link, nil
+}
+
+func (r *linuxRouter) linkIndex() (int, error) {
+	// TODO(bradfitz): cache this? It doesn't change often, and on start-up
+	// hundreds of addRoute calls to add /32s can happen quickly.
+	link, err := r.link()
+	if err != nil {
+		return 0, err
+	}
+	return link.Attrs().Index, nil
+}
+
+// routeTable returns the route table to use.
+func (r *linuxRouter) routeTable() int {
+	if r.ipRuleAvailable {
+		return tailscaleRouteTableNum
+	}
+	return 0
+}
+
 // upInterface brings up the tunnel interface.
 func (r *linuxRouter) upInterface() error {
-	return r.cmd.run("ip", "link", "set", "dev", r.tunname, "up")
+	if r.useIPCommand() {
+		return r.cmd.run("ip", "link", "set", "dev", r.tunname, "up")
+	}
+	link, err := r.link()
+	if err != nil {
+		return fmt.Errorf("bringing interface up, %w", err)
+	}
+	return netlink.LinkSetUp(link)
 }
 
 // downInterface sets the tunnel interface administratively down.
 func (r *linuxRouter) downInterface() error {
-	return r.cmd.run("ip", "link", "set", "dev", r.tunname, "down")
+	if r.useIPCommand() {
+		return r.cmd.run("ip", "link", "set", "dev", r.tunname, "down")
+	}
+	link, err := r.link()
+	if err != nil {
+		return fmt.Errorf("bringing interface down, %w", err)
+	}
+	return netlink.LinkSetDown(link)
 }
 
 func (r *linuxRouter) iprouteFamilies() []string {
@@ -1301,3 +1440,9 @@ func checkIPRuleSupportsV6() error {
 	exec.Command("ip", del...).Run()
 	return nil
 }
+
+func nlAddrOfPrefix(p netaddr.IPPrefix) *netlink.Addr {
+	return &netlink.Addr{
+		IPNet: p.IPNet(),
+	}
+}

wgengine/router/router_linux_test.go

@@ -15,6 +15,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/vishvananda/netlink"
 	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 	"tailscale.com/tstest"
@@ -708,3 +709,27 @@ func TestDelRouteIdempotent(t *testing.T) {
 		t.Logf("Log output:\n%s", out)
 	}
 }
+
+func TestDebugListLinks(t *testing.T) {
+	links, err := netlink.LinkList()
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, ln := range links {
+		t.Logf("Link: %+v", ln)
+	}
+}
+
+func TestDebugListRoutes(t *testing.T) {
+	// We need to pass a non-nil route to RouteListFiltered, along
+	// with the netlink.RT_FILTER_TABLE bit set in the filter
+	// mask, otherwise it ignores non-main routes.
+	filter := &netlink.Route{}
+	routes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, filter, netlink.RT_FILTER_TABLE)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range routes {
+		t.Logf("Route: %+v", r)
+	}
+}