wgengine/router: use netlink for ip rules on Linux

Using temporary netlink fork in github.com/tailscale/netlink until we
get the necessary changes upstream in either vishvananda/netlink
or jsimonetti/rtnetlink.

Updates #391

Change-Id: I6e1de96cf0750ccba53dabff670aca0c56dffb7c
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

cmd/tailscaled/depaware.txt

@@ -92,13 +92,13 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         github.com/tailscale/goupnp/scpd                             from github.com/tailscale/goupnp
         github.com/tailscale/goupnp/soap                             from github.com/tailscale/goupnp+
         github.com/tailscale/goupnp/ssdp                             from github.com/tailscale/goupnp
+   L 💣 github.com/tailscale/netlink                                 from tailscale.com/wgengine/router
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
    L    github.com/u-root/uio/rand                                   from github.com/insomniacslk/dhcp/dhcpv4
    L    github.com/u-root/uio/ubinary                                from github.com/u-root/uio/uio
    L    github.com/u-root/uio/uio                                    from github.com/insomniacslk/dhcp/dhcpv4+
-   L 💣 github.com/vishvananda/netlink                               from tailscale.com/wgengine/router
+   L 💣 github.com/vishvananda/netlink/nl                            from github.com/tailscale/netlink
-   L 💣 github.com/vishvananda/netlink/nl                            from github.com/vishvananda/netlink
+   L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
-   L    github.com/vishvananda/netns                                 from github.com/vishvananda/netlink+
      💣 go4.org/intern                                               from inet.af/netaddr
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
         go4.org/unsafe/assume-no-moving-gc                           from go4.org/intern

go.mod

@@ -39,10 +39,11 @@ require (
 	github.com/tailscale/goexpect v0.0.0-20210902213824-6e8c725cea41
 	github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05
 	github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2
+	github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	github.com/ulikunitz/xz v0.5.10 // indirect
-	github.com/vishvananda/netlink v1.1.0
+	github.com/vishvananda/netlink v1.1.1-0.20211101163509-b10eb8fe5cf6
 	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174
 	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
 	golang.org/x/net v0.0.0-20211020060615-d418f374d309
@@ -192,7 +193,7 @@ require (
 	github.com/ultraware/funlen v0.0.3 // indirect
 	github.com/ultraware/whitespace v0.0.4 // indirect
 	github.com/uudashr/gocognit v1.0.1 // indirect
-	github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect
+	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect
 	github.com/xanzy/ssh-agent v0.3.0 // indirect
 	go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect
 	go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect

go.sum

@@ -643,6 +643,8 @@ github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 h1:4chzWmimtJPx
 github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
 github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2 h1:reREUgl2FG+o7YCsrZB8XLjnuKv5hEIWtnOdAbRAXZI=
 github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2/go.mod h1:STqf+YV0ADdzk4ejtXFsGqDpATP9JoL0OB+hiFQbkdE=
+github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85 h1:zrsUcqrG2uQSPhaUPjUQwozcRdDdSxxqhNgNZ3drZFk=
+github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85/go.mod h1:NzVQi3Mleb+qzq8VmcWpSkcSYxXIg0DkI6XDzpVkhJ0=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
@@ -681,10 +683,10 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
 github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
 github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD5KW4lOuSdPKzY=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
-github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
+github.com/vishvananda/netlink v1.1.1-0.20211101163509-b10eb8fe5cf6 h1:167a2omrzz+nN9Of6lN/0yOB9itzw+IOioRThNZ30jA=
-github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netlink v1.1.1-0.20211101163509-b10eb8fe5cf6/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
-github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
+github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns=
-github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
 github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
 github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
@@ -836,7 +838,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606122018-79a91cf218c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -847,11 +848,13 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

wgengine/router/router_linux.go

@@ -18,7 +18,7 @@ import (
 
 	"github.com/coreos/go-iptables/iptables"
 	"github.com/go-multierror/multierror"
-	"github.com/vishvananda/netlink"
+	"github.com/tailscale/netlink"
 	"golang.org/x/sys/unix"
 	"golang.org/x/time/rate"
 	"golang.zx2c4.com/wireguard/tun"
@@ -604,7 +604,11 @@ func (r *linuxRouter) addRouteDef(routeDef []string, cidr netaddr.IPPrefix) erro
 	return err
 }
 
-var errESRCH error = syscall.ESRCH
+var (
+	errESRCH  error = syscall.ESRCH
+	errENOENT error = syscall.ENOENT
+	errEEXIST error = syscall.EEXIST
+)
 
 // delRoute removes the route for cidr pointing to the tunnel
 // interface. Fails if the route doesn't exist, or if removing the
@@ -766,6 +770,16 @@ func (f addrFamily) dashArg() string {
 	panic("illegal")
 }
 
+func (f addrFamily) netlinkInt() int {
+	switch f {
+	case 4:
+		return netlink.FAMILY_V4
+	case 6:
+		return netlink.FAMILY_V6
+	}
+	panic("illegal")
+}
+
 func (r *linuxRouter) addrFamilies() []addrFamily {
 	if r.v6Available {
 		return []addrFamily{v4, v6}
@@ -878,7 +892,7 @@ var ipRules = []netlink.Rule{
 	{
 		Priority: 5250,
 		Mark:     tailscaleBypassMarkNum,
-		Table:    0, // unreachable
+		Type:     unix.RTN_UNREACHABLE,
 	},
 	// If we get to this point, capture all packets and send them
 	// through to the tailscale route table. For apps other than us
@@ -898,7 +912,34 @@ func (r *linuxRouter) justAddIPRules() error {
 	if !r.ipRuleAvailable {
 		return nil
 	}
+	if r.useIPCommand() {
+		return r.addIPRulesWithIPCommand()
+	}
+	var errAcc error
+	for _, family := range r.addrFamilies() {
+		for _, ru := range ipRules {
+			// Note: r is a value type here; safe to mutate it.
+			ru.Family = family.netlinkInt()
+			ru.Mask = -1
+			ru.Goto = -1
+			ru.SuppressIfgroup = -1
+			ru.SuppressPrefixlen = -1
+			ru.Flow = -1
+
+			err := netlink.RuleAdd(&ru)
+			if errors.Is(err, errEEXIST) {
+				// Ignore dups.
+				continue
+			}
+			if err != nil && errAcc == nil {
+				errAcc = err
+			}
+		}
+	}
+	return errAcc
+}
 
+func (r *linuxRouter) addIPRulesWithIPCommand() error {
 	rg := newRunGroup(nil, r.cmd)
 
 	for _, family := range r.addrFamilies() {
@@ -913,7 +954,8 @@ func (r *linuxRouter) justAddIPRules() error {
 			}
 			if r.Table != 0 {
 				args = append(args, "table", mustRouteTable(r.Table).ipCmdArg())
-			} else {
+			}
+			if r.Type == unix.RTN_UNREACHABLE {
 				args = append(args, "type", "unreachable")
 			}
 			rg.Run(args...)
@@ -940,7 +982,39 @@ func (r *linuxRouter) delIPRules() error {
 	if !r.ipRuleAvailable {
 		return nil
 	}
+	if r.useIPCommand() {
+		return r.delIPRulesWithIPCommand()
+	}
+	var errAcc error
+	for _, family := range r.addrFamilies() {
+		for _, ru := range ipRules {
+			// Note: r is a value type here; safe to mutate it.
+			// When deleting rules, we want to be a bit specific (mention which
+			// table we were routing to) but not *too* specific (fwmarks, etc).
+			// That leaves us some flexibility to change these values in later
+			// versions without having ongoing hacks for every possible
+			// combination.
+			ru.Family = family.netlinkInt()
+			ru.Mark = -1
+			ru.Mask = -1
+			ru.Goto = -1
+			ru.SuppressIfgroup = -1
+			ru.SuppressPrefixlen = -1
+
+			err := netlink.RuleDel(&ru)
+			if errors.Is(err, errENOENT) {
+				// Didn't exist to begin with.
+				continue
+			}
+			if err != nil && errAcc == nil {
+				errAcc = err
+			}
+		}
+	}
+	return errAcc
+}
 
+func (r *linuxRouter) delIPRulesWithIPCommand() error {
 	// Error codes: 'ip rule' returns error code 2 if the rule is a
 	// duplicate (add) or not found (del). It returns a different code
 	// for syntax errors. This is also true of busybox.

wgengine/router/router_linux_test.go

@@ -654,62 +654,110 @@ func createTestTUN(t *testing.T) tun.Device {
 	return tun
 }
 
-func TestDelRouteIdempotent(t *testing.T) {
+type linuxTest struct {
+	tun       tun.Device
+	mon       *monitor.Mon
+	r         *linuxRouter
+	logOutput tstest.MemLogger
+}
+
+func (lt *linuxTest) Close() error {
+	if lt.tun != nil {
+		lt.tun.Close()
+	}
+	if lt.mon != nil {
+		lt.mon.Close()
+	}
+	return nil
+}
+
+func newLinuxRootTest(t *testing.T) *linuxTest {
 	if os.Getuid() != 0 {
 		t.Skip("test requires root")
 	}
-	tun := createTestTUN(t)
-	defer tun.Close()
 
-	var logOutput tstest.MemLogger
+	lt := new(linuxTest)
-	logf := logOutput.Logf
+	lt.tun = createTestTUN(t)
+
+	logf := lt.logOutput.Logf
 
 	mon, err := monitor.New(logger.Discard)
 	if err != nil {
+		lt.Close()
 		t.Fatal(err)
 	}
 	mon.Start()
-	defer mon.Close()
+	lt.mon = mon
 
-	r, err := newUserspaceRouter(logf, tun, mon)
+	r, err := newUserspaceRouter(logf, lt.tun, mon)
 	if err != nil {
+		lt.Close()
 		t.Fatal(err)
 	}
 	lr := r.(*linuxRouter)
 	if err := lr.upInterface(); err != nil {
+		lt.Close()
 		t.Fatal(err)
 	}
+	lt.r = lr
+	return lt
+}
+
+func TestDelRouteIdempotent(t *testing.T) {
+	lt := newLinuxRootTest(t)
+	defer lt.Close()
+
 	for _, s := range []string{
 		"192.0.2.0/24",  // RFC 5737
 		"2001:DB8::/32", // RFC 3849
 	} {
 		cidr := netaddr.MustParseIPPrefix(s)
-		if err := lr.addRoute(cidr); err != nil {
+		if err := lt.r.addRoute(cidr); err != nil {
-			t.Fatal(err)
+			t.Error(err)
+			continue
 		}
 		for i := 0; i < 2; i++ {
-			if err := lr.delRoute(cidr); err != nil {
+			if err := lt.r.delRoute(cidr); err != nil {
-				t.Fatalf("delRoute(i=%d): %v", i, err)
+				t.Errorf("delRoute(i=%d): %v", i, err)
 			}
 		}
 	}
 
-	wantSubs := map[string]int{
-		"warning: tried to delete route 192.0.2.0/24 but it was already gone; ignoring error":  1,
-		"warning: tried to delete route 2001:db8::/32 but it was already gone; ignoring error": 1,
-	}
-	out := logOutput.String()
-	for sub, want := range wantSubs {
-		got := strings.Count(out, sub)
-		if got != want {
-			t.Errorf("log output substring %q occurred %d time; want %d", sub, got, want)
-		}
-	}
 	if t.Failed() {
+		out := lt.logOutput.String()
 		t.Logf("Log output:\n%s", out)
 	}
 }
 
+func TestAddRemoveRules(t *testing.T) {
+	lt := newLinuxRootTest(t)
+	defer lt.Close()
+	r := lt.r
+
+	step := func(name string, f func() error) {
+		t.Logf("Doing %v ...", name)
+		if err := f(); err != nil {
+			t.Fatalf("%s: %v", name, err)
+		}
+		rules, err := netlink.RuleList(netlink.FAMILY_ALL)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, r := range rules {
+			if r.Priority >= 5000 && r.Priority <= 5999 {
+				t.Logf("Rule: %+v", r)
+			}
+		}
+
+	}
+
+	step("init_del_and_add", r.addIPRules)
+	step("dup_add", r.justAddIPRules)
+	step("del", r.delIPRules)
+	step("dup_del", r.delIPRules)
+
+}
+
 func TestDebugListLinks(t *testing.T) {
 	links, err := netlink.LinkList()
 	if err != nil {