From ff1954cfd9be4685f3295ebc61a0fedcccd8137e Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 28 Oct 2021 15:22:03 -0700 Subject: [PATCH] wgengine/router: use netlink for ip rules on Linux Using temporary netlink fork in github.com/tailscale/netlink until we get the necessary changes upstream in either vishvananda/netlink or jsimonetti/rtnetlink. Updates #391 Change-Id: I6e1de96cf0750ccba53dabff670aca0c56dffb7c Signed-off-by: Brad Fitzpatrick --- cmd/tailscaled/depaware.txt | 6 +- go.mod | 5 +- go.sum | 13 ++-- wgengine/router/router_linux.go | 82 +++++++++++++++++++++++-- wgengine/router/router_linux_test.go | 92 +++++++++++++++++++++------- 5 files changed, 162 insertions(+), 36 deletions(-) diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 048d6ea7a..b032a9dc3 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -92,13 +92,13 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de github.com/tailscale/goupnp/scpd from github.com/tailscale/goupnp github.com/tailscale/goupnp/soap from github.com/tailscale/goupnp+ github.com/tailscale/goupnp/ssdp from github.com/tailscale/goupnp + L 💣 github.com/tailscale/netlink from tailscale.com/wgengine/router github.com/tcnksm/go-httpstat from tailscale.com/net/netcheck L github.com/u-root/uio/rand from github.com/insomniacslk/dhcp/dhcpv4 L github.com/u-root/uio/ubinary from github.com/u-root/uio/uio L github.com/u-root/uio/uio from github.com/insomniacslk/dhcp/dhcpv4+ - L 💣 github.com/vishvananda/netlink from tailscale.com/wgengine/router - L 💣 github.com/vishvananda/netlink/nl from github.com/vishvananda/netlink - L github.com/vishvananda/netns from github.com/vishvananda/netlink+ + L 💣 github.com/vishvananda/netlink/nl from github.com/tailscale/netlink + L github.com/vishvananda/netns from github.com/tailscale/netlink+ 💣 go4.org/intern from inet.af/netaddr 💣 go4.org/mem from tailscale.com/client/tailscale+ go4.org/unsafe/assume-no-moving-gc from go4.org/intern diff --git a/go.mod b/go.mod index 09f444a17..857cc7122 100644 --- a/go.mod +++ b/go.mod @@ -39,10 +39,11 @@ require ( github.com/tailscale/goexpect v0.0.0-20210902213824-6e8c725cea41 github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2 + github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85 github.com/tcnksm/go-httpstat v0.2.0 github.com/toqueteos/webbrowser v1.2.0 github.com/ulikunitz/xz v0.5.10 // indirect - github.com/vishvananda/netlink v1.1.0 + github.com/vishvananda/netlink v1.1.1-0.20211101163509-b10eb8fe5cf6 go4.org/mem v0.0.0-20201119185036-c04c5a6ff174 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 golang.org/x/net v0.0.0-20211020060615-d418f374d309 @@ -192,7 +193,7 @@ require ( github.com/ultraware/funlen v0.0.3 // indirect github.com/ultraware/whitespace v0.0.4 // indirect github.com/uudashr/gocognit v1.0.1 // indirect - github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect + github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect github.com/xanzy/ssh-agent v0.3.0 // indirect go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect diff --git a/go.sum b/go.sum index 35bff7064..dc3f7f602 100644 --- a/go.sum +++ b/go.sum @@ -643,6 +643,8 @@ github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 h1:4chzWmimtJPx github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8= github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2 h1:reREUgl2FG+o7YCsrZB8XLjnuKv5hEIWtnOdAbRAXZI= github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2/go.mod h1:STqf+YV0ADdzk4ejtXFsGqDpATP9JoL0OB+hiFQbkdE= +github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85 h1:zrsUcqrG2uQSPhaUPjUQwozcRdDdSxxqhNgNZ3drZFk= +github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85/go.mod h1:NzVQi3Mleb+qzq8VmcWpSkcSYxXIg0DkI6XDzpVkhJ0= github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0= github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8= github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= @@ -681,10 +683,10 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA= github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD5KW4lOuSdPKzY= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/vishvananda/netlink v1.1.1-0.20211101163509-b10eb8fe5cf6 h1:167a2omrzz+nN9Of6lN/0yOB9itzw+IOioRThNZ30jA= +github.com/vishvananda/netlink v1.1.1-0.20211101163509-b10eb8fe5cf6/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= @@ -836,7 +838,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606122018-79a91cf218c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -847,11 +848,13 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/wgengine/router/router_linux.go b/wgengine/router/router_linux.go index c30b6fafa..7fb42c9cf 100644 --- a/wgengine/router/router_linux.go +++ b/wgengine/router/router_linux.go @@ -18,7 +18,7 @@ import ( "github.com/coreos/go-iptables/iptables" "github.com/go-multierror/multierror" - "github.com/vishvananda/netlink" + "github.com/tailscale/netlink" "golang.org/x/sys/unix" "golang.org/x/time/rate" "golang.zx2c4.com/wireguard/tun" @@ -604,7 +604,11 @@ func (r *linuxRouter) addRouteDef(routeDef []string, cidr netaddr.IPPrefix) erro return err } -var errESRCH error = syscall.ESRCH +var ( + errESRCH error = syscall.ESRCH + errENOENT error = syscall.ENOENT + errEEXIST error = syscall.EEXIST +) // delRoute removes the route for cidr pointing to the tunnel // interface. Fails if the route doesn't exist, or if removing the @@ -766,6 +770,16 @@ func (f addrFamily) dashArg() string { panic("illegal") } +func (f addrFamily) netlinkInt() int { + switch f { + case 4: + return netlink.FAMILY_V4 + case 6: + return netlink.FAMILY_V6 + } + panic("illegal") +} + func (r *linuxRouter) addrFamilies() []addrFamily { if r.v6Available { return []addrFamily{v4, v6} @@ -878,7 +892,7 @@ var ipRules = []netlink.Rule{ { Priority: 5250, Mark: tailscaleBypassMarkNum, - Table: 0, // unreachable + Type: unix.RTN_UNREACHABLE, }, // If we get to this point, capture all packets and send them // through to the tailscale route table. For apps other than us @@ -898,7 +912,34 @@ func (r *linuxRouter) justAddIPRules() error { if !r.ipRuleAvailable { return nil } + if r.useIPCommand() { + return r.addIPRulesWithIPCommand() + } + var errAcc error + for _, family := range r.addrFamilies() { + for _, ru := range ipRules { + // Note: r is a value type here; safe to mutate it. + ru.Family = family.netlinkInt() + ru.Mask = -1 + ru.Goto = -1 + ru.SuppressIfgroup = -1 + ru.SuppressPrefixlen = -1 + ru.Flow = -1 + + err := netlink.RuleAdd(&ru) + if errors.Is(err, errEEXIST) { + // Ignore dups. + continue + } + if err != nil && errAcc == nil { + errAcc = err + } + } + } + return errAcc +} +func (r *linuxRouter) addIPRulesWithIPCommand() error { rg := newRunGroup(nil, r.cmd) for _, family := range r.addrFamilies() { @@ -913,7 +954,8 @@ func (r *linuxRouter) justAddIPRules() error { } if r.Table != 0 { args = append(args, "table", mustRouteTable(r.Table).ipCmdArg()) - } else { + } + if r.Type == unix.RTN_UNREACHABLE { args = append(args, "type", "unreachable") } rg.Run(args...) @@ -940,7 +982,39 @@ func (r *linuxRouter) delIPRules() error { if !r.ipRuleAvailable { return nil } + if r.useIPCommand() { + return r.delIPRulesWithIPCommand() + } + var errAcc error + for _, family := range r.addrFamilies() { + for _, ru := range ipRules { + // Note: r is a value type here; safe to mutate it. + // When deleting rules, we want to be a bit specific (mention which + // table we were routing to) but not *too* specific (fwmarks, etc). + // That leaves us some flexibility to change these values in later + // versions without having ongoing hacks for every possible + // combination. + ru.Family = family.netlinkInt() + ru.Mark = -1 + ru.Mask = -1 + ru.Goto = -1 + ru.SuppressIfgroup = -1 + ru.SuppressPrefixlen = -1 + + err := netlink.RuleDel(&ru) + if errors.Is(err, errENOENT) { + // Didn't exist to begin with. + continue + } + if err != nil && errAcc == nil { + errAcc = err + } + } + } + return errAcc +} +func (r *linuxRouter) delIPRulesWithIPCommand() error { // Error codes: 'ip rule' returns error code 2 if the rule is a // duplicate (add) or not found (del). It returns a different code // for syntax errors. This is also true of busybox. diff --git a/wgengine/router/router_linux_test.go b/wgengine/router/router_linux_test.go index 0b12e4740..7d59051a9 100644 --- a/wgengine/router/router_linux_test.go +++ b/wgengine/router/router_linux_test.go @@ -654,62 +654,110 @@ func createTestTUN(t *testing.T) tun.Device { return tun } -func TestDelRouteIdempotent(t *testing.T) { +type linuxTest struct { + tun tun.Device + mon *monitor.Mon + r *linuxRouter + logOutput tstest.MemLogger +} + +func (lt *linuxTest) Close() error { + if lt.tun != nil { + lt.tun.Close() + } + if lt.mon != nil { + lt.mon.Close() + } + return nil +} + +func newLinuxRootTest(t *testing.T) *linuxTest { if os.Getuid() != 0 { t.Skip("test requires root") } - tun := createTestTUN(t) - defer tun.Close() - var logOutput tstest.MemLogger - logf := logOutput.Logf + lt := new(linuxTest) + lt.tun = createTestTUN(t) + + logf := lt.logOutput.Logf mon, err := monitor.New(logger.Discard) if err != nil { + lt.Close() t.Fatal(err) } mon.Start() - defer mon.Close() + lt.mon = mon - r, err := newUserspaceRouter(logf, tun, mon) + r, err := newUserspaceRouter(logf, lt.tun, mon) if err != nil { + lt.Close() t.Fatal(err) } lr := r.(*linuxRouter) if err := lr.upInterface(); err != nil { + lt.Close() t.Fatal(err) } + lt.r = lr + return lt +} + +func TestDelRouteIdempotent(t *testing.T) { + lt := newLinuxRootTest(t) + defer lt.Close() + for _, s := range []string{ "192.0.2.0/24", // RFC 5737 "2001:DB8::/32", // RFC 3849 } { cidr := netaddr.MustParseIPPrefix(s) - if err := lr.addRoute(cidr); err != nil { - t.Fatal(err) + if err := lt.r.addRoute(cidr); err != nil { + t.Error(err) + continue } for i := 0; i < 2; i++ { - if err := lr.delRoute(cidr); err != nil { - t.Fatalf("delRoute(i=%d): %v", i, err) + if err := lt.r.delRoute(cidr); err != nil { + t.Errorf("delRoute(i=%d): %v", i, err) } } } - wantSubs := map[string]int{ - "warning: tried to delete route 192.0.2.0/24 but it was already gone; ignoring error": 1, - "warning: tried to delete route 2001:db8::/32 but it was already gone; ignoring error": 1, - } - out := logOutput.String() - for sub, want := range wantSubs { - got := strings.Count(out, sub) - if got != want { - t.Errorf("log output substring %q occurred %d time; want %d", sub, got, want) - } - } if t.Failed() { + out := lt.logOutput.String() t.Logf("Log output:\n%s", out) } } +func TestAddRemoveRules(t *testing.T) { + lt := newLinuxRootTest(t) + defer lt.Close() + r := lt.r + + step := func(name string, f func() error) { + t.Logf("Doing %v ...", name) + if err := f(); err != nil { + t.Fatalf("%s: %v", name, err) + } + rules, err := netlink.RuleList(netlink.FAMILY_ALL) + if err != nil { + t.Fatal(err) + } + for _, r := range rules { + if r.Priority >= 5000 && r.Priority <= 5999 { + t.Logf("Rule: %+v", r) + } + } + + } + + step("init_del_and_add", r.addIPRules) + step("dup_add", r.justAddIPRules) + step("del", r.delIPRules) + step("dup_del", r.delIPRules) + +} + func TestDebugListLinks(t *testing.T) { links, err := netlink.LinkList() if err != nil {