From dc2fbf5877ff50b02f5315c04288966244d9e7b1 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 27 Oct 2021 21:36:29 -0700 Subject: [PATCH] wgengine/router: start using netlink instead of 'ip' on Linux Converts up, down, add/del addresses, add/del routes. Not yet done: rules. Updates #391 Change-Id: I02554ca07046d18f838e04a626ba99bbd35266fb Signed-off-by: Brad Fitzpatrick --- cmd/tailscaled/depaware.txt | 3 + go.mod | 2 + go.sum | 5 + wgengine/router/router_linux.go | 177 ++++++++++++++++++++++++--- wgengine/router/router_linux_test.go | 25 ++++ 5 files changed, 196 insertions(+), 16 deletions(-) diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 9c7607784..ba5d2eeb2 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -96,6 +96,9 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de L github.com/u-root/uio/rand from github.com/insomniacslk/dhcp/dhcpv4 L github.com/u-root/uio/ubinary from github.com/u-root/uio/uio L github.com/u-root/uio/uio from github.com/insomniacslk/dhcp/dhcpv4+ + L 💣 github.com/vishvananda/netlink from tailscale.com/wgengine/router + L 💣 github.com/vishvananda/netlink/nl from github.com/vishvananda/netlink + L github.com/vishvananda/netns from github.com/vishvananda/netlink+ 💣 go4.org/intern from inet.af/netaddr 💣 go4.org/mem from tailscale.com/client/tailscale+ go4.org/unsafe/assume-no-moving-gc from go4.org/intern diff --git a/go.mod b/go.mod index 47c377f7f..3276774c7 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( github.com/tcnksm/go-httpstat v0.2.0 github.com/toqueteos/webbrowser v1.2.0 github.com/ulikunitz/xz v0.5.10 // indirect + github.com/vishvananda/netlink v1.1.0 go4.org/mem v0.0.0-20201119185036-c04c5a6ff174 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 golang.org/x/net v0.0.0-20211020060615-d418f374d309 @@ -191,6 +192,7 @@ require ( github.com/ultraware/funlen v0.0.3 // indirect github.com/ultraware/whitespace v0.0.4 // indirect github.com/uudashr/gocognit v1.0.1 // indirect + github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect github.com/xanzy/ssh-agent v0.3.0 // indirect go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect diff --git a/go.sum b/go.sum index 93f0a9de5..71e622e65 100644 --- a/go.sum +++ b/go.sum @@ -681,6 +681,10 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA= github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD5KW4lOuSdPKzY= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= @@ -832,6 +836,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606122018-79a91cf218c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/wgengine/router/router_linux.go b/wgengine/router/router_linux.go index 8acb54cd8..2bfd7254c 100644 --- a/wgengine/router/router_linux.go +++ b/wgengine/router/router_linux.go @@ -13,10 +13,13 @@ import ( "os/exec" "strconv" "strings" + "syscall" "time" "github.com/coreos/go-iptables/iptables" "github.com/go-multierror/multierror" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" "golang.org/x/time/rate" "golang.zx2c4.com/wireguard/tun" "inet.af/netaddr" @@ -81,7 +84,8 @@ const ( // implementation believes that table numbers are 8-bit integers, so // for maximum compatibility we have to stay in the 0-255 range even // though linux itself supports larger numbers. - tailscaleRouteTable = "52" + tailscaleRouteTable = "52" + tailscaleRouteTableNum = 52 ) // netfilterRunner abstracts helpers to run netfilter commands. It @@ -196,6 +200,17 @@ func useAmbientCaps() bool { return v >= 7 } +// useIPCommand reports whether r should use the "ip" command (or its +// fake commandRunner for tests) instead of netlink. +func (r *linuxRouter) useIPCommand() bool { + // In the future we might need to fall back to using the "ip" + // command if, say, netlink is blocked somewhere but the ip + // command is allowed to use netlink. For now we only use the ip + // command runner in tests. + _, ok := r.cmd.(osCommandRunner) + return !ok +} + // onIPRuleDeleted is the callback from the link monitor for when an IP policy // rule is deleted. See Issue 1591. // @@ -449,8 +464,18 @@ func (r *linuxRouter) addAddress(addr netaddr.IPPrefix) error { if !r.v6Available && addr.IP().Is6() { return nil } - if err := r.cmd.run("ip", "addr", "add", addr.String(), "dev", r.tunname); err != nil { - return fmt.Errorf("adding address %q to tunnel interface: %w", addr, err) + if r.useIPCommand() { + if err := r.cmd.run("ip", "addr", "add", addr.String(), "dev", r.tunname); err != nil { + return fmt.Errorf("adding address %q to tunnel interface: %w", addr, err) + } + } else { + link, err := r.link() + if err != nil { + return fmt.Errorf("adding address %v, %w", addr, err) + } + if err := netlink.AddrReplace(link, nlAddrOfPrefix(addr)); err != nil { + return fmt.Errorf("adding address %v from tunnel interface: %w", addr, err) + } } if err := r.addLoopbackRule(addr.IP()); err != nil { return err @@ -468,8 +493,18 @@ func (r *linuxRouter) delAddress(addr netaddr.IPPrefix) error { if err := r.delLoopbackRule(addr.IP()); err != nil { return err } - if err := r.cmd.run("ip", "addr", "del", addr.String(), "dev", r.tunname); err != nil { - return fmt.Errorf("deleting address %q from tunnel interface: %w", addr, err) + if r.useIPCommand() { + if err := r.cmd.run("ip", "addr", "del", addr.String(), "dev", r.tunname); err != nil { + return fmt.Errorf("deleting address %q from tunnel interface: %w", addr, err) + } + } else { + link, err := r.link() + if err != nil { + return fmt.Errorf("deleting address %v, %w", addr, err) + } + if err := netlink.AddrDel(link, nlAddrOfPrefix(addr)); err != nil { + return fmt.Errorf("deleting address %v from tunnel interface: %w", addr, err) + } } return nil } @@ -522,7 +557,21 @@ func (r *linuxRouter) delLoopbackRule(addr netaddr.IP) error { // interface. Fails if the route already exists, or if adding the // route fails. func (r *linuxRouter) addRoute(cidr netaddr.IPPrefix) error { - return r.addRouteDef([]string{normalizeCIDR(cidr), "dev", r.tunname}, cidr) + if !r.v6Available && cidr.IP().Is6() { + return nil + } + if r.useIPCommand() { + return r.addRouteDef([]string{normalizeCIDR(cidr), "dev", r.tunname}, cidr) + } + linkIndex, err := r.linkIndex() + if err != nil { + return err + } + return netlink.RouteReplace(&netlink.Route{ + LinkIndex: linkIndex, + Dst: cidr.Masked().IPNet(), + Table: r.routeTable(), + }) } // addThrowRoute adds a throw route for the provided cidr. @@ -533,7 +582,21 @@ func (r *linuxRouter) addThrowRoute(cidr netaddr.IPPrefix) error { if !r.ipRuleAvailable { return nil } - return r.addRouteDef([]string{"throw", normalizeCIDR(cidr)}, cidr) + if !r.v6Available && cidr.IP().Is6() { + return nil + } + if r.useIPCommand() { + return r.addRouteDef([]string{"throw", normalizeCIDR(cidr)}, cidr) + } + err := netlink.RouteReplace(&netlink.Route{ + Dst: cidr.Masked().IPNet(), + Table: tailscaleRouteTableNum, + Type: unix.RTN_THROW, + }) + if err != nil { + r.logf("THROW ERROR adding %v: %#v", cidr, err) + } + return err } func (r *linuxRouter) addRouteDef(routeDef []string, cidr netaddr.IPPrefix) error { @@ -549,11 +612,11 @@ func (r *linuxRouter) addRouteDef(routeDef []string, cidr netaddr.IPPrefix) erro return nil } - // TODO(bradfitz): remove this ugly hack to detect failure to - // add a route that already exists (as happens in when we're - // racing to add kernel-maintained routes when enabling exit - // nodes w/o Local LAN access, Issue 3060) and use netlink - // directly instead (Issue 391). + // This is an ugly hack to detect failure to add a route that + // already exists (as happens in when we're racing to add + // kernel-maintained routes when enabling exit nodes w/o Local + // LAN access, Issue 3060). Fortunately in the common case we + // use netlink directly instead and don't exercise this code. if errCode(err) == 2 && strings.Contains(err.Error(), "RTNETLINK answers: File exists") { r.logf("ignoring route add of %v; already exists", cidr) return nil @@ -561,11 +624,32 @@ func (r *linuxRouter) addRouteDef(routeDef []string, cidr netaddr.IPPrefix) erro return err } +var errESRCH error = syscall.ESRCH + // delRoute removes the route for cidr pointing to the tunnel // interface. Fails if the route doesn't exist, or if removing the // route fails. func (r *linuxRouter) delRoute(cidr netaddr.IPPrefix) error { - return r.delRouteDef([]string{normalizeCIDR(cidr), "dev", r.tunname}, cidr) + if !r.v6Available && cidr.IP().Is6() { + return nil + } + if r.useIPCommand() { + return r.delRouteDef([]string{normalizeCIDR(cidr), "dev", r.tunname}, cidr) + } + linkIndex, err := r.linkIndex() + if err != nil { + return err + } + err = netlink.RouteDel(&netlink.Route{ + LinkIndex: linkIndex, + Dst: cidr.Masked().IPNet(), + Table: r.routeTable(), + }) + if errors.Is(err, errESRCH) { + // Didn't exist to begin with. + return nil + } + return err } // delThrowRoute removes the throw route for the cidr. Fails if the route @@ -574,7 +658,22 @@ func (r *linuxRouter) delThrowRoute(cidr netaddr.IPPrefix) error { if !r.ipRuleAvailable { return nil } - return r.delRouteDef([]string{"throw", normalizeCIDR(cidr)}, cidr) + if !r.v6Available && cidr.IP().Is6() { + return nil + } + if r.useIPCommand() { + return r.delRouteDef([]string{"throw", normalizeCIDR(cidr)}, cidr) + } + err := netlink.RouteDel(&netlink.Route{ + Dst: cidr.Masked().IPNet(), + Table: r.routeTable(), + Type: unix.RTN_THROW, + }) + if errors.Is(err, errESRCH) { + // Didn't exist to begin with. + return nil + } + return err } func (r *linuxRouter) delRouteDef(routeDef []string, cidr netaddr.IPPrefix) error { @@ -619,14 +718,54 @@ func (r *linuxRouter) hasRoute(routeDef []string, cidr netaddr.IPPrefix) (bool, return len(out) > 0, nil } +func (r *linuxRouter) link() (netlink.Link, error) { + link, err := netlink.LinkByName(r.tunname) + if err != nil { + return nil, fmt.Errorf("failed to look up link %q: %w", r.tunname, err) + } + return link, nil +} + +func (r *linuxRouter) linkIndex() (int, error) { + // TODO(bradfitz): cache this? It doesn't change often, and on start-up + // hundreds of addRoute calls to add /32s can happen quickly. + link, err := r.link() + if err != nil { + return 0, err + } + return link.Attrs().Index, nil +} + +// routeTable returns the route table to use. +func (r *linuxRouter) routeTable() int { + if r.ipRuleAvailable { + return tailscaleRouteTableNum + } + return 0 +} + // upInterface brings up the tunnel interface. func (r *linuxRouter) upInterface() error { - return r.cmd.run("ip", "link", "set", "dev", r.tunname, "up") + if r.useIPCommand() { + return r.cmd.run("ip", "link", "set", "dev", r.tunname, "up") + } + link, err := r.link() + if err != nil { + return fmt.Errorf("bringing interface up, %w", err) + } + return netlink.LinkSetUp(link) } // downInterface sets the tunnel interface administratively down. func (r *linuxRouter) downInterface() error { - return r.cmd.run("ip", "link", "set", "dev", r.tunname, "down") + if r.useIPCommand() { + return r.cmd.run("ip", "link", "set", "dev", r.tunname, "down") + } + link, err := r.link() + if err != nil { + return fmt.Errorf("bringing interface down, %w", err) + } + return netlink.LinkSetDown(link) } func (r *linuxRouter) iprouteFamilies() []string { @@ -1301,3 +1440,9 @@ func checkIPRuleSupportsV6() error { exec.Command("ip", del...).Run() return nil } + +func nlAddrOfPrefix(p netaddr.IPPrefix) *netlink.Addr { + return &netlink.Addr{ + IPNet: p.IPNet(), + } +} diff --git a/wgengine/router/router_linux_test.go b/wgengine/router/router_linux_test.go index ab42752c8..906b82d2f 100644 --- a/wgengine/router/router_linux_test.go +++ b/wgengine/router/router_linux_test.go @@ -15,6 +15,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/vishvananda/netlink" "golang.zx2c4.com/wireguard/tun" "inet.af/netaddr" "tailscale.com/tstest" @@ -708,3 +709,27 @@ func TestDelRouteIdempotent(t *testing.T) { t.Logf("Log output:\n%s", out) } } + +func TestDebugListLinks(t *testing.T) { + links, err := netlink.LinkList() + if err != nil { + t.Fatal(err) + } + for _, ln := range links { + t.Logf("Link: %+v", ln) + } +} + +func TestDebugListRoutes(t *testing.T) { + // We need to pass a non-nil route to RouteListFiltered, along + // with the netlink.RT_FILTER_TABLE bit set in the filter + // mask, otherwise it ignores non-main routes. + filter := &netlink.Route{} + routes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, filter, netlink.RT_FILTER_TABLE) + if err != nil { + t.Fatal(err) + } + for _, r := range routes { + t.Logf("Route: %+v", r) + } +}