From 0d80904fc2dde85fec6ec2f63d0890332d99b3b2 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Tue, 22 Sep 2020 00:49:44 +0000 Subject: [PATCH] wgengine/router: set up basic IPv6 routing/firewalling. Part of #19. Signed-off-by: David Anderson --- wgengine/router/router_linux.go | 371 ++++++++++++++++----------- wgengine/router/router_linux_test.go | 363 +++++++++++++++----------- wgengine/router/runner.go | 2 + 3 files changed, 434 insertions(+), 302 deletions(-) diff --git a/wgengine/router/router_linux.go b/wgengine/router/router_linux.go index 404fd03af..da371f19a 100644 --- a/wgengine/router/router_linux.go +++ b/wgengine/router/router_linux.go @@ -6,7 +6,6 @@ package router import ( "fmt" - "os/exec" "strings" "github.com/coreos/go-iptables/iptables" @@ -90,6 +89,7 @@ type linuxRouter struct { dns *dns.Manager ipt4 netfilterRunner + ipt6 netfilterRunner cmd commandRunner } @@ -104,12 +104,16 @@ func newUserspaceRouter(logf logger.Logf, _ *device.Device, tunDev tun.Device) ( return nil, err } - return newUserspaceRouterAdvanced(logf, tunname, ipt4, osCommandRunner{}) + ipt6, err := iptables.NewWithProtocol(iptables.ProtocolIPv6) + if err != nil { + return nil, err + } + + return newUserspaceRouterAdvanced(logf, tunname, ipt4, ipt6, osCommandRunner{}) } -func newUserspaceRouterAdvanced(logf logger.Logf, tunname string, netfilter netfilterRunner, cmd commandRunner) (Router, error) { - _, err := exec.Command("ip", "rule").Output() - ipRuleAvailable := (err == nil) +func newUserspaceRouterAdvanced(logf logger.Logf, tunname string, netfilter4, netfilter6 netfilterRunner, cmd commandRunner) (Router, error) { + ipRuleAvailable := (cmd.run("ip", "rule") == nil) mconfig := dns.ManagerConfig{ Logf: logf, @@ -121,7 +125,8 @@ func newUserspaceRouterAdvanced(logf logger.Logf, tunname string, netfilter netf ipRuleAvailable: ipRuleAvailable, tunname: tunname, netfilterMode: NetfilterOff, - ipt4: netfilter, + ipt4: netfilter4, + ipt6: netfilter6, cmd: cmd, dns: dns.NewManager(mconfig), }, nil @@ -434,58 +439,60 @@ func (r *linuxRouter) addIPRules() error { rg := newRunGroup(nil, r.cmd) - // NOTE(apenwarr): We leave spaces between each pref number. - // This is so the sysadmin can override by inserting rules in - // between if they want. - - // NOTE(apenwarr): This sequence seems complicated, right? - // If we could simply have a rule that said "match packets that - // *don't* have this fwmark", then we would only need to add one - // link to table 52 and we'd be done. Unfortunately, older kernels - // and 'ip rule' implementations (including busybox), don't support - // checking for the lack of a fwmark, only the presence. The technique - // below works even on very old kernels. - - // Packets from us, tagged with our fwmark, first try the kernel's - // main routing table. - rg.Run( - "ip", "rule", "add", - "pref", tailscaleRouteTable+"10", - "fwmark", tailscaleBypassMark, - "table", "main", - ) - // ...and then we try the 'default' table, for correctness, - // even though it's been empty on every Linux system I've ever seen. - rg.Run( - "ip", "rule", "add", - "pref", tailscaleRouteTable+"30", - "fwmark", tailscaleBypassMark, - "table", "default", - ) - // If neither of those matched (no default route on this system?) - // then packets from us should be aborted rather than falling through - // to the tailscale routes, because that would create routing loops. - rg.Run( - "ip", "rule", "add", - "pref", tailscaleRouteTable+"50", - "fwmark", tailscaleBypassMark, - "type", "unreachable", - ) - // If we get to this point, capture all packets and send them - // through to the tailscale route table. For apps other than us - // (ie. with no fwmark set), this is the first routing table, so - // it takes precedence over all the others, ie. VPN routes always - // beat non-VPN routes. - // - // NOTE(apenwarr): tables >255 are not supported in busybox, so we - // can't use a table number that aligns with the rule preferences. - rg.Run( - "ip", "rule", "add", - "pref", tailscaleRouteTable+"70", - "table", tailscaleRouteTable, - ) - // If that didn't match, then non-fwmark packets fall through to the - // usual rules (pref 32766 and 32767, ie. main and default). + for _, family := range []string{"-4", "-6"} { + // NOTE(apenwarr): We leave spaces between each pref number. + // This is so the sysadmin can override by inserting rules in + // between if they want. + + // NOTE(apenwarr): This sequence seems complicated, right? + // If we could simply have a rule that said "match packets that + // *don't* have this fwmark", then we would only need to add one + // link to table 52 and we'd be done. Unfortunately, older kernels + // and 'ip rule' implementations (including busybox), don't support + // checking for the lack of a fwmark, only the presence. The technique + // below works even on very old kernels. + + // Packets from us, tagged with our fwmark, first try the kernel's + // main routing table. + rg.Run( + "ip", family, "rule", "add", + "pref", tailscaleRouteTable+"10", + "fwmark", tailscaleBypassMark, + "table", "main", + ) + // ...and then we try the 'default' table, for correctness, + // even though it's been empty on every Linux system I've ever seen. + rg.Run( + "ip", family, "rule", "add", + "pref", tailscaleRouteTable+"30", + "fwmark", tailscaleBypassMark, + "table", "default", + ) + // If neither of those matched (no default route on this system?) + // then packets from us should be aborted rather than falling through + // to the tailscale routes, because that would create routing loops. + rg.Run( + "ip", family, "rule", "add", + "pref", tailscaleRouteTable+"50", + "fwmark", tailscaleBypassMark, + "type", "unreachable", + ) + // If we get to this point, capture all packets and send them + // through to the tailscale route table. For apps other than us + // (ie. with no fwmark set), this is the first routing table, so + // it takes precedence over all the others, ie. VPN routes always + // beat non-VPN routes. + // + // NOTE(apenwarr): tables >255 are not supported in busybox, so we + // can't use a table number that aligns with the rule preferences. + rg.Run( + "ip", family, "rule", "add", + "pref", tailscaleRouteTable+"70", + "table", tailscaleRouteTable, + ) + // If that didn't match, then non-fwmark packets fall through to the + // usual rules (pref 32766 and 32767, ie. main and default). + } return rg.ErrAcc } @@ -505,73 +512,91 @@ func (r *linuxRouter) delIPRules() error { // unknown rules during deletion. rg := newRunGroup([]int{2, 254}, r.cmd) - // When deleting rules, we want to be a bit specific (mention which - // table we were routing to) but not *too* specific (fwmarks, etc). - // That leaves us some flexibility to change these values in later - // versions without having ongoing hacks for every possible - // combination. - - // Delete old-style tailscale rules - // (never released in a stable version, so we can drop this - // support eventually). - rg.Run( - "ip", "rule", "del", - "pref", "10000", - "table", "main", - ) - - // Delete new-style tailscale rules. - rg.Run( - "ip", "rule", "del", - "pref", tailscaleRouteTable+"10", - "table", "main", - ) - rg.Run( - "ip", "rule", "del", - "pref", tailscaleRouteTable+"30", - "table", "default", - ) - rg.Run( - "ip", "rule", "del", - "pref", tailscaleRouteTable+"50", - "type", "unreachable", - ) - rg.Run( - "ip", "rule", "del", - "pref", tailscaleRouteTable+"70", - "table", tailscaleRouteTable, - ) + for _, family := range []string{"-4", "-6"} { + // When deleting rules, we want to be a bit specific (mention which + // table we were routing to) but not *too* specific (fwmarks, etc). + // That leaves us some flexibility to change these values in later + // versions without having ongoing hacks for every possible + // combination. + + // Delete old-style tailscale rules + // (never released in a stable version, so we can drop this + // support eventually). + rg.Run( + "ip", family, "rule", "del", + "pref", "10000", + "table", "main", + ) + + // Delete new-style tailscale rules. + rg.Run( + "ip", family, "rule", "del", + "pref", tailscaleRouteTable+"10", + "table", "main", + ) + rg.Run( + "ip", family, "rule", "del", + "pref", tailscaleRouteTable+"30", + "table", "default", + ) + rg.Run( + "ip", family, "rule", "del", + "pref", tailscaleRouteTable+"50", + "type", "unreachable", + ) + rg.Run( + "ip", family, "rule", "del", + "pref", tailscaleRouteTable+"70", + "table", tailscaleRouteTable, + ) + } + return rg.ErrAcc } // addNetfilterChains creates custom Tailscale chains in netfilter. func (r *linuxRouter) addNetfilterChains() error { - create := func(table, chain string) error { - err := r.ipt4.ClearChain(table, chain) + create := func(ipt netfilterRunner, table, chain string) error { + err := ipt.ClearChain(table, chain) if errCode(err) == 1 { // nonexistent chain. let's create it! - return r.ipt4.NewChain(table, chain) + return ipt.NewChain(table, chain) } if err != nil { return fmt.Errorf("setting up %s/%s: %w", table, chain, err) } return nil } - if err := create("filter", "ts-input"); err != nil { - return err + + for _, ipt := range []netfilterRunner{r.ipt4, r.ipt6} { + if err := create(ipt, "filter", "ts-input"); err != nil { + return err + } + if err := create(ipt, "filter", "ts-forward"); err != nil { + return err + } + if err := create(ipt, "nat", "ts-postrouting"); err != nil { + return err + } } - if err := create("filter", "ts-forward"); err != nil { + return nil +} + +// addNetfilterBase adds some basic processing rules to be +// supplemented by later calls to other helpers. +func (r *linuxRouter) addNetfilterBase() error { + if err := r.addNetfilterBase4(); err != nil { return err } - if err := create("nat", "ts-postrouting"); err != nil { + if err := r.addNetfilterBase6(); err != nil { return err } return nil } -// addNetfilterBase adds with some basic processing rules to be supplemented -// by later calls to other helpers. -func (r *linuxRouter) addNetfilterBase() error { +// addNetfilterBase4 adds some basic IPv4 processing rules to be +// supplemented by later calls to other helpers. +func (r *linuxRouter) addNetfilterBase4() error { // Only allow CGNAT range traffic to come from tailscale0. There // is an exception carved out for ranges used by ChromeOS, for // which we fall out of the Tailscale chain. @@ -580,11 +605,11 @@ func (r *linuxRouter) addNetfilterBase() error { // CGNAT range for other purposes :(. args := []string{"!", "-i", r.tunname, "-s", tsaddr.ChromeOSVMRange().String(), "-j", "RETURN"} if err := r.ipt4.Append("filter", "ts-input", args...); err != nil { - return fmt.Errorf("adding %v in filter/ts-input: %w", args, err) + return fmt.Errorf("adding %v in v4/filter/ts-input: %w", args, err) } args = []string{"!", "-i", r.tunname, "-s", tsaddr.CGNATRange().String(), "-j", "DROP"} if err := r.ipt4.Append("filter", "ts-input", args...); err != nil { - return fmt.Errorf("adding %v in filter/ts-input: %w", args, err) + return fmt.Errorf("adding %v in v4/filter/ts-input: %w", args, err) } // Forward all traffic from the Tailscale interface, and drop @@ -600,19 +625,43 @@ func (r *linuxRouter) addNetfilterBase() error { // use to effectively run that same test again. args = []string{"-i", r.tunname, "-j", "MARK", "--set-mark", tailscaleSubnetRouteMark} if err := r.ipt4.Append("filter", "ts-forward", args...); err != nil { - return fmt.Errorf("adding %v in filter/ts-forward: %w", args, err) + return fmt.Errorf("adding %v in v4/filter/ts-forward: %w", args, err) } args = []string{"-m", "mark", "--mark", tailscaleSubnetRouteMark, "-j", "ACCEPT"} if err := r.ipt4.Append("filter", "ts-forward", args...); err != nil { - return fmt.Errorf("adding %v in filter/ts-forward: %w", args, err) + return fmt.Errorf("adding %v in v4/filter/ts-forward: %w", args, err) } args = []string{"-o", r.tunname, "-s", tsaddr.CGNATRange().String(), "-j", "DROP"} if err := r.ipt4.Append("filter", "ts-forward", args...); err != nil { - return fmt.Errorf("adding %v in filter/ts-forward: %w", args, err) + return fmt.Errorf("adding %v in v4/filter/ts-forward: %w", args, err) } args = []string{"-o", r.tunname, "-j", "ACCEPT"} if err := r.ipt4.Append("filter", "ts-forward", args...); err != nil { - return fmt.Errorf("adding %v in filter/ts-forward: %w", args, err) + return fmt.Errorf("adding %v in v4/filter/ts-forward: %w", args, err) + } + + return nil +} + +// addNetfilterBase4 adds some basic IPv6 processing rules to be +// supplemented by later calls to other helpers. +func (r *linuxRouter) addNetfilterBase6() error { + // TODO: only allow traffic from Tailscale's ULA range to come + // from tailscale0. + + args := []string{"-i", r.tunname, "-j", "MARK", "--set-mark", tailscaleSubnetRouteMark} + if err := r.ipt6.Append("filter", "ts-forward", args...); err != nil { + return fmt.Errorf("adding %v in v6/filter/ts-forward: %w", args, err) + } + args = []string{"-m", "mark", "--mark", tailscaleSubnetRouteMark, "-j", "ACCEPT"} + if err := r.ipt6.Append("filter", "ts-forward", args...); err != nil { + return fmt.Errorf("adding %v in v6/filter/ts-forward: %w", args, err) + } + // TODO: drop forwarded traffic to tailscale0 from tailscale's ULA + // (see corresponding IPv4 CGNAT rule). + args = []string{"-o", r.tunname, "-j", "ACCEPT"} + if err := r.ipt6.Append("filter", "ts-forward", args...); err != nil { + return fmt.Errorf("adding %v in v6/filter/ts-forward: %w", args, err) } return nil @@ -620,8 +669,8 @@ func (r *linuxRouter) addNetfilterBase() error { // delNetfilterChains removes the custom Tailscale chains from netfilter. func (r *linuxRouter) delNetfilterChains() error { - del := func(table, chain string) error { - if err := r.ipt4.ClearChain(table, chain); err != nil { + del := func(ipt netfilterRunner, table, chain string) error { + if err := ipt.ClearChain(table, chain); err != nil { if errCode(err) == 1 { // nonexistent chain. That's fine, since it's // the desired state anyway. @@ -629,7 +678,7 @@ func (r *linuxRouter) delNetfilterChains() error { } return fmt.Errorf("flushing %s/%s: %w", table, chain, err) } - if err := r.ipt4.DeleteChain(table, chain); err != nil { + if err := ipt.DeleteChain(table, chain); err != nil { // this shouldn't fail, because if the chain didn't // exist, we would have returned after ClearChain. return fmt.Errorf("deleting %s/%s: %v", table, chain, err) @@ -637,14 +686,16 @@ func (r *linuxRouter) delNetfilterChains() error { return nil } - if err := del("filter", "ts-input"); err != nil { - return err - } - if err := del("filter", "ts-forward"); err != nil { - return err - } - if err := del("nat", "ts-postrouting"); err != nil { - return err + for _, ipt := range []netfilterRunner{r.ipt4, r.ipt6} { + if err := del(ipt, "filter", "ts-input"); err != nil { + return err + } + if err := del(ipt, "filter", "ts-forward"); err != nil { + return err + } + if err := del(ipt, "nat", "ts-postrouting"); err != nil { + return err + } } return nil @@ -653,8 +704,8 @@ func (r *linuxRouter) delNetfilterChains() error { // delNetfilterBase empties but does not remove custom Tailscale chains from // netfilter. func (r *linuxRouter) delNetfilterBase() error { - del := func(table, chain string) error { - if err := r.ipt4.ClearChain(table, chain); err != nil { + del := func(ipt netfilterRunner, table, chain string) error { + if err := ipt.ClearChain(table, chain); err != nil { if errCode(err) == 1 { // nonexistent chain. That's fine, since it's // the desired state anyway. @@ -665,14 +716,16 @@ func (r *linuxRouter) delNetfilterBase() error { return nil } - if err := del("filter", "ts-input"); err != nil { - return err - } - if err := del("filter", "ts-forward"); err != nil { - return err - } - if err := del("nat", "ts-postrouting"); err != nil { - return err + for _, ipt := range []netfilterRunner{r.ipt4, r.ipt6} { + if err := del(ipt, "filter", "ts-input"); err != nil { + return err + } + if err := del(ipt, "filter", "ts-forward"); err != nil { + return err + } + if err := del(ipt, "nat", "ts-postrouting"); err != nil { + return err + } } return nil @@ -682,31 +735,33 @@ func (r *linuxRouter) delNetfilterBase() error { // the relevant main netfilter chains. The tailscale chains must // already exist. func (r *linuxRouter) addNetfilterHooks() error { - divert := func(table, chain string) error { + divert := func(ipt netfilterRunner, table, chain string) error { tsChain := tsChain(chain) args := []string{"-j", tsChain} - exists, err := r.ipt4.Exists(table, chain, args...) + exists, err := ipt.Exists(table, chain, args...) if err != nil { return fmt.Errorf("checking for %v in %s/%s: %w", args, table, chain, err) } if exists { return nil } - if err := r.ipt4.Insert(table, chain, 1, args...); err != nil { + if err := ipt.Insert(table, chain, 1, args...); err != nil { return fmt.Errorf("adding %v in %s/%s: %w", args, table, chain, err) } return nil } - if err := divert("filter", "INPUT"); err != nil { - return err - } - if err := divert("filter", "FORWARD"); err != nil { - return err - } - if err := divert("nat", "POSTROUTING"); err != nil { - return err + for _, ipt := range []netfilterRunner{r.ipt4, r.ipt6} { + if err := divert(ipt, "filter", "INPUT"); err != nil { + return err + } + if err := divert(ipt, "filter", "FORWARD"); err != nil { + return err + } + if err := divert(ipt, "nat", "POSTROUTING"); err != nil { + return err + } } return nil } @@ -714,10 +769,10 @@ func (r *linuxRouter) addNetfilterHooks() error { // delNetfilterHooks deletes the calls to tailscale's netfilter chains // in the relevant main netfilter chains. func (r *linuxRouter) delNetfilterHooks() error { - del := func(table, chain string) error { + del := func(ipt netfilterRunner, table, chain string) error { tsChain := tsChain(chain) args := []string{"-j", tsChain} - if err := r.ipt4.Delete(table, chain, args...); err != nil { + if err := ipt.Delete(table, chain, args...); err != nil { // TODO(apenwarr): check for errCode(1) here. // Unfortunately the error code from the iptables // module resists unwrapping, unlike with other @@ -729,14 +784,16 @@ func (r *linuxRouter) delNetfilterHooks() error { return nil } - if err := del("filter", "INPUT"); err != nil { - return err - } - if err := del("filter", "FORWARD"); err != nil { - return err - } - if err := del("nat", "POSTROUTING"); err != nil { - return err + for _, ipt := range []netfilterRunner{r.ipt4, r.ipt6} { + if err := del(ipt, "filter", "INPUT"); err != nil { + return err + } + if err := del(ipt, "filter", "FORWARD"); err != nil { + return err + } + if err := del(ipt, "nat", "POSTROUTING"); err != nil { + return err + } } return nil } @@ -750,7 +807,10 @@ func (r *linuxRouter) addSNATRule() error { args := []string{"-m", "mark", "--mark", tailscaleSubnetRouteMark, "-j", "MASQUERADE"} if err := r.ipt4.Append("nat", "ts-postrouting", args...); err != nil { - return fmt.Errorf("adding %v in nat/ts-postrouting: %w", args, err) + return fmt.Errorf("adding %v in v4/nat/ts-postrouting: %w", args, err) + } + if err := r.ipt6.Append("nat", "ts-postrouting", args...); err != nil { + return fmt.Errorf("adding %v in v6/nat/ts-postrouting: %w", args, err) } return nil } @@ -764,7 +824,10 @@ func (r *linuxRouter) delSNATRule() error { args := []string{"-m", "mark", "--mark", tailscaleSubnetRouteMark, "-j", "MASQUERADE"} if err := r.ipt4.Delete("nat", "ts-postrouting", args...); err != nil { - return fmt.Errorf("deleting %v in nat/ts-postrouting: %w", args, err) + return fmt.Errorf("deleting %v in v4/nat/ts-postrouting: %w", args, err) + } + if err := r.ipt6.Delete("nat", "ts-postrouting", args...); err != nil { + return fmt.Errorf("deleting %v in v6/nat/ts-postrouting: %w", args, err) } return nil } diff --git a/wgengine/router/router_linux_test.go b/wgengine/router/router_linux_test.go index 1ecfbf250..6f9ffe932 100644 --- a/wgengine/router/router_linux_test.go +++ b/wgengine/router/router_linux_test.go @@ -34,10 +34,14 @@ func mustCIDRs(ss ...string) []netaddr.IPPrefix { func TestRouterStates(t *testing.T) { basic := ` -ip rule add pref 5210 fwmark 0x80000 table main -ip rule add pref 5230 fwmark 0x80000 table default -ip rule add pref 5250 fwmark 0x80000 type unreachable -ip rule add pref 5270 table 52 +ip rule add -4 pref 5210 fwmark 0x80000 table main +ip rule add -4 pref 5230 fwmark 0x80000 table default +ip rule add -4 pref 5250 fwmark 0x80000 type unreachable +ip rule add -4 pref 5270 table 52 +ip rule add -6 pref 5210 fwmark 0x80000 table main +ip rule add -6 pref 5230 fwmark 0x80000 table default +ip rule add -6 pref 5250 fwmark 0x80000 type unreachable +ip rule add -6 pref 5270 table 52 ` states := []struct { name string @@ -104,17 +108,24 @@ up ip addr add 100.101.102.104/10 dev tailscale0 ip route add 10.0.0.0/8 dev tailscale0 table 52 ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic + - `filter/FORWARD -j ts-forward -filter/INPUT -j ts-input -filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 -filter/ts-forward -m mark --mark 0x40000 -j ACCEPT -filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP -filter/ts-forward -o tailscale0 -j ACCEPT -filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT -filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN -filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP -nat/POSTROUTING -j ts-postrouting -nat/ts-postrouting -m mark --mark 0x40000 -j MASQUERADE + `v4/filter/FORWARD -j ts-forward +v4/filter/INPUT -j ts-input +v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 +v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT +v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP +v4/filter/ts-forward -o tailscale0 -j ACCEPT +v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT +v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN +v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP +v4/nat/POSTROUTING -j ts-postrouting +v4/nat/ts-postrouting -m mark --mark 0x40000 -j MASQUERADE +v6/filter/FORWARD -j ts-forward +v6/filter/INPUT -j ts-input +v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 +v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT +v6/filter/ts-forward -o tailscale0 -j ACCEPT +v6/nat/POSTROUTING -j ts-postrouting +v6/nat/ts-postrouting -m mark --mark 0x40000 -j MASQUERADE `, }, { @@ -129,16 +140,22 @@ up ip addr add 100.101.102.104/10 dev tailscale0 ip route add 10.0.0.0/8 dev tailscale0 table 52 ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic + - `filter/FORWARD -j ts-forward -filter/INPUT -j ts-input -filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 -filter/ts-forward -m mark --mark 0x40000 -j ACCEPT -filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP -filter/ts-forward -o tailscale0 -j ACCEPT -filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT -filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN -filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP -nat/POSTROUTING -j ts-postrouting + `v4/filter/FORWARD -j ts-forward +v4/filter/INPUT -j ts-input +v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 +v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT +v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP +v4/filter/ts-forward -o tailscale0 -j ACCEPT +v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT +v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN +v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP +v4/nat/POSTROUTING -j ts-postrouting +v6/filter/FORWARD -j ts-forward +v6/filter/INPUT -j ts-input +v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 +v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT +v6/filter/ts-forward -o tailscale0 -j ACCEPT +v6/nat/POSTROUTING -j ts-postrouting `, }, @@ -156,16 +173,22 @@ up ip addr add 100.101.102.104/10 dev tailscale0 ip route add 10.0.0.0/8 dev tailscale0 table 52 ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic + - `filter/FORWARD -j ts-forward -filter/INPUT -j ts-input -filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 -filter/ts-forward -m mark --mark 0x40000 -j ACCEPT -filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP -filter/ts-forward -o tailscale0 -j ACCEPT -filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT -filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN -filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP -nat/POSTROUTING -j ts-postrouting + `v4/filter/FORWARD -j ts-forward +v4/filter/INPUT -j ts-input +v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 +v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT +v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP +v4/filter/ts-forward -o tailscale0 -j ACCEPT +v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT +v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN +v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP +v4/nat/POSTROUTING -j ts-postrouting +v6/filter/FORWARD -j ts-forward +v6/filter/INPUT -j ts-input +v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 +v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT +v6/filter/ts-forward -o tailscale0 -j ACCEPT +v6/nat/POSTROUTING -j ts-postrouting `, }, { @@ -180,16 +203,22 @@ up ip addr add 100.101.102.104/10 dev tailscale0 ip route add 10.0.0.0/8 dev tailscale0 table 52 ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic + - `filter/FORWARD -j ts-forward -filter/INPUT -j ts-input -filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 -filter/ts-forward -m mark --mark 0x40000 -j ACCEPT -filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP -filter/ts-forward -o tailscale0 -j ACCEPT -filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT -filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN -filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP -nat/POSTROUTING -j ts-postrouting + `v4/filter/FORWARD -j ts-forward +v4/filter/INPUT -j ts-input +v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 +v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT +v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP +v4/filter/ts-forward -o tailscale0 -j ACCEPT +v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT +v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN +v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP +v4/nat/POSTROUTING -j ts-postrouting +v6/filter/FORWARD -j ts-forward +v6/filter/INPUT -j ts-input +v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 +v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT +v6/filter/ts-forward -o tailscale0 -j ACCEPT +v6/nat/POSTROUTING -j ts-postrouting `, }, @@ -205,13 +234,16 @@ up ip addr add 100.101.102.104/10 dev tailscale0 ip route add 10.0.0.0/8 dev tailscale0 table 52 ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic + - `filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 -filter/ts-forward -m mark --mark 0x40000 -j ACCEPT -filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP -filter/ts-forward -o tailscale0 -j ACCEPT -filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT -filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN -filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP + `v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 +v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT +v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP +v4/filter/ts-forward -o tailscale0 -j ACCEPT +v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT +v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN +v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP +v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 +v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT +v6/filter/ts-forward -o tailscale0 -j ACCEPT `, }, { @@ -226,22 +258,28 @@ up ip addr add 100.101.102.104/10 dev tailscale0 ip route add 10.0.0.0/8 dev tailscale0 table 52 ip route add 100.100.100.100/32 dev tailscale0 table 52` + basic + - `filter/FORWARD -j ts-forward -filter/INPUT -j ts-input -filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 -filter/ts-forward -m mark --mark 0x40000 -j ACCEPT -filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP -filter/ts-forward -o tailscale0 -j ACCEPT -filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT -filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN -filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP -nat/POSTROUTING -j ts-postrouting + `v4/filter/FORWARD -j ts-forward +v4/filter/INPUT -j ts-input +v4/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 +v4/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT +v4/filter/ts-forward -o tailscale0 -s 100.64.0.0/10 -j DROP +v4/filter/ts-forward -o tailscale0 -j ACCEPT +v4/filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT +v4/filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -j RETURN +v4/filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP +v4/nat/POSTROUTING -j ts-postrouting +v6/filter/FORWARD -j ts-forward +v6/filter/INPUT -j ts-input +v6/filter/ts-forward -i tailscale0 -j MARK --set-mark 0x40000 +v6/filter/ts-forward -m mark --mark 0x40000 -j ACCEPT +v6/filter/ts-forward -o tailscale0 -j ACCEPT +v6/nat/POSTROUTING -j ts-postrouting `, }, } fake := NewFakeOS(t) - router, err := newUserspaceRouterAdvanced(t.Logf, "tailscale0", fake, fake) + router, err := newUserspaceRouterAdvanced(t.Logf, "tailscale0", fake.netfilter4, fake.netfilter6, fake) if err != nil { t.Fatalf("failed to create router: %v", err) } @@ -275,21 +313,15 @@ nat/POSTROUTING -j ts-postrouting } } -// fakeOS implements netfilterRunner and commandRunner, but captures -// changes without touching the OS. -type fakeOS struct { - t *testing.T - up bool - ips []string - routes []string - rules []string - netfilter map[string][]string +type fakeNetfilter struct { + t *testing.T + n map[string][]string } -func NewFakeOS(t *testing.T) *fakeOS { - return &fakeOS{ +func newNetfilter(t *testing.T) *fakeNetfilter { + return &fakeNetfilter{ t: t, - netfilter: map[string][]string{ + n: map[string][]string{ "filter/INPUT": nil, "filter/OUTPUT": nil, "filter/FORWARD": nil, @@ -300,67 +332,32 @@ func NewFakeOS(t *testing.T) *fakeOS { } } -var errExec = errors.New("execution failed") - -func (o *fakeOS) String() string { - var b strings.Builder - if o.up { - b.WriteString("up\n") - } else { - b.WriteString("down\n") - } - - for _, ip := range o.ips { - fmt.Fprintf(&b, "ip addr add %s\n", ip) - } - - for _, route := range o.routes { - fmt.Fprintf(&b, "ip route add %s\n", route) - } - - for _, rule := range o.rules { - fmt.Fprintf(&b, "ip rule add %s\n", rule) - } - - var chains []string - for chain := range o.netfilter { - chains = append(chains, chain) - } - sort.Strings(chains) - for _, chain := range chains { - for _, rule := range o.netfilter[chain] { - fmt.Fprintf(&b, "%s %s\n", chain, rule) - } - } - return b.String()[:len(b.String())-1] -} - -func (o *fakeOS) Insert(table, chain string, pos int, args ...string) error { +func (n *fakeNetfilter) Insert(table, chain string, pos int, args ...string) error { k := table + "/" + chain - if rules, ok := o.netfilter[k]; ok { + if rules, ok := n.n[k]; ok { if pos > len(rules)+1 { - o.t.Errorf("bad position %d in %s", pos, k) + n.t.Errorf("bad position %d in %s", pos, k) return errExec } rules = append(rules, "") copy(rules[pos:], rules[pos-1:]) rules[pos-1] = strings.Join(args, " ") - o.netfilter[k] = rules + n.n[k] = rules } else { - o.t.Errorf("unknown table/chain %s", k) + n.t.Errorf("unknown table/chain %s", k) return errExec } return nil } -func (o *fakeOS) Append(table, chain string, args ...string) error { +func (n *fakeNetfilter) Append(table, chain string, args ...string) error { k := table + "/" + chain - return o.Insert(table, chain, len(o.netfilter[k])+1, args...) + return n.Insert(table, chain, len(n.n[k])+1, args...) } -func (o *fakeOS) Exists(table, chain string, args ...string) (bool, error) { +func (n *fakeNetfilter) Exists(table, chain string, args ...string) (bool, error) { k := table + "/" + chain - if rules, ok := o.netfilter[k]; ok { + if rules, ok := n.n[k]; ok { for _, rule := range rules { if rule == strings.Join(args, " ") { return true, nil @@ -368,75 +365,132 @@ func (o *fakeOS) Exists(table, chain string, args ...string) (bool, error) { } return false, nil } else { - o.t.Errorf("unknown table/chain %s", k) + n.t.Errorf("unknown table/chain %s", k) return false, errExec } } -func (o *fakeOS) Delete(table, chain string, args ...string) error { +func (n *fakeNetfilter) Delete(table, chain string, args ...string) error { k := table + "/" + chain - if rules, ok := o.netfilter[k]; ok { + if rules, ok := n.n[k]; ok { for i, rule := range rules { if rule == strings.Join(args, " ") { rules = append(rules[:i], rules[i+1:]...) - o.netfilter[k] = rules + n.n[k] = rules return nil } } - o.t.Errorf("delete of unknown rule %q from %s", strings.Join(args, " "), k) + n.t.Errorf("delete of unknown rule %q from %s", strings.Join(args, " "), k) return errExec } else { - o.t.Errorf("unknown table/chain %s", k) + n.t.Errorf("unknown table/chain %s", k) return errExec } } -func (o *fakeOS) ListChains(table string) (ret []string, err error) { - for chain := range o.netfilter { - pfx := table + "/" - if strings.HasPrefix(chain, pfx) { - ret = append(ret, chain[len(pfx):]) - } - } - return ret, nil -} - -func (o *fakeOS) ClearChain(table, chain string) error { +func (n *fakeNetfilter) ClearChain(table, chain string) error { k := table + "/" + chain - if _, ok := o.netfilter[k]; ok { - o.netfilter[k] = nil + if _, ok := n.n[k]; ok { + n.n[k] = nil return nil } else { - o.t.Logf("note: ClearChain: unknown table/chain %s", k) + n.t.Logf("note: ClearChain: unknown table/chain %s", k) return errors.New("exitcode:1") } } -func (o *fakeOS) NewChain(table, chain string) error { +func (n *fakeNetfilter) NewChain(table, chain string) error { k := table + "/" + chain - if _, ok := o.netfilter[k]; ok { - o.t.Errorf("table/chain %s already exists", k) + if _, ok := n.n[k]; ok { + n.t.Errorf("table/chain %s already exists", k) return errExec } - o.netfilter[k] = nil + n.n[k] = nil return nil } -func (o *fakeOS) DeleteChain(table, chain string) error { +func (n *fakeNetfilter) DeleteChain(table, chain string) error { k := table + "/" + chain - if rules, ok := o.netfilter[k]; ok { + if rules, ok := n.n[k]; ok { if len(rules) != 0 { - o.t.Errorf("%s is not empty", k) + n.t.Errorf("%s is not empty", k) return errExec } - delete(o.netfilter, k) + delete(n.n, k) return nil } else { - o.t.Errorf("%s does not exist", k) + n.t.Errorf("%s does not exist", k) return errExec } } +// fakeOS implements commandRunner and provides v4 and v6 +// netfilterRunners, but captures changes without touching the OS. +type fakeOS struct { + t *testing.T + up bool + ips []string + routes []string + rules []string + netfilter4 *fakeNetfilter + netfilter6 *fakeNetfilter +} + +func NewFakeOS(t *testing.T) *fakeOS { + return &fakeOS{ + t: t, + netfilter4: newNetfilter(t), + netfilter6: newNetfilter(t), + } +} + +var errExec = errors.New("execution failed") + +func (o *fakeOS) String() string { + var b strings.Builder + if o.up { + b.WriteString("up\n") + } else { + b.WriteString("down\n") + } + + for _, ip := range o.ips { + fmt.Fprintf(&b, "ip addr add %s\n", ip) + } + + for _, route := range o.routes { + fmt.Fprintf(&b, "ip route add %s\n", route) + } + + for _, rule := range o.rules { + fmt.Fprintf(&b, "ip rule add %s\n", rule) + } + + var chains []string + for chain := range o.netfilter4.n { + chains = append(chains, chain) + } + sort.Strings(chains) + for _, chain := range chains { + for _, rule := range o.netfilter4.n[chain] { + fmt.Fprintf(&b, "v4/%s %s\n", chain, rule) + } + } + + chains = nil + for chain := range o.netfilter6.n { + chains = append(chains, chain) + } + sort.Strings(chains) + for _, chain := range chains { + for _, rule := range o.netfilter6.n[chain] { + fmt.Fprintf(&b, "v6/%s %s\n", chain, rule) + } + } + + return b.String()[:len(b.String())-1] +} + func (o *fakeOS) run(args ...string) error { unexpected := func() error { o.t.Errorf("unexpected invocation %q", strings.Join(args, " ")) @@ -446,7 +500,20 @@ func (o *fakeOS) run(args ...string) error { return unexpected() } + if len(args) == 2 && args[1] == "rule" { + // naked invocation of `ip rule` is a feature test. Return + // successfully. + return nil + } + + family := "" rest := strings.Join(args[3:], " ") + if args[1] == "-4" || args[1] == "-6" { + family = args[1] + copy(args[1:], args[2:]) + args = args[:len(args)-1] + rest = family + " " + strings.Join(args[3:], " ") + } var l *[]string switch args[1] { diff --git a/wgengine/router/runner.go b/wgengine/router/runner.go index 5e6bee0bb..b626eea23 100644 --- a/wgengine/router/runner.go +++ b/wgengine/router/runner.go @@ -24,6 +24,8 @@ type commandRunner interface { type osCommandRunner struct{} +// errCode extracts and returns the process exit code from err, or +// zero if err is nil. func errCode(err error) int { if err == nil { return 0