archive
/
tailscale
mirror of https://github.com/tailscale/tailscale/
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

wgengine/router: add a test for linux router state transitions.

Browse Source 
Signed-off-by: David Anderson <danderson@tailscale.com>
pull/395/head
David Anderson 5 years ago committed by Dave Anderson
parent 292606a975
commit c67c8913c3
2 changed files with 560 additions and 15 deletions
Show Stats Download Patch File Download Diff File

59
wgengine/router/router_linux.go
Unescape Escape View File

@ -53,6 +53,22 @@ const (
// avoid allocating Tailscale IPs from it, to avoid conflicts.
const chromeOSVMRange = "100.115.92.0/23"
type netfilterRunner interface {
Insert(table, chain string, pos int, args ...string) error
Append(table, chain string, args ...string) error
Exists(table, chain string, args ...string) (bool, error)
Delete(table, chain string, args ...string) error
ListChains(table string) ([]string, error)
ClearChain(table, chain string) error
NewChain(table, chain string) error
DeleteChain(table, chain string) error
}
type commandRunner interface {
run(...string) error
output(...string) ([]byte, error)
}
type linuxRouter struct {
logf func(fmt string, args ...interface{})
tunname string
@ -62,7 +78,8 @@ type linuxRouter struct {
snatSubnetRoutes bool
netfilterMode NetfilterMode
ipt4 *iptables.IPTables
ipt4 netfilterRunner
cmd commandRunner
}
func newUserspaceRouter(logf logger.Logf, _ *device.Device, tunDev tun.Device) (Router, error) {
@ -76,25 +93,37 @@ func newUserspaceRouter(logf logger.Logf, _ *device.Device, tunDev tun.Device) (
return nil, err
}
return newUserspaceRouterAdvanced(logf, tunname, ipt4, osCommandRunner{})
}
func newUserspaceRouterAdvanced(logf logger.Logf, tunname string, netfilter netfilterRunner, cmd commandRunner) (Router, error) {
return &linuxRouter{
logf: logf,
tunname: tunname,
netfilterMode: NetfilterOff,
ipt4: ipt4,
ipt4: netfilter,
cmd: cmd,
}, nil
}
func cmd(args ...string) error {
type osCommandRunner struct{}
func (o osCommandRunner) run(args ...string) error {
_, err := o.output(args...)
return err
}
func (o osCommandRunner) output(args ...string) ([]byte, error) {
if len(args) == 0 {
return errors.New("cmd: no argv[0]")
return nil, errors.New("cmd: no argv[0]")
}
out, err := exec.Command(args[0], args[1:]...).CombinedOutput()
if err != nil {
return fmt.Errorf("running %q failed: %v\n%s", strings.Join(args, " "), err, out)
return nil, fmt.Errorf("running %q failed: %v\n%s", strings.Join(args, " "), err, out)
}
return nil
return out, nil
}
func (r *linuxRouter) Up() error {
@ -384,7 +413,7 @@ func (r *linuxRouter) restoreResolvConf() error {
// address is already assigned to the interface, or if the addition
// fails.
func (r *linuxRouter) addAddress(addr netaddr.IPPrefix) error {
if err := cmd("ip", "addr", "add", addr.String(), "dev", r.tunname); err != nil {
if err := r.cmd.run("ip", "addr", "add", addr.String(), "dev", r.tunname); err != nil {
return fmt.Errorf("adding address %q to tunnel interface: %v", addr, err)
}
if err := r.addLoopbackRule(addr.IP); err != nil {
@ -400,7 +429,7 @@ func (r *linuxRouter) delAddress(addr netaddr.IPPrefix) error {
if err := r.delLoopbackRule(addr.IP); err != nil {
return err
}
if err := cmd("ip", "addr", "del", addr.String(), "dev", r.tunname); err != nil {
if err := r.cmd.run("ip", "addr", "del", addr.String(), "dev", r.tunname); err != nil {
return fmt.Errorf("deleting address %q from tunnel interface: %v", addr, err)
}
return nil
@ -434,14 +463,14 @@ func (r *linuxRouter) delLoopbackRule(addr netaddr.IP) error {
// interface. Fails if the route already exists, or if adding the
// route fails.
func (r *linuxRouter) addRoute(cidr netaddr.IPPrefix) error {
return cmd("ip", "route", "add", normalizeCIDR(cidr), "dev", r.tunname, "scope", "global")
return r.cmd.run("ip", "route", "add", normalizeCIDR(cidr), "dev", r.tunname, "scope", "global")
}
// delRoute removes the route for cidr pointing to the tunnel
// interface. Fails if the route doesn't exist, or if removing the
// route fails.
func (r *linuxRouter) delRoute(cidr netaddr.IPPrefix) error {
return cmd("ip", "route", "del", normalizeCIDR(cidr), "dev", r.tunname, "scope", "global")
return r.cmd.run("ip", "route", "del", normalizeCIDR(cidr), "dev", r.tunname, "scope", "global")
}
// addSubnetRule adds a netfilter rule that allows traffic to flow
@ -480,13 +509,13 @@ func (r *linuxRouter) delSubnetRule(cidr netaddr.IPPrefix) error {
// upInterface brings up the tunnel interface and adds it to the
// Tailscale interface group.
func (r *linuxRouter) upInterface() error {
return cmd("ip", "link", "set", "dev", r.tunname, "group", "10000", "up")
return r.cmd.run("ip", "link", "set", "dev", r.tunname, "group", "10000", "up")
}
// downInterface sets the tunnel interface administratively down, and
// returns it to the default interface group.
func (r *linuxRouter) downInterface() error {
return cmd("ip", "link", "set", "dev", r.tunname, "group", "0", "down")
return r.cmd.run("ip", "link", "set", "dev", r.tunname, "group", "0", "down")
}
// addBypassRule adds the policy routing rule that avoids tailscaled
@ -496,13 +525,13 @@ func (r *linuxRouter) addBypassRule() error {
if err := r.delBypassRule(); err != nil {
return err
}
return cmd("ip", "rule", "add", "fwmark", tailscaleBypassMark, "priority", "10000", "table", "main", "suppress_ifgroup", "10000")
return r.cmd.run("ip", "rule", "add", "fwmark", tailscaleBypassMark, "priority", "10000", "table", "main", "suppress_ifgroup", "10000")
}
// delBypassrule removes the policy routing rule that avoids
// tailscaled routing loops, if it exists.
func (r *linuxRouter) delBypassRule() error {
out, err := exec.Command("ip", "rule", "list", "priority", "10000").CombinedOutput()
out, err := r.cmd.output("ip", "rule", "list", "priority", "10000")
if err != nil {
// Busybox ships an `ip` binary that doesn't understand
// uncommon rules. Try to detect this explicitly, and steer
@ -523,7 +552,7 @@ func (r *linuxRouter) delBypassRule() error {
if !bytes.Contains(out, []byte(" fwmark "+tailscaleBypassMark)) {
return fmt.Errorf("ip rule 10000 doesn't look like a Tailscale policy rule: %q", string(out))
}
return cmd("ip", "rule", "del", "priority", "10000")
return r.cmd.run("ip", "rule", "del", "priority", "10000")
}
// addNetfilterBase adds custom Tailscale chains to netfilter, along

516
wgengine/router/router_linux_test.go
Unescape Escape View File

@ -0,0 +1,516 @@
// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package router
import (
"errors"
"fmt"
"math/rand"
"sort"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"inet.af/netaddr"
)
func mustCIDR(s string) netaddr.IPPrefix {
pfx, err := netaddr.ParseIPPrefix(s)
if err != nil {
panic(err)
}
return pfx
}
func mustCIDRs(ss ...string) []netaddr.IPPrefix {
var ret []netaddr.IPPrefix
for _, s := range ss {
ret = append(ret, mustCIDR(s))
}
return ret
}
func TestRouterStates(t *testing.T) {
states := []struct {
name string
in *Config
want string
}{
{
name: "no config",
in: nil,
want: `
up
ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
`,
},
{
name: "local addr only",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.103/10"),
},
want: `
up
ip addr add 100.101.102.103/10 dev tailscale0
ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
`,
},
{
name: "addr and routes",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.103/10"),
Routes: mustCIDRs("100.100.100.100/32", "192.168.16.0/24"),
},
want: `
up
ip addr add 100.101.102.103/10 dev tailscale0
ip route add 100.100.100.100/32 dev tailscale0 scope global
ip route add 192.168.16.0/24 dev tailscale0 scope global
ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
`,
},
{
name: "addr and routes and subnet routes",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.103/10"),
Routes: mustCIDRs("100.100.100.100/32", "192.168.16.0/24"),
SubnetRoutes: mustCIDRs("200.0.0.0/8"),
},
want: `
up
ip addr add 100.101.102.103/10 dev tailscale0
ip route add 100.100.100.100/32 dev tailscale0 scope global
ip route add 192.168.16.0/24 dev tailscale0 scope global
ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
`,
},
{
name: "addr and routes and subnet routes with netfilter",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.104/10"),
Routes: mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
SubnetRoutes: mustCIDRs("200.0.0.0/8"),
SNATSubnetRoutes: true,
NetfilterMode: NetfilterOn,
},
want: `
up
ip addr add 100.101.102.104/10 dev tailscale0
ip route add 10.0.0.0/8 dev tailscale0 scope global
ip route add 100.100.100.100/32 dev tailscale0 scope global
ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
filter/FORWARD -j ts-forward
filter/INPUT -j ts-input
filter/ts-forward -o tailscale0 -s 200.0.0.0/8 -j ACCEPT
filter/ts-forward -i tailscale0 -d 200.0.0.0/8 -j MARK --set-mark 0x10000/0x10000
filter/ts-forward -m mark --mark 0x10000/0x10000 -j ACCEPT
filter/ts-forward -i tailscale0 -j DROP
filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -m comment --comment ChromeOS VM connectivity -j RETURN
filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
nat/POSTROUTING -j ts-postrouting
nat/ts-postrouting -m mark --mark 0x10000/0x10000 -j MASQUERADE
`,
},
{
name: "addr and routes with netfilter",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.104/10"),
Routes: mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
NetfilterMode: NetfilterOn,
},
want: `
up
ip addr add 100.101.102.104/10 dev tailscale0
ip route add 10.0.0.0/8 dev tailscale0 scope global
ip route add 100.100.100.100/32 dev tailscale0 scope global
ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
filter/FORWARD -j ts-forward
filter/INPUT -j ts-input
filter/ts-forward -m mark --mark 0x10000/0x10000 -j ACCEPT
filter/ts-forward -i tailscale0 -j DROP
filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -m comment --comment ChromeOS VM connectivity -j RETURN
filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
nat/POSTROUTING -j ts-postrouting
`,
},
{
name: "addr and routes and subnet routes with netfilter but no SNAT",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.104/10"),
Routes: mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
SubnetRoutes: mustCIDRs("200.0.0.0/8"),
SNATSubnetRoutes: false,
NetfilterMode: NetfilterOn,
},
want: `
up
ip addr add 100.101.102.104/10 dev tailscale0
ip route add 10.0.0.0/8 dev tailscale0 scope global
ip route add 100.100.100.100/32 dev tailscale0 scope global
ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
filter/FORWARD -j ts-forward
filter/INPUT -j ts-input
filter/ts-forward -o tailscale0 -s 200.0.0.0/8 -j ACCEPT
filter/ts-forward -i tailscale0 -d 200.0.0.0/8 -j MARK --set-mark 0x10000/0x10000
filter/ts-forward -m mark --mark 0x10000/0x10000 -j ACCEPT
filter/ts-forward -i tailscale0 -j DROP
filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -m comment --comment ChromeOS VM connectivity -j RETURN
filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
nat/POSTROUTING -j ts-postrouting
`,
},
{
name: "addr and routes with netfilter",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.104/10"),
Routes: mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
NetfilterMode: NetfilterOn,
},
want: `
up
ip addr add 100.101.102.104/10 dev tailscale0
ip route add 10.0.0.0/8 dev tailscale0 scope global
ip route add 100.100.100.100/32 dev tailscale0 scope global
ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
filter/FORWARD -j ts-forward
filter/INPUT -j ts-input
filter/ts-forward -m mark --mark 0x10000/0x10000 -j ACCEPT
filter/ts-forward -i tailscale0 -j DROP
filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -m comment --comment ChromeOS VM connectivity -j RETURN
filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
nat/POSTROUTING -j ts-postrouting
`,
},
{
name: "addr and routes with half netfilter",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.104/10"),
Routes: mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
NetfilterMode: NetfilterNoDivert,
},
want: `
up
ip addr add 100.101.102.104/10 dev tailscale0
ip route add 10.0.0.0/8 dev tailscale0 scope global
ip route add 100.100.100.100/32 dev tailscale0 scope global
ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
filter/ts-forward -m mark --mark 0x10000/0x10000 -j ACCEPT
filter/ts-forward -i tailscale0 -j DROP
filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -m comment --comment ChromeOS VM connectivity -j RETURN
filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
`,
},
{
name: "addr and routes with netfilter",
in: &Config{
LocalAddrs: mustCIDRs("100.101.102.104/10"),
Routes: mustCIDRs("100.100.100.100/32", "10.0.0.0/8"),
NetfilterMode: NetfilterOn,
},
want: `
up
ip addr add 100.101.102.104/10 dev tailscale0
ip route add 10.0.0.0/8 dev tailscale0 scope global
ip route add 100.100.100.100/32 dev tailscale0 scope global
ip rule add fwmark 0x20000/0x20000 priority 10000 table main suppress_ifgroup 10000
filter/FORWARD -j ts-forward
filter/INPUT -j ts-input
filter/ts-forward -m mark --mark 0x10000/0x10000 -j ACCEPT
filter/ts-forward -i tailscale0 -j DROP
filter/ts-input -i lo -s 100.101.102.104 -j ACCEPT
filter/ts-input ! -i tailscale0 -s 100.115.92.0/23 -m comment --comment ChromeOS VM connectivity -j RETURN
filter/ts-input ! -i tailscale0 -s 100.64.0.0/10 -j DROP
nat/POSTROUTING -j ts-postrouting
`,
},
}
fake := NewFakeOS(t)
router, err := newUserspaceRouterAdvanced(t.Logf, "tailscale0", fake, fake)
if err != nil {
t.Fatalf("failed to create router: %v", err)
}
if err := router.Up(); err != nil {
t.Fatalf("failed to up router: %v", err)
}
testState := func(t *testing.T, i int) {
t.Helper()
if err := router.Set(states[i].in); err != nil {
t.Fatalf("failed to set router config: %v", err)
}
got := fake.String()
want := strings.TrimSpace(states[i].want)
if diff := cmp.Diff(got, want); diff != "" {
t.Fatalf("unexpected OS state (-got+want):\n%s", diff)
}
}
for i, state := range states {
t.Run(state.name, func(t *testing.T) { testState(t, i) })
}
// Cycle through a bunch of states in pseudorandom order, to
// verify that we transition cleanly from state to state no matter
// the order.
for randRun := 0; randRun < 5*len(states); randRun++ {
i := rand.Intn(len(states))
state := states[i]
t.Run(state.name, func(t *testing.T) { testState(t, i) })
}
}
// fakeOS implements netfilterRunner and commandRunner, but captures
// changes without touching the OS.
type fakeOS struct {
t *testing.T
up bool
ips []string
routes []string
rules []string
netfilter map[string][]string
}
func NewFakeOS(t *testing.T) *fakeOS {
return &fakeOS{
t: t,
netfilter: map[string][]string{
"filter/INPUT": nil,
"filter/OUTPUT": nil,
"filter/FORWARD": nil,
"nat/PREROUTING": nil,
"nat/OUTPUT": nil,
"nat/POSTROUTING": nil,
},
}
}
var errExec = errors.New("execution failed")
func (o *fakeOS) String() string {
var b strings.Builder
if o.up {
b.WriteString("up\n")
} else {
b.WriteString("down\n")
}
for _, ip := range o.ips {
fmt.Fprintf(&b, "ip addr add %s\n", ip)
}
for _, route := range o.routes {
fmt.Fprintf(&b, "ip route add %s\n", route)
}
for _, rule := range o.rules {
fmt.Fprintf(&b, "ip rule add %s\n", rule)
}
var chains []string
for chain := range o.netfilter {
chains = append(chains, chain)
}
sort.Strings(chains)
for _, chain := range chains {
for _, rule := range o.netfilter[chain] {
fmt.Fprintf(&b, "%s %s\n", chain, rule)
}
}
return b.String()[:len(b.String())-1]
}
func (o *fakeOS) Insert(table, chain string, pos int, args ...string) error {
k := table + "/" + chain
if rules, ok := o.netfilter[k]; ok {
if pos > len(rules)+1 {
o.t.Errorf("bad position %d in %s", pos, k)
return errExec
}
rules = append(rules, "")
copy(rules[pos:], rules[pos-1:])
rules[pos-1] = strings.Join(args, " ")
o.netfilter[k] = rules
} else {
o.t.Errorf("unknown table/chain %s", k)
return errExec
}
return nil
}
func (o *fakeOS) Append(table, chain string, args ...string) error {
k := table + "/" + chain
return o.Insert(table, chain, len(o.netfilter[k])+1, args...)
}
func (o *fakeOS) Exists(table, chain string, args ...string) (bool, error) {
k := table + "/" + chain
if rules, ok := o.netfilter[k]; ok {
for _, rule := range rules {
if rule == strings.Join(args, " ") {
return true, nil
}
}
return false, nil
} else {
o.t.Errorf("unknown table/chain %s", k)
return false, errExec
}
}
func (o *fakeOS) Delete(table, chain string, args ...string) error {
k := table + "/" + chain
if rules, ok := o.netfilter[k]; ok {
for i, rule := range rules {
if rule == strings.Join(args, " ") {
rules = append(rules[:i], rules[i+1:]...)
o.netfilter[k] = rules
return nil
}
}
o.t.Errorf("delete of unknown rule %q from %s", strings.Join(args, " "), k)
return errExec
} else {
o.t.Errorf("unknown table/chain %s", k)
return errExec
}
}
func (o *fakeOS) ListChains(table string) (ret []string, err error) {
for chain := range o.netfilter {
pfx := table + "/"
if strings.HasPrefix(chain, pfx) {
ret = append(ret, chain[len(pfx):])
}
}
return ret, nil
}
func (o *fakeOS) ClearChain(table, chain string) error {
k := table + "/" + chain
if _, ok := o.netfilter[k]; ok {
o.netfilter[k] = nil
return nil
} else {
o.t.Errorf("unknown table/chain %s", k)
return errExec
}
}
func (o *fakeOS) NewChain(table, chain string) error {
k := table + "/" + chain
if _, ok := o.netfilter[k]; ok {
o.t.Errorf("table/chain %s already exists", k)
return errExec
}
o.netfilter[k] = nil
return nil
}
func (o *fakeOS) DeleteChain(table, chain string) error {
k := table + "/" + chain
if rules, ok := o.netfilter[k]; ok {
if len(rules) != 0 {
o.t.Errorf("%s is not empty", k)
return errExec
}
delete(o.netfilter, k)
return nil
} else {
o.t.Errorf("%s does not exist", k)
return errExec
}
}
func (o *fakeOS) run(args ...string) error {
unexpected := func() error {
o.t.Errorf("unexpected invocation %q", strings.Join(args, " "))
return errors.New("unrecognized invocation")
}
if args[0] != "ip" {
return unexpected()
}
rest := strings.Join(args[3:], " ")
var l *[]string
switch args[1] {
case "link":
got := strings.Join(args[2:], " ")
switch got {
case "set dev tailscale0 group 10000 up":
o.up = true
case "set dev tailscale0 group 0 down":
o.up = false
default:
return unexpected()
}
return nil
case "addr":
l = &o.ips
case "route":
l = &o.routes
case "rule":
l = &o.rules
default:
return unexpected()
}
switch args[2] {
case "add":
for _, el := range *l {
if el == rest {
o.t.Errorf("can't add %q, already present", rest)
return errors.New("already exists")
}
}
*l = append(*l, rest)
sort.Strings(*l)
case "del":
found := false
for i, el := range *l {
if el == rest {
found = true
*l = append((*l)[:i], (*l)[i+1:]...)
break
}
}
if !found {
o.t.Errorf("can't delete %q, not present", rest)
return errors.New("not present")
}
default:
return unexpected()
}
return nil
}
func (o *fakeOS) output(args ...string) ([]byte, error) {
want := "ip rule list priority 10000"
got := strings.Join(args, " ")
if got != want {
o.t.Errorf("unexpected command that wants output: %v", got)
return nil, errExec
}
var ret []string
for _, rule := range o.rules {
if strings.Contains(rule, "10000") {
ret = append(ret, rule)
}
}
return []byte(strings.Join(ret, "\n")), nil
}
Write Preview
Loading…
Cancel
Save