wgengine/router: support multiple levels of netfilter involvement.

Signed-off-by: David Anderson <danderson@tailscale.com>

cmd/tailscale/tailscale.go

@@ -54,10 +54,14 @@ func main() {
 	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
 	upf.BoolVar(&upArgs.noSingleRoutes, "no-single-routes", false, "don't install routes to single nodes")
 	upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
-	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. 10.0.0.0/8,192.168.0.0/24)")
 	upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "ACL tags to request (comma-separated, e.g. eng,montreal,ssh)")
-	upf.BoolVar(&upArgs.noSNAT, "no-snat", false, "disable SNAT of traffic to local routes advertised with -advertise-routes")
 	upf.StringVar(&upArgs.authKey, "authkey", "", "node authorization key")
+	if runtime.GOOS == "linux" {
+		upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. 10.0.0.0/8,192.168.0.0/24)")
+		upf.BoolVar(&upArgs.noSNAT, "no-snat", false, "disable SNAT of traffic to local routes advertised with -advertise-routes")
+		upf.BoolVar(&upArgs.noNetfilterCalls, "no-netfilter-calls", false, "don't call Tailscale netfilter chains from the main netfilter chains")
+		upf.BoolVar(&upArgs.noNetfilter, "no-netfilter", false, "disable all netfilter rule management")
+	}
 	upCmd := &ffcli.Command{
 		Name:       "up",
 		ShortUsage: "up [flags]",
@@ -100,14 +104,16 @@ change in the future.
 }
 
 var upArgs struct {
-	server          string
+	server           string
-	acceptRoutes    bool
+	acceptRoutes     bool
-	noSingleRoutes  bool
+	noSingleRoutes   bool
-	shieldsUp       bool
+	shieldsUp        bool
-	advertiseRoutes string
+	advertiseRoutes  string
-	advertiseTags   string
+	advertiseTags    string
-	noSNAT          bool
+	noSNAT           bool
-	authKey         string
+	noNetfilterCalls bool
+	noNetfilter      bool
+	authKey          string
 }
 
 // parseIPOrCIDR parses an IP address or a CIDR prefix. If the input
@@ -194,6 +200,8 @@ func runUp(ctx context.Context, args []string) error {
 	prefs.AdvertiseRoutes = routes
 	prefs.AdvertiseTags = tags
 	prefs.NoSNAT = upArgs.noSNAT
+	prefs.NoNetfilter = upArgs.noNetfilter
+	prefs.NoNetfilterCalls = upArgs.noNetfilterCalls
 
 	c, bc, ctx, cancel := connect(ctx)
 	defer cancel()

ipn/local.go

@@ -734,11 +734,19 @@ func routerConfig(cfg *wgcfg.Config, prefs *Prefs, dnsDomains []string) *router.
 	}
 
 	rs := &router.Config{
-		LocalAddrs:   wgCIDRToNetaddr(addrs),
+		LocalAddrs:       wgCIDRToNetaddr(addrs),
-		DNS:          wgIPToNetaddr(cfg.DNS),
+		DNS:              wgIPToNetaddr(cfg.DNS),
-		DNSDomains:   dnsDomains,
+		DNSDomains:       dnsDomains,
-		SubnetRoutes: wgCIDRToNetaddr(prefs.AdvertiseRoutes),
+		SubnetRoutes:     wgCIDRToNetaddr(prefs.AdvertiseRoutes),
-		NoSNAT:       prefs.NoSNAT,
+		SNATSubnetRoutes: !prefs.NoSNAT,
+	}
+	switch {
+	case prefs.NoNetfilter:
+		rs.NetfilterMode = router.NetfilterOff
+	case prefs.NoNetfilterCalls:
+		rs.NetfilterMode = router.NetfilterNoDivert
+	default:
+		rs.NetfilterMode = router.NetfilterOn
 	}
 
 	for _, peer := range cfg.Peers {

ipn/prefs.go

@@ -44,24 +44,12 @@ type Prefs struct {
 	// use the packet filter as provided. If true, we block incoming
 	// connections.
 	ShieldsUp bool
-	// AdvertiseRoutes specifies CIDR prefixes to advertise into the
-	// Tailscale network as reachable through the current node.
-	AdvertiseRoutes []wgcfg.CIDR
 	// AdvertiseTags specifies groups that this node wants to join, for
 	// purposes of ACL enforcement. These can be referenced from the ACL
 	// security policy. Note that advertising a tag doesn't guarantee that
 	// the control server will allow you to take on the rights for that
 	// tag.
 	AdvertiseTags []string
-	// NoSNAT specifies whether to source NAT traffic going to
-	// destinations in AdvertiseRoutes. The default is to apply source
-	// NAT, which makes the traffic appear to come from the router
-	// machine rather than the peer's Tailscale IP.
-	//
-	// Disabling SNAT requires additional manual configuration in your
-	// network to route Tailscale traffic back to the subnet relay
-	// machine.
-	NoSNAT bool
 
 	// NotepadURLs is a debugging setting that opens OAuth URLs in
 	// notepad.exe on Windows, rather than loading them in a browser.
@@ -74,6 +62,34 @@ type Prefs struct {
 	// DisableDERP prevents DERP from being used.
 	DisableDERP bool
 
+	// The following block of options only have an effect on Linux.
+
+	// AdvertiseRoutes specifies CIDR prefixes to advertise into the
+	// Tailscale network as reachable through the current
+	// node.
+	AdvertiseRoutes []wgcfg.CIDR
+	// NoSNAT specifies whether to source NAT traffic going to
+	// destinations in AdvertiseRoutes. The default is to apply source
+	// NAT, which makes the traffic appear to come from the router
+	// machine rather than the peer's Tailscale IP.
+	//
+	// Disabling SNAT requires additional manual configuration in your
+	// network to route Tailscale traffic back to the subnet relay
+	// machine.
+	//
+	// Linux-only.
+	NoSNAT bool
+	// NoNetfilter, if set, disables all management of firewall rules
+	// for Tailscale traffic. The resulting configuration is not
+	// secure, and it is the user's responsibility to correct that.
+	NoNetfilter bool
+	// NoNetfilterDivert, if set, disables calling Tailscale netfilter
+	// chains from the main netfilter chains, but still manages the
+	// contents of the Tailscale chains. The resulting configuration
+	// is not secure, and it is the user's responsibility to insert
+	// calls to Tailscale's chains at the right place.
+	NoNetfilterCalls bool
+
 	// The Persist field is named 'Config' in the file for backward
 	// compatibility with earlier versions.
 	// TODO(apenwarr): We should move this out of here, it's not a pref.
@@ -92,9 +108,9 @@ func (p *Prefs) Pretty() string {
 	} else {
 		pp = "Persist=nil"
 	}
-	return fmt.Sprintf("Prefs{ra=%v mesh=%v dns=%v want=%v notepad=%v derp=%v shields=%v routes=%v snat=%v %v}",
+	return fmt.Sprintf("Prefs{ra=%v mesh=%v dns=%v want=%v notepad=%v derp=%v shields=%v routes=%v snat=%v nf=%v nfd=%v %v}",
 		p.RouteAll, p.AllowSingleHosts, p.CorpDNS, p.WantRunning,
-		p.NotepadURLs, !p.DisableDERP, p.ShieldsUp, p.AdvertiseRoutes, !p.NoSNAT, pp)
+		p.NotepadURLs, !p.DisableDERP, p.ShieldsUp, p.AdvertiseRoutes, !p.NoSNAT, !p.NoNetfilter, !p.NoNetfilterCalls, pp)
 }
 
 func (p *Prefs) ToBytes() []byte {
@@ -123,6 +139,8 @@ func (p *Prefs) Equals(p2 *Prefs) bool {
 		p.DisableDERP == p2.DisableDERP &&
 		p.ShieldsUp == p2.ShieldsUp &&
 		p.NoSNAT == p2.NoSNAT &&
+		p.NoNetfilter == p2.NoNetfilter &&
+		p.NoNetfilterCalls == p2.NoNetfilterCalls &&
 		compareIPNets(p.AdvertiseRoutes, p2.AdvertiseRoutes) &&
 		compareStrings(p.AdvertiseTags, p2.AdvertiseTags) &&
 		p.Persist.Equals(p2.Persist)

ipn/prefs_test.go

@@ -23,7 +23,7 @@ func fieldsOf(t reflect.Type) (fields []string) {
 func TestPrefsEqual(t *testing.T) {
 	tstest.PanicOnLog()
 
-	prefsHandles := []string{"ControlURL", "RouteAll", "AllowSingleHosts", "CorpDNS", "WantRunning", "ShieldsUp", "AdvertiseRoutes", "AdvertiseTags", "NoSNAT", "NotepadURLs", "DisableDERP", "Persist"}
+	prefsHandles := []string{"ControlURL", "RouteAll", "AllowSingleHosts", "CorpDNS", "WantRunning", "ShieldsUp", "AdvertiseTags", "NotepadURLs", "DisableDERP", "AdvertiseRoutes", "NoSNAT", "NoNetfilter", "NoNetfilterCalls", "Persist"}
 	if have := fieldsOf(reflect.TypeOf(Prefs{})); !reflect.DeepEqual(have, prefsHandles) {
 		t.Errorf("Prefs.Equal check might be out of sync\nfields: %q\nhandled: %q\n",
 			have, prefsHandles)
@@ -173,6 +173,28 @@ func TestPrefsEqual(t *testing.T) {
 			true,
 		},
 
+		{
+			&Prefs{NoNetfilter: true},
+			&Prefs{NoNetfilter: false},
+			false,
+		},
+		{
+			&Prefs{NoNetfilter: true},
+			&Prefs{NoNetfilter: true},
+			true,
+		},
+
+		{
+			&Prefs{NoNetfilterCalls: true},
+			&Prefs{NoNetfilterCalls: false},
+			false,
+		},
+		{
+			&Prefs{NoNetfilterCalls: true},
+			&Prefs{NoNetfilterCalls: true},
+			true,
+		},
+
 		{
 			&Prefs{Persist: &controlclient.Persist{}},
 			&Prefs{Persist: &controlclient.Persist{LoginName: "dave"}},

wgengine/router/router.go

@@ -35,22 +35,30 @@ func New(logf logger.Logf, wgdev *device.Device, tundev tun.Device) (Router, err
 	return newUserspaceRouter(logf, wgdev, tundev)
 }
 
+type NetfilterMode int
+
+const (
+	NetfilterOff      NetfilterMode = iota // remove all tailscale netfilter state
+	NetfilterNoDivert                      // manage tailscale chains, but don't call them
+	NetfilterOn                            // manage tailscale chains and call them from main chains
+)
+
 // Config is the subset of Tailscale configuration that is relevant to
 // the OS's network stack.
 type Config struct {
-	LocalAddrs   []netaddr.IPPrefix
+	LocalAddrs []netaddr.IPPrefix
-	DNS          []netaddr.IP
+	DNS        []netaddr.IP
-	DNSDomains   []string
+	DNSDomains []string
-	Routes       []netaddr.IPPrefix // routes to point into the Tailscale interface
+	Routes     []netaddr.IPPrefix // routes to point into the Tailscale interface
-	SubnetRoutes []netaddr.IPPrefix // subnets being advertised to other Tailscale nodes
+
-	NoSNAT       bool               // don't SNAT traffic to local subnets
+	// Linux-only things below, ignored on other platforms.
+
+	SubnetRoutes     []netaddr.IPPrefix // subnets being advertised to other Tailscale nodes
+	SNATSubnetRoutes bool               // SNAT traffic to local subnets
+	NetfilterMode    NetfilterMode      // how much to manage netfilter rules
 }
 
 // shutdownConfig is a routing configuration that removes all router
 // state from the OS. It's the config used when callers pass in a nil
 // Config.
-var shutdownConfig = Config{
+var shutdownConfig = Config{}
-	// TODO(danderson): set more things in here to disable all
-	// firewall rules and routing overrides when nil.
-	NoSNAT: true,
-}

wgengine/router/router_linux.go

@@ -48,13 +48,19 @@ const (
 	tailscaleBypassMark = "0x20000/0x20000"
 )
 
+// chromeOSVMRange is the subset of the CGNAT IPv4 range used by
+// ChromeOS to interconnect the host OS to containers and VMs. We
+// avoid allocating Tailscale IPs from it, to avoid conflicts.
+const chromeOSVMRange = "100.115.92.0/23"
+
 type linuxRouter struct {
-	logf         func(fmt string, args ...interface{})
+	logf             func(fmt string, args ...interface{})
-	tunname      string
+	tunname          string
-	addrs        map[netaddr.IPPrefix]bool
+	addrs            map[netaddr.IPPrefix]bool
-	routes       map[netaddr.IPPrefix]bool
+	routes           map[netaddr.IPPrefix]bool
-	subnetRoutes map[netaddr.IPPrefix]bool
+	subnetRoutes     map[netaddr.IPPrefix]bool
-	noSNAT       bool
+	snatSubnetRoutes bool
+	netfilterMode    NetfilterMode
 
 	ipt4 *iptables.IPTables
 }
@@ -71,10 +77,10 @@ func newUserspaceRouter(logf logger.Logf, _ *device.Device, tunDev tun.Device) (
 	}
 
 	return &linuxRouter{
-		logf:    logf,
+		logf:          logf,
-		tunname: tunname,
+		tunname:       tunname,
-		noSNAT:  true,
+		netfilterMode: NetfilterOff,
-		ipt4:    ipt4,
+		ipt4:          ipt4,
 	}, nil
 }
 
@@ -92,12 +98,16 @@ func cmd(args ...string) error {
 }
 
 func (r *linuxRouter) Up() error {
-	if err := r.deleteLegacyNetfilter(); err != nil {
+	if err := r.delLegacyNetfilter(); err != nil {
 		return err
 	}
-	if err := r.addBaseNetfilter4(); err != nil {
+	if err := r.delNetfilterHooks(); err != nil {
 		return err
 	}
+	if err := r.delNetfilterBase(); err != nil {
+		return err
+	}
+
 	if err := r.addBypassRule(); err != nil {
 		return err
 	}
@@ -115,7 +125,10 @@ func (r *linuxRouter) down() error {
 	if err := r.delBypassRule(); err != nil {
 		return err
 	}
-	if err := r.delNetfilter4(); err != nil {
+	if err := r.delNetfilterHooks(); err != nil {
+		return err
+	}
+	if err := r.delNetfilterBase(); err != nil {
 		return err
 	}
 
@@ -146,84 +159,129 @@ func (r *linuxRouter) Set(cfg *Config) error {
 		cfg = &shutdownConfig
 	}
 
-	// cidrDiff calls add and del as needed to make the set of prefixes in
+	if err := r.setNetfilterMode(cfg.NetfilterMode); err != nil {
-	// old and new match. Returns a map version of new, and the first
+		return err
-	// error encountered while reconfiguring, if any.
-	cidrDiff := func(kind string, old map[netaddr.IPPrefix]bool, new []netaddr.IPPrefix, add, del func(netaddr.IPPrefix) error) (map[netaddr.IPPrefix]bool, error) {
-		var (
-			ret  = make(map[netaddr.IPPrefix]bool, len(new))
-			errq error
-		)
-
-		for _, cidr := range new {
-			ret[cidr] = true
-		}
-		for cidr := range old {
-			if ret[cidr] {
-				continue
-			}
-			if err := del(cidr); err != nil {
-				r.logf("%s del failed: %v", kind, err)
-				if errq == nil {
-					errq = err
-				}
-			}
-		}
-		for cidr := range ret {
-			if old[cidr] {
-				continue
-			}
-			if err := add(cidr); err != nil {
-				r.logf("%s add failed: %v", kind, err)
-				if errq == nil {
-					errq = err
-				}
-			}
-		}
-
-		return ret, errq
 	}
 
-	var errq error
+	newAddrs, err := cidrDiff("addr", r.addrs, cfg.LocalAddrs, r.addAddress, r.delAddress, r.logf)
-
+	if err != nil {
-	newAddrs, err := cidrDiff("addr", r.addrs, cfg.LocalAddrs, r.addAddress, r.delAddress)
+		return err
-	if err != nil && errq == nil {
-		errq = err
 	}
-	newRoutes, err := cidrDiff("route", r.routes, cfg.Routes, r.addRoute, r.delRoute)
+	r.addrs = newAddrs
-	if err != nil && errq == nil {
+
-		errq = err
+	newRoutes, err := cidrDiff("route", r.routes, cfg.Routes, r.addRoute, r.delRoute, r.logf)
+	if err != nil {
+		return err
 	}
-	newSubnetRoutes, err := cidrDiff("subnet rule", r.subnetRoutes, cfg.SubnetRoutes, r.addSubnetRule, r.delSubnetRule)
+	r.routes = newRoutes
-	if err != nil && errq == nil {
+
-		errq = err
+	newSubnetRoutes, err := cidrDiff("subnet rule", r.subnetRoutes, cfg.SubnetRoutes, r.addSubnetRule, r.delSubnetRule, r.logf)
+	if err != nil {
+		return err
 	}
+	r.subnetRoutes = newSubnetRoutes
 
 	switch {
-	case cfg.NoSNAT == r.noSNAT:
+	case cfg.SNATSubnetRoutes == r.snatSubnetRoutes:
 		// state already correct, nothing to do.
-	case cfg.NoSNAT:
+	case cfg.SNATSubnetRoutes:
-		if err := r.delSNATRule(); err != nil && errq == nil {
+		if err := r.addSNATRule(); err != nil {
-			errq = err
+			return err
 		}
 	default:
-		if err := r.addSNATRule(); err != nil && errq == nil {
+		if err := r.delSNATRule(); err != nil {
-			errq = err
+			return err
 		}
 	}
-
+	r.snatSubnetRoutes = cfg.SNATSubnetRoutes
-	r.addrs = newAddrs
-	r.routes = newRoutes
-	r.subnetRoutes = newSubnetRoutes
-	r.noSNAT = cfg.NoSNAT
 
 	// TODO: this:
 	if false {
 		if err := r.replaceResolvConf(cfg.DNS, cfg.DNSDomains); err != nil {
-			errq = fmt.Errorf("replacing resolv.conf failed: %v", err)
+			return fmt.Errorf("replacing resolv.conf failed: %v", err)
 		}
 	}
-	return errq
+	return nil
+}
+
+// setNetfilterMode switches the router to the given netfilter
+// mode. Netfilter state is created or deleted appropriately to
+// reflect the new mode, and r.snatSubnetRoutes is updated to reflect
+// the current state of subnet SNATing.
+func (r *linuxRouter) setNetfilterMode(mode NetfilterMode) error {
+	if r.netfilterMode == mode {
+		return nil
+	}
+
+	// Depending on the netfilter mode we switch from and to, we may
+	// have created the Tailscale netfilter chains. If so, we have to
+	// go back through existing router state, and add the netfilter
+	// rules for that state.
+	//
+	// This bool keeps track of whether the current state transition
+	// is one that requires adding rules of existing state.
+	reprocess := false
+
+	switch mode {
+	case NetfilterOff:
+		if err := r.delNetfilterHooks(); err != nil {
+			return err
+		}
+		if err := r.delNetfilterBase(); err != nil {
+			return err
+		}
+		r.snatSubnetRoutes = false
+	case NetfilterNoDivert:
+		switch r.netfilterMode {
+		case NetfilterOff:
+			reprocess = true
+			if err := r.addNetfilterBase(); err != nil {
+				return err
+			}
+			r.snatSubnetRoutes = false
+		case NetfilterOn:
+			if err := r.delNetfilterHooks(); err != nil {
+				return err
+			}
+		}
+	case NetfilterOn:
+		switch r.netfilterMode {
+		case NetfilterOff:
+			reprocess = true
+			if err := r.addNetfilterBase(); err != nil {
+				return err
+			}
+			if err := r.addNetfilterHooks(); err != nil {
+				return err
+			}
+			r.snatSubnetRoutes = false
+		case NetfilterNoDivert:
+			if err := r.addNetfilterHooks(); err != nil {
+				return err
+			}
+		}
+	default:
+		panic("unhandled netfilter mode")
+	}
+
+	r.netfilterMode = mode
+
+	if !reprocess {
+		return nil
+	}
+
+	for cidr := range r.addrs {
+		if err := r.addLoopbackRule(cidr.IP); err != nil {
+			return err
+		}
+	}
+	for cidr := range r.subnetRoutes {
+		if err := r.addSubnetRule(cidr); err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 const (
@@ -322,38 +380,54 @@ func (r *linuxRouter) restoreResolvConf() error {
 	return nil
 }
 
-// addAddress adds an IP/mask to the tunnel interface, and firewall
+// addAddress adds an IP/mask to the tunnel interface. Fails if the
-// rules to permit loopback traffic. Fails if the address is already
+// address is already assigned to the interface, or if the addition
-// assigned to the interface, or if the addition fails.
+// fails.
 func (r *linuxRouter) addAddress(addr netaddr.IPPrefix) error {
 	if err := cmd("ip", "addr", "add", addr.String(), "dev", r.tunname); err != nil {
-		return err
+		return fmt.Errorf("adding address %q to tunnel interface: %v", addr, err)
 	}
-	if err := r.ipt4.Insert("filter", "ts-input", 1, "-i", "lo", "-s", addr.IP.String(), "-j", "ACCEPT"); err != nil {
+	if err := r.addLoopbackRule(addr.IP); err != nil {
 		return err
 	}
 	return nil
 }
 
-// delAddress removes an IP/mask from the tunnel interface, and
+// delAddress removes an IP/mask from the tunnel interface. Fails if
-// firewall rules permitting loopback traffic. Fails if the address is
+// the address is not assigned to the interface, or if the removal
-// not assigned to the interface, or if the removal fails.
+// fails.
 func (r *linuxRouter) delAddress(addr netaddr.IPPrefix) error {
-	if err := r.ipt4.Delete("filter", "ts-input", "-i", "lo", "-s", addr.IP.String(), "-j", "ACCEPT"); err != nil {
+	if err := r.delLoopbackRule(addr.IP); err != nil {
 		return err
 	}
 	if err := cmd("ip", "addr", "del", addr.String(), "dev", r.tunname); err != nil {
-		return err
+		return fmt.Errorf("deleting address %q from tunnel interface: %v", addr, err)
 	}
 	return nil
 }
 
-// normalizeCIDR returns cidr as an ip/mask string, with the host bits
+// addLoopbackRule adds a firewall rule to permit loopback traffic to
-// of the IP address zeroed out.
+// a local Tailscale IP.
-func normalizeCIDR(cidr netaddr.IPPrefix) string {
+func (r *linuxRouter) addLoopbackRule(addr netaddr.IP) error {
-	ncidr := cidr.IPNet()
+	if r.netfilterMode == NetfilterOff {
-	nip := ncidr.IP.Mask(ncidr.Mask)
+		return nil
-	return fmt.Sprintf("%s/%d", nip, cidr.Bits)
+	}
+	if err := r.ipt4.Insert("filter", "ts-input", 1, "-i", "lo", "-s", addr.String(), "-j", "ACCEPT"); err != nil {
+		return fmt.Errorf("adding loopback allow rule for %q: %v", addr, err)
+	}
+	return nil
+}
+
+// delLoopbackRule removes the firewall rule permitting loopback
+// traffic to a Tailscale IP.
+func (r *linuxRouter) delLoopbackRule(addr netaddr.IP) error {
+	if r.netfilterMode == NetfilterOff {
+		return nil
+	}
+	if err := r.ipt4.Delete("filter", "ts-input", "-i", "lo", "-s", addr.String(), "-j", "ACCEPT"); err != nil {
+		return fmt.Errorf("deleting loopback allow rule for %q: %v", addr, err)
+	}
+	return nil
 }
 
 // addRoute adds a route for cidr, pointing to the tunnel
@@ -371,9 +445,12 @@ func (r *linuxRouter) delRoute(cidr netaddr.IPPrefix) error {
 }
 
 // addSubnetRule adds a netfilter rule that allows traffic to flow
-// from Tailscale to cidr. Fails if the rule already exists, or if
+// from Tailscale to cidr.
-// adding the route fails.
 func (r *linuxRouter) addSubnetRule(cidr netaddr.IPPrefix) error {
+	if r.netfilterMode == NetfilterOff {
+		return nil
+	}
+
 	if err := r.ipt4.Insert("filter", "ts-forward", 1, "-i", r.tunname, "-d", normalizeCIDR(cidr), "-j", "MARK", "--set-mark", tailscaleSubnetRouteMark); err != nil {
 		return fmt.Errorf("adding subnet mark rule for %q: %v", cidr, err)
 	}
@@ -384,9 +461,13 @@ func (r *linuxRouter) addSubnetRule(cidr netaddr.IPPrefix) error {
 }
 
 // delSubnetRule deletes the netfilter subnet forwarding rule for
-// cidr. Fails if the rule doesn't exist, or if removing the rule
+// cidr. Fails if the rule doesn't exist, or if removing the route
 // fails.
 func (r *linuxRouter) delSubnetRule(cidr netaddr.IPPrefix) error {
+	if r.netfilterMode == NetfilterOff {
+		return nil
+	}
+
 	if err := r.ipt4.Delete("filter", "ts-forward", "-i", r.tunname, "-d", normalizeCIDR(cidr), "-j", "MARK", "--set-mark", tailscaleSubnetRouteMark); err != nil {
 		return fmt.Errorf("deleting subnet mark rule for %q: %v", cidr, err)
 	}
@@ -445,92 +526,121 @@ func (r *linuxRouter) delBypassRule() error {
 	return cmd("ip", "rule", "del", "priority", "10000")
 }
 
-// deleteLegacyNetfilter removes the netfilter rules installed by
+// addNetfilterBase adds custom Tailscale chains to netfilter, along
-// older versions of Tailscale, if they exist.
+// with some basic processing rules to be supplemented by later calls
-func (r *linuxRouter) deleteLegacyNetfilter() error {
+// to other helpers.
-	del := func(table, chain string, args ...string) error {
+func (r *linuxRouter) addNetfilterBase() error {
-		exists, err := r.ipt4.Exists(table, chain, args...)
+	create := func(table, chain string) error {
+		chains, err := r.ipt4.ListChains(table)
 		if err != nil {
-			return fmt.Errorf("checking for %v in %s/%s: %v", args, table, chain, err)
+			return fmt.Errorf("listing iptables chains: %v", err)
 		}
-		if exists {
+		found := false
-			if err := r.ipt4.Delete(table, chain, args...); err != nil {
+		for _, ch := range chains {
-				return fmt.Errorf("deleting %v in %s/%s: %v", args, table, chain, err)
+			if ch == chain {
+				found = true
+				break
 			}
 		}
+		if found {
+			err = r.ipt4.ClearChain(table, chain)
+		} else {
+			err = r.ipt4.NewChain(table, chain)
+		}
+		if err != nil {
+			return fmt.Errorf("setting up %s/%s: %v", table, chain, err)
+		}
 		return nil
 	}
-
+	if err := create("filter", "ts-input"); err != nil {
-	if err := del("filter", "FORWARD", "-m", "comment", "--comment", "tailscale", "-i", r.tunname, "-j", "ACCEPT"); err != nil {
 		return err
 	}
-	if err := del("nat", "POSTROUTING", "-m", "comment", "--comment", "tailscale", "-o", "eth0", "-j", "MASQUERADE"); err != nil {
+	if err := create("filter", "ts-forward"); err != nil {
+		return err
+	}
+	if err := create("nat", "ts-postrouting"); err != nil {
 		return err
 	}
 
+	// Only allow CGNAT range traffic to come from tailscale0. There
+	// is an exception carved out for ranges used by ChromeOS, for
+	// which we fall out of the Tailscale chain.
+	//
+	// Note, this will definitely break nodes that end up using the
+	// CGNAT range for other purposes :(.
+	args := []string{"!", "-i", r.tunname, "-s", chromeOSVMRange, "-m", "comment", "--comment", "ChromeOS VM connectivity", "-j", "RETURN"}
+	if err := r.ipt4.Append("filter", "ts-input", args...); err != nil {
+		return fmt.Errorf("adding %v in filter/ts-input: %v", args, err)
+	}
+	args = []string{"!", "-i", r.tunname, "-s", "100.64.0.0/10", "-j", "DROP"}
+	if err := r.ipt4.Append("filter", "ts-input", args...); err != nil {
+		return fmt.Errorf("adding %v in filter/ts-input: %v", args, err)
+	}
+
+	// Forward and mark packets that have the Tailscale subnet route
+	// bit set. The bit gets set by rules inserted into filter/FORWARD
+	// later on. We use packet marks here so both filter/FORWARD and
+	// nat/POSTROUTING can match on these packets of interest.
+	//
+	// In particular, we only want to apply SNAT rules in
+	// nat/POSTROUTING to packets that originated from the Tailscale
+	// interface, but we can't match on the inbound interface in
+	// POSTROUTING. So instead, we match on the inbound interface and
+	// destination IP in filter/FORWARD, and set a packet mark that
+	// nat/POSTROUTING can use to effectively run that same test
+	// again.
+	args = []string{"-m", "mark", "--mark", tailscaleSubnetRouteMark, "-j", "ACCEPT"}
+	if err := r.ipt4.Append("filter", "ts-forward", args...); err != nil {
+		return fmt.Errorf("adding %v in filter/ts-forward: %v", args, err)
+	}
+	args = []string{"-i", r.tunname, "-j", "DROP"}
+	if err := r.ipt4.Append("filter", "ts-forward", args...); err != nil {
+		return fmt.Errorf("adding %v in filter/ts-forward: %v", args, err)
+	}
+
 	return nil
 }
 
-// deleteNetfilter4 removes custom Tailscale chains and processing
+// delNetfilterBase removes custom Tailscale chains from netfilter.
-// hooks from netfilter.
+func (r *linuxRouter) delNetfilterBase() error {
-func (r *linuxRouter) delNetfilter4() error {
 	del := func(table, chain string) error {
-		tsChain := "ts-" + strings.ToLower(chain)
-
-		args := []string{"-j", tsChain}
-		exists, err := r.ipt4.Exists(table, chain, args...)
-		if err != nil {
-			return fmt.Errorf("checking for %v in %s/%s: %v", args, table, chain, err)
-		}
-		if exists {
-			if err := r.ipt4.Delete(table, chain, "-j", tsChain); err != nil {
-				return fmt.Errorf("deleting %v in %s/%s: %v", args, table, chain, err)
-			}
-		}
-
 		chains, err := r.ipt4.ListChains(table)
 		if err != nil {
 			return fmt.Errorf("listing iptables chains: %v", err)
 		}
-		for _, chain := range chains {
+		for _, ch := range chains {
-			if chain == tsChain {
+			if ch == chain {
-				if err := r.ipt4.DeleteChain(table, tsChain); err != nil {
+				if err := r.ipt4.ClearChain(table, chain); err != nil {
-					return fmt.Errorf("deleting %s/%s: %v", table, tsChain, err)
+					return fmt.Errorf("flushing %s/%s: %v", table, chain, err)
 				}
-				break
+				if err := r.ipt4.DeleteChain(table, chain); err != nil {
+					return fmt.Errorf("deleting %s/%s: %v", table, chain, err)
+				}
+				return nil
 			}
 		}
-
 		return nil
 	}
 
-	if err := del("filter", "INPUT"); err != nil {
+	if err := del("filter", "ts-input"); err != nil {
 		return err
 	}
-	if err := del("filter", "FORWARD"); err != nil {
+	if err := del("filter", "ts-forward"); err != nil {
 		return err
 	}
-	if err := del("nat", "POSTROUTING"); err != nil {
+	if err := del("nat", "ts-postrouting"); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-// chromeOSVMRange is the subset of the CGNAT IPv4 range used by
+// addNetfilterHooks inserts calls to tailscale's netfilter chains in
-// ChromeOS to interconnect the host OS to containers and VMs. We
+// the relevant main netfilter chains. The tailscale chains must
-// avoid allocating Tailscale IPs from it, to avoid conflicts.
+// already exist.
-const chromeOSVMRange = "100.115.92.0/23"
+func (r *linuxRouter) addNetfilterHooks() error {
-
-// addBaseNetfilter4 installs the basic IPv4 netfilter framework for
-// Tailscale, in preparation for inserting more rules later.
-func (r *linuxRouter) addBaseNetfilter4() error {
-	// Create our own filtering chains, and hook them into the head of
-	// various main tables. If the hooks already exist, we don't try
-	// to fight for first place, because other software does the
-	// same. We're happy with "someplace up before most other stuff".
 	divert := func(table, chain string) error {
-		tsChain := "ts-" + strings.ToLower(chain)
+		tsChain := tsChain(chain)
 
 		chains, err := r.ipt4.ListChains(table)
 		if err != nil {
@@ -543,13 +653,8 @@ func (r *linuxRouter) addBaseNetfilter4() error {
 				break
 			}
 		}
-		if found {
+		if !found {
-			err = r.ipt4.ClearChain(table, tsChain)
+			return fmt.Errorf("chain %q does not exist, cannot divert to it", tsChain)
-		} else {
-			err = r.ipt4.NewChain(table, tsChain)
-		}
-		if err != nil {
-			return fmt.Errorf("setting up %s/%s: %v", table, tsChain, err)
 		}
 
 		args := []string{"-j", tsChain}
@@ -565,6 +670,7 @@ func (r *linuxRouter) addBaseNetfilter4() error {
 		}
 		return nil
 	}
+
 	if err := divert("filter", "INPUT"); err != nil {
 		return err
 	}
@@ -574,49 +680,48 @@ func (r *linuxRouter) addBaseNetfilter4() error {
 	if err := divert("nat", "POSTROUTING"); err != nil {
 		return err
 	}
+	return nil
+}
 
-	// Only allow CGNAT range traffic to come from tailscale0. There
+// delNetfilterHooks deletes the calls to tailscale's netfilter chains
-	// is an exception carved out for ranges used by ChromeOS, for
+// in the relevant main netfilter chains.
-	// which we fall out of the Tailscale chain.
+func (r *linuxRouter) delNetfilterHooks() error {
-	//
+	del := func(table, chain string) error {
-	// Note, this will definitely break nodes that end up using the
+		tsChain := tsChain(chain)
-	// CGNAT range for other purposes :(.
+
-	args := []string{"!", "-i", r.tunname, "-s", chromeOSVMRange, "-m", "comment", "--comment", "ChromeOS VM connectivity", "-j", "RETURN"}
+		args := []string{"-j", tsChain}
-	if err := r.ipt4.Append("filter", "ts-input", args...); err != nil {
+		exists, err := r.ipt4.Exists(table, chain, args...)
-		return fmt.Errorf("adding %v in filter/ts-input: %v", args, err)
+		if err != nil {
-	}
+			return fmt.Errorf("checking for %v in %s/%s: %v", args, table, chain, err)
-	args = []string{"!", "-i", r.tunname, "-s", "100.64.0.0/10", "-j", "DROP"}
+		}
-	if err := r.ipt4.Append("filter", "ts-input", args...); err != nil {
+		if !exists {
-		return fmt.Errorf("adding %v in filter/ts-input: %v", args, err)
+			return nil
+		}
+		if err := r.ipt4.Delete(table, chain, args...); err != nil {
+			return fmt.Errorf("deleting %v in %s/%s: %v", args, table, chain, err)
+		}
+		return nil
 	}
 
-	// Forward and mark packets that have the Tailscale subnet route
+	if err := del("filter", "INPUT"); err != nil {
-	// bit set. The bit gets set by rules inserted into filter/FORWARD
+		return err
-	// later on. We use packet marks here so both filter/FORWARD and
-	// nat/POSTROUTING can match on these packets of interest.
-	//
-	// In particular, we only want to apply SNAT rules in
-	// nat/POSTROUTING to packets that originated from the Tailscale
-	// interface, but we can't match on the inbound interface in
-	// POSTROUTING. So instead, we match on the inbound interface and
-	// destination IP in filter/FORWARD, and set a packet mark that
-	// nat/POSTROUTING can use to effectively run that same test
-	// again.
-	args = []string{"-m", "mark", "--mark", tailscaleSubnetRouteMark, "-j", "ACCEPT"}
-	if err := r.ipt4.Append("filter", "ts-forward", args...); err != nil {
-		return fmt.Errorf("adding %v in filter/ts-forward: %v", args, err)
 	}
-	args = []string{"-i", r.tunname, "-j", "DROP"}
+	if err := del("filter", "FORWARD"); err != nil {
-	if err := r.ipt4.Append("filter", "ts-forward", args...); err != nil {
+		return err
-		return fmt.Errorf("adding %v in filter/ts-forward: %v", args, err)
+	}
+	if err := del("nat", "POSTROUTING"); err != nil {
+		return err
 	}
-
 	return nil
 }
 
 // addSNATRule adds a netfilter rule to SNAT traffic destined for
 // local subnets.
 func (r *linuxRouter) addSNATRule() error {
+	if r.netfilterMode == NetfilterOff {
+		return nil
+	}
+
 	args := []string{"-m", "mark", "--mark", tailscaleSubnetRouteMark, "-j", "MASQUERADE"}
 	if err := r.ipt4.Append("nat", "ts-postrouting", args...); err != nil {
 		return fmt.Errorf("adding %v in nat/ts-postrouting: %v", args, err)
@@ -627,9 +732,93 @@ func (r *linuxRouter) addSNATRule() error {
 // delSNATRule removes the netfilter rule to SNAT traffic destined for
 // local subnets. Fails if the rule does not exist.
 func (r *linuxRouter) delSNATRule() error {
+	if r.netfilterMode == NetfilterOff {
+		return nil
+	}
+
 	args := []string{"-m", "mark", "--mark", tailscaleSubnetRouteMark, "-j", "MASQUERADE"}
 	if err := r.ipt4.Delete("nat", "ts-postrouting", args...); err != nil {
 		return fmt.Errorf("deleting %v in nat/ts-postrouting: %v", args, err)
 	}
 	return nil
 }
+
+func (r *linuxRouter) delLegacyNetfilter() error {
+	del := func(table, chain string, args ...string) error {
+		exists, err := r.ipt4.Exists(table, chain, args...)
+		if err != nil {
+			return fmt.Errorf("checking for %v in %s/%s: %v", args, table, chain, err)
+		}
+		if exists {
+			if err := r.ipt4.Delete(table, chain, args...); err != nil {
+				return fmt.Errorf("deleting %v in %s/%s: %v", args, table, chain, err)
+			}
+		}
+		return nil
+	}
+
+	if err := del("filter", "FORWARD", "-m", "comment", "--comment", "tailscale", "-i", r.tunname, "-j", "ACCEPT"); err != nil {
+		return err
+	}
+	if err := del("nat", "POSTROUTING", "-m", "comment", "--comment", "tailscale", "-o", "eth0", "-j", "MASQUERADE"); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// cidrDiff calls add and del as needed to make the set of prefixes in
+// old and new match. Returns a map reflecting the actual new state
+// (which may be somewhere in between old and new if some commands
+// failed), and any error encountered while reconfiguring.
+func cidrDiff(kind string, old map[netaddr.IPPrefix]bool, new []netaddr.IPPrefix, add, del func(netaddr.IPPrefix) error, logf logger.Logf) (map[netaddr.IPPrefix]bool, error) {
+	newMap := make(map[netaddr.IPPrefix]bool, len(new))
+	for _, cidr := range new {
+		newMap[cidr] = true
+	}
+
+	// ret starts out as a copy of old, and updates as we
+	// add/delete. That way we can always return it and have it be the
+	// true state of what we've done so far.
+	ret := make(map[netaddr.IPPrefix]bool, len(old))
+	for cidr := range old {
+		ret[cidr] = true
+	}
+
+	for cidr := range old {
+		if newMap[cidr] {
+			continue
+		}
+		if err := del(cidr); err != nil {
+			logf("%s del failed: %v", kind, err)
+			return ret, err
+		}
+		delete(ret, cidr)
+	}
+	for cidr := range newMap {
+		if old[cidr] {
+			continue
+		}
+		if err := add(cidr); err != nil {
+			logf("%s add failed: %v", kind, err)
+			return ret, err
+		}
+		ret[cidr] = true
+	}
+
+	return ret, nil
+}
+
+// tsChain returns the name of the tailscale sub-chain corresponding
+// to the given "parent" chain (e.g. INPUT, FORWARD, ...).
+func tsChain(chain string) string {
+	return "ts-" + strings.ToLower(chain)
+}
+
+// normalizeCIDR returns cidr as an ip/mask string, with the host bits
+// of the IP address zeroed out.
+func normalizeCIDR(cidr netaddr.IPPrefix) string {
+	ncidr := cidr.IPNet()
+	nip := ncidr.IP.Mask(ncidr.Mask)
+	return fmt.Sprintf("%s/%d", nip, cidr.Bits)
+}