diff --git a/wgengine/router/router_linux.go b/wgengine/router/router_linux.go index 3a95c00ca..c2cf32889 100644 --- a/wgengine/router/router_linux.go +++ b/wgengine/router/router_linux.go @@ -105,6 +105,9 @@ type linuxRouter struct { v6Available bool v6NATAvailable bool + // ipPolicyPrefBase is the base priority at which ip rules are installed. + ipPolicyPrefBase int + ipt4 netfilterRunner ipt6 netfilterRunner cmd commandRunner @@ -163,6 +166,7 @@ func newUserspaceRouterAdvanced(logf logger.Logf, tunname string, linkMon *monit cmd: cmd, ipRuleFixLimiter: rate.NewLimiter(rate.Every(5*time.Second), 10), + ipPolicyPrefBase: 5200, } if r.useIPCommand() { r.ipRuleAvailable = (cmd.run("ip", "rule") == nil) @@ -176,6 +180,29 @@ func newUserspaceRouterAdvanced(logf logger.Logf, tunname string, linkMon *monit } } + // A common installation of OpenWRT involves use of the 'mwan3' package. + // This package installs ip-tables rules like: + // -A mwan3_fallback_policy -m mark --mark 0x0/0x3f00 -j MARK --set-xmark 0x100/0x3f00 + // + // which coupled with an ip rule: + // 2001: from all fwmark 0x100/0x3f00 lookup 1 + // + // has the effect of gobbling tailscale packets, because tailscale by default installs + // its policy routing rules at priority 52xx. + // + // As such, if we are running on openWRT, detect a mwan3 config, AND detect a rule + // with a preference 2001 (corresponding to the first interface wman3 manages), we + // shift the priority of our policies to 13xx. This effectively puts us betwen mwan3's + // permit-by-src-ip rules and mwan3 lookup of its own routing table which would drop + // the packet. + isMWAN3, err := checkOpenWRTUsingMWAN3() + if err != nil { + r.logf("error checking mwan3 installation: %v", err) + } else if isMWAN3 { + r.ipPolicyPrefBase = 1300 + r.logf("mwan3 on openWRT detected, switching policy base priority to 1300") + } + return r, nil } @@ -219,7 +246,7 @@ func (r *linuxRouter) useIPCommand() bool { // about the priority number. We could just do this in response to any netlink // change. Filtering by known priority ranges cuts back on some logspam. func (r *linuxRouter) onIPRuleDeleted(table uint8, priority uint32) { - if priority < 5200 || priority >= 5300 { + if int(priority) < r.ipPolicyPrefBase || int(priority) >= (r.ipPolicyPrefBase+100) { // Not our rule. return } @@ -870,6 +897,8 @@ var ( ) // ipRules are the policy routing rules that Tailscale uses. +// The priority is the value represented here added to r.ipPolicyPrefBase, +// which is usually 5200. // // NOTE(apenwarr): We leave spaces between each pref number. // This is so the sysadmin can override by inserting rules in @@ -886,14 +915,14 @@ var ipRules = []netlink.Rule{ // Packets from us, tagged with our fwmark, first try the kernel's // main routing table. { - Priority: 5210, + Priority: 10, Mark: tailscaleBypassMarkNum, Table: mainRouteTable.num, }, // ...and then we try the 'default' table, for correctness, // even though it's been empty on every Linux system I've ever seen. { - Priority: 5230, + Priority: 30, Mark: tailscaleBypassMarkNum, Table: defaultRouteTable.num, }, @@ -901,7 +930,7 @@ var ipRules = []netlink.Rule{ // then packets from us should be aborted rather than falling through // to the tailscale routes, because that would create routing loops. { - Priority: 5250, + Priority: 50, Mark: tailscaleBypassMarkNum, Type: unix.RTN_UNREACHABLE, }, @@ -911,7 +940,7 @@ var ipRules = []netlink.Rule{ // it takes precedence over all the others, ie. VPN routes always // beat non-VPN routes. { - Priority: 5270, + Priority: 70, Table: tailscaleRouteTable.num, }, // If that didn't match, then non-fwmark packets fall through to the @@ -928,6 +957,7 @@ func (r *linuxRouter) justAddIPRules() error { } var errAcc error for _, family := range r.addrFamilies() { + for _, ru := range ipRules { // Note: r is a value type here; safe to mutate it. ru.Family = family.netlinkInt() @@ -936,6 +966,7 @@ func (r *linuxRouter) justAddIPRules() error { ru.SuppressIfgroup = -1 ru.SuppressPrefixlen = -1 ru.Flow = -1 + ru.Priority += r.ipPolicyPrefBase err := netlink.RuleAdd(&ru) if errors.Is(err, errEEXIST) { @@ -954,19 +985,19 @@ func (r *linuxRouter) addIPRulesWithIPCommand() error { rg := newRunGroup(nil, r.cmd) for _, family := range r.addrFamilies() { - for _, r := range ipRules { + for _, rule := range ipRules { args := []string{ "ip", family.dashArg(), "rule", "add", - "pref", strconv.Itoa(r.Priority), + "pref", strconv.Itoa(rule.Priority + r.ipPolicyPrefBase), } - if r.Mark != 0 { - args = append(args, "fwmark", fmt.Sprintf("0x%x", r.Mark)) + if rule.Mark != 0 { + args = append(args, "fwmark", fmt.Sprintf("0x%x", rule.Mark)) } - if r.Table != 0 { - args = append(args, "table", mustRouteTable(r.Table).ipCmdArg()) + if rule.Table != 0 { + args = append(args, "table", mustRouteTable(rule.Table).ipCmdArg()) } - if r.Type == unix.RTN_UNREACHABLE { + if rule.Type == unix.RTN_UNREACHABLE { args = append(args, "type", "unreachable") } rg.Run(args...) @@ -1040,14 +1071,14 @@ func (r *linuxRouter) delIPRulesWithIPCommand() error { // That leaves us some flexibility to change these values in later // versions without having ongoing hacks for every possible // combination. - for _, r := range ipRules { + for _, rule := range ipRules { args := []string{ "ip", family.dashArg(), "rule", "del", - "pref", strconv.Itoa(r.Priority), + "pref", strconv.Itoa(rule.Priority + r.ipPolicyPrefBase), } - if r.Table != 0 { - args = append(args, "table", mustRouteTable(r.Table).ipCmdArg()) + if rule.Table != 0 { + args = append(args, "table", mustRouteTable(rule.Table).ipCmdArg()) } else { args = append(args, "type", "unreachable") } @@ -1548,6 +1579,39 @@ func checkIPRuleSupportsV6(logf logger.Logf) error { return netlink.RuleAdd(rule) } +// Checks if the running openWRT system is using mwan3, based on the heuristic +// of the config file being present as well as a policy rule with a specific +// priority (2000 + 1 - first interface mwan3 manages) and non-zero mark. +func checkOpenWRTUsingMWAN3() (bool, error) { + if distro.Get() != distro.OpenWrt { + return false, nil + } + + if _, err := os.Stat("/etc/config/mwan3"); err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + rules, err := netlink.RuleList(netlink.FAMILY_V4) + if err != nil { + return false, err + } + for _, r := range rules { + // We want to match on a rule like this: + // 2001: from all fwmark 0x100/0x3f00 lookup 1 + // + // We dont match on the mask because it can vary, or the + // table because I'm not sure if it can vary. + if r.Priority == 2001 && r.Mark != 0 { + return true, nil + } + } + + return false, nil +} + func nlAddrOfPrefix(p netip.Prefix) *netlink.Addr { return &netlink.Addr{ IPNet: netipx.PrefixIPNet(p),