doctor: add package for running in-depth healthchecks; use in bugreport (#5413)

Change-Id: Iaa4e5b021a545447f319cfe8b3da2bd3e5e5782b
Signed-off-by: Andrew Dunham <andrew@du.nham.ca>
pull/5756/head
Andrew Dunham 2 years ago committed by GitHub
parent e3beb4429f
commit b1867457a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -267,15 +267,47 @@ func (lc *LocalClient) Profile(ctx context.Context, pprofType string, sec int) (
return lc.get200(ctx, fmt.Sprintf("/localapi/v0/profile?name=%s&seconds=%v", url.QueryEscape(pprofType), secArg)) return lc.get200(ctx, fmt.Sprintf("/localapi/v0/profile?name=%s&seconds=%v", url.QueryEscape(pprofType), secArg))
} }
// BugReport logs and returns a log marker that can be shared by the user with support. // BugReportOpts contains options to pass to the Tailscale daemon when
func (lc *LocalClient) BugReport(ctx context.Context, note string) (string, error) { // generating a bug report.
body, err := lc.send(ctx, "POST", "/localapi/v0/bugreport?note="+url.QueryEscape(note), 200, nil) type BugReportOpts struct {
// Note contains an optional user-provided note to add to the logs.
Note string
// Diagnose specifies whether to print additional diagnostic information to
// the logs when generating this bugreport.
Diagnose bool
}
// BugReportWithOpts logs and returns a log marker that can be shared by the
// user with support.
//
// The opts type specifies options to pass to the Tailscale daemon when
// generating this bug report.
func (lc *LocalClient) BugReportWithOpts(ctx context.Context, opts BugReportOpts) (string, error) {
var qparams url.Values
if opts.Note != "" {
qparams.Set("note", opts.Note)
}
if opts.Diagnose {
qparams.Set("diagnose", "true")
}
uri := fmt.Sprintf("/localapi/v0/bugreport?%s", qparams.Encode())
body, err := lc.send(ctx, "POST", uri, 200, nil)
if err != nil { if err != nil {
return "", err return "", err
} }
return strings.TrimSpace(string(body)), nil return strings.TrimSpace(string(body)), nil
} }
// BugReport logs and returns a log marker that can be shared by the user with support.
//
// This is the same as calling BugReportWithOpts and only specifying the Note
// field.
func (lc *LocalClient) BugReport(ctx context.Context, note string) (string, error) {
return lc.BugReportWithOpts(ctx, BugReportOpts{Note: note})
}
// DebugAction invokes a debug action, such as "rebind" or "restun". // DebugAction invokes a debug action, such as "rebind" or "restun".
// These are development tools and subject to change or removal over time. // These are development tools and subject to change or removal over time.
func (lc *LocalClient) DebugAction(ctx context.Context, action string) error { func (lc *LocalClient) DebugAction(ctx context.Context, action string) error {

@ -7,8 +7,10 @@ package cli
import ( import (
"context" "context"
"errors" "errors"
"flag"
"github.com/peterbourgon/ff/v3/ffcli" "github.com/peterbourgon/ff/v3/ffcli"
"tailscale.com/client/tailscale"
) )
var bugReportCmd = &ffcli.Command{ var bugReportCmd = &ffcli.Command{
@ -16,6 +18,15 @@ var bugReportCmd = &ffcli.Command{
Exec: runBugReport, Exec: runBugReport,
ShortHelp: "Print a shareable identifier to help diagnose issues", ShortHelp: "Print a shareable identifier to help diagnose issues",
ShortUsage: "bugreport [note]", ShortUsage: "bugreport [note]",
FlagSet: (func() *flag.FlagSet {
fs := newFlagSet("bugreport")
fs.BoolVar(&bugReportArgs.diagnose, "diagnose", false, "run additional in-depth checks")
return fs
})(),
}
var bugReportArgs struct {
diagnose bool
} }
func runBugReport(ctx context.Context, args []string) error { func runBugReport(ctx context.Context, args []string) error {
@ -27,7 +38,10 @@ func runBugReport(ctx context.Context, args []string) error {
default: default:
return errors.New("unknown argumets") return errors.New("unknown argumets")
} }
logMarker, err := localClient.BugReport(ctx, note) logMarker, err := localClient.BugReportWithOpts(ctx, tailscale.BugReportOpts{
Note: note,
Diagnose: bugReportArgs.diagnose,
})
if err != nil { if err != nil {
return err return err
} }

@ -109,7 +109,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
github.com/tailscale/goupnp/scpd from github.com/tailscale/goupnp github.com/tailscale/goupnp/scpd from github.com/tailscale/goupnp
github.com/tailscale/goupnp/soap from github.com/tailscale/goupnp+ github.com/tailscale/goupnp/soap from github.com/tailscale/goupnp+
github.com/tailscale/goupnp/ssdp from github.com/tailscale/goupnp github.com/tailscale/goupnp/ssdp from github.com/tailscale/goupnp
L 💣 github.com/tailscale/netlink from tailscale.com/wgengine/router L 💣 github.com/tailscale/netlink from tailscale.com/wgengine/router+
github.com/tcnksm/go-httpstat from tailscale.com/net/netcheck github.com/tcnksm/go-httpstat from tailscale.com/net/netcheck
LD github.com/u-root/u-root/pkg/termios from tailscale.com/ssh/tailssh LD github.com/u-root/u-root/pkg/termios from tailscale.com/ssh/tailssh
L github.com/u-root/uio/rand from github.com/insomniacslk/dhcp/dhcpv4 L github.com/u-root/uio/rand from github.com/insomniacslk/dhcp/dhcpv4
@ -190,6 +190,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
tailscale.com/derp from tailscale.com/derp/derphttp+ tailscale.com/derp from tailscale.com/derp/derphttp+
tailscale.com/derp/derphttp from tailscale.com/net/netcheck+ tailscale.com/derp/derphttp from tailscale.com/net/netcheck+
tailscale.com/disco from tailscale.com/derp+ tailscale.com/disco from tailscale.com/derp+
tailscale.com/doctor from tailscale.com/ipn/ipnlocal
tailscale.com/doctor/routetable from tailscale.com/ipn/ipnlocal
tailscale.com/envknob from tailscale.com/control/controlclient+ tailscale.com/envknob from tailscale.com/control/controlclient+
tailscale.com/health from tailscale.com/control/controlclient+ tailscale.com/health from tailscale.com/control/controlclient+
tailscale.com/hostinfo from tailscale.com/control/controlclient+ tailscale.com/hostinfo from tailscale.com/control/controlclient+
@ -230,6 +232,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
tailscale.com/net/ping from tailscale.com/net/netcheck tailscale.com/net/ping from tailscale.com/net/netcheck
tailscale.com/net/portmapper from tailscale.com/net/netcheck+ tailscale.com/net/portmapper from tailscale.com/net/netcheck+
tailscale.com/net/proxymux from tailscale.com/cmd/tailscaled tailscale.com/net/proxymux from tailscale.com/cmd/tailscaled
tailscale.com/net/routetable from tailscale.com/doctor/routetable
tailscale.com/net/socks5 from tailscale.com/cmd/tailscaled tailscale.com/net/socks5 from tailscale.com/cmd/tailscaled
tailscale.com/net/stun from tailscale.com/net/netcheck+ tailscale.com/net/stun from tailscale.com/net/netcheck+
tailscale.com/net/tlsdial from tailscale.com/control/controlclient+ tailscale.com/net/tlsdial from tailscale.com/control/controlclient+

@ -0,0 +1,80 @@
// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package doctor contains more in-depth healthchecks that can be run to aid in
// diagnosing Tailscale issues.
package doctor
import (
"context"
"sync"
"tailscale.com/types/logger"
)
// Check is the interface defining a singular check.
//
// A check should log information that it gathers using the provided log
// function, and should attempt to make as much progress as possible in error
// conditions.
type Check interface {
// Name should return a name describing this check, in lower-kebab-case
// (i.e. "my-check", not "MyCheck" or "my_check").
Name() string
// Run executes the check, logging diagnostic information to the
// provided logger function.
Run(context.Context, logger.Logf) error
}
// RunChecks runs a list of checks in parallel, and logs any returned errors
// after all checks have returned.
func RunChecks(ctx context.Context, log logger.Logf, checks ...Check) {
if len(checks) == 0 {
return
}
type namedErr struct {
name string
err error
}
errs := make(chan namedErr, len(checks))
var wg sync.WaitGroup
wg.Add(len(checks))
for _, check := range checks {
go func(c Check) {
defer wg.Done()
plog := logger.WithPrefix(log, c.Name()+": ")
errs <- namedErr{
name: c.Name(),
err: c.Run(ctx, plog),
}
}(check)
}
wg.Wait()
close(errs)
for n := range errs {
if n.err == nil {
continue
}
log("check %s: %v", n.name, n.err)
}
}
// CheckFunc creates a Check from a name and a function.
func CheckFunc(name string, run func(context.Context, logger.Logf) error) Check {
return checkFunc{name, run}
}
type checkFunc struct {
name string
run func(context.Context, logger.Logf) error
}
func (c checkFunc) Name() string { return c.name }
func (c checkFunc) Run(ctx context.Context, log logger.Logf) error { return c.run(ctx, log) }

@ -0,0 +1,50 @@
// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package doctor
import (
"context"
"fmt"
"sync"
"testing"
qt "github.com/frankban/quicktest"
"tailscale.com/types/logger"
)
func TestRunChecks(t *testing.T) {
c := qt.New(t)
var (
mu sync.Mutex
lines []string
)
logf := func(format string, args ...any) {
mu.Lock()
defer mu.Unlock()
lines = append(lines, fmt.Sprintf(format, args...))
}
ctx := context.Background()
RunChecks(ctx, logf,
testCheck1{},
CheckFunc("testcheck2", func(_ context.Context, log logger.Logf) error {
log("check 2")
return nil
}),
)
mu.Lock()
defer mu.Unlock()
c.Assert(lines, qt.Contains, "testcheck1: check 1")
c.Assert(lines, qt.Contains, "testcheck2: check 2")
}
type testCheck1 struct{}
func (t testCheck1) Name() string { return "testcheck1" }
func (t testCheck1) Run(_ context.Context, log logger.Logf) error {
log("check 1")
return nil
}

@ -0,0 +1,35 @@
// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package routetable provides a doctor.Check that dumps the current system's
// route table to the log.
package routetable
import (
"context"
"tailscale.com/net/routetable"
"tailscale.com/types/logger"
)
// MaxRoutes is the maximum number of routes that will be displayed.
const MaxRoutes = 1000
// Check implements the doctor.Check interface.
type Check struct{}
func (Check) Name() string {
return "routetable"
}
func (Check) Run(_ context.Context, logf logger.Logf) error {
rs, err := routetable.Get(MaxRoutes)
if err != nil {
return err
}
for _, r := range rs {
logf("%s", r)
}
return nil
}

@ -26,6 +26,8 @@ import (
"go4.org/netipx" "go4.org/netipx"
"tailscale.com/client/tailscale/apitype" "tailscale.com/client/tailscale/apitype"
"tailscale.com/control/controlclient" "tailscale.com/control/controlclient"
"tailscale.com/doctor"
"tailscale.com/doctor/routetable"
"tailscale.com/envknob" "tailscale.com/envknob"
"tailscale.com/health" "tailscale.com/health"
"tailscale.com/hostinfo" "tailscale.com/hostinfo"
@ -3684,3 +3686,19 @@ func (b *LocalBackend) handleQuad100Port80Conn(w http.ResponseWriter, r *http.Re
} }
io.WriteString(w, "</ul>\n") io.WriteString(w, "</ul>\n")
} }
func (b *LocalBackend) Doctor(ctx context.Context, logf logger.Logf) {
var checks []doctor.Check
checks = append(checks, routetable.Check{})
// TODO(andrew): more
numChecks := len(checks)
checks = append(checks, doctor.CheckFunc("numchecks", func(_ context.Context, log logger.Logf) error {
log("%d checks", numChecks)
return nil
}))
doctor.RunChecks(ctx, logf, checks...)
}

@ -221,6 +221,9 @@ func (h *Handler) serveBugReport(w http.ResponseWriter, r *http.Request) {
if note := r.FormValue("note"); len(note) > 0 { if note := r.FormValue("note"); len(note) > 0 {
h.logf("user bugreport note: %s", note) h.logf("user bugreport note: %s", note)
} }
if defBool(r.FormValue("diagnose"), false) {
h.b.Doctor(r.Context(), logger.WithPrefix(h.logf, "diag: "))
}
w.Header().Set("Content-Type", "text/plain") w.Header().Set("Content-Type", "text/plain")
fmt.Fprintln(w, logMarker) fmt.Fprintln(w, logMarker)
} }

@ -0,0 +1,151 @@
// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package routetable provides functions that operate on the system's route
// table.
package routetable
import (
"bufio"
"fmt"
"net/netip"
"strconv"
"tailscale.com/types/logger"
)
var (
defaultRouteIPv4 = RouteDestination{Prefix: netip.PrefixFrom(netip.IPv4Unspecified(), 0)}
defaultRouteIPv6 = RouteDestination{Prefix: netip.PrefixFrom(netip.IPv6Unspecified(), 0)}
)
// RouteEntry contains common cross-platform fields describing an entry in the
// system route table.
type RouteEntry struct {
// Family is the IP family of the route; it will be either 4 or 6.
Family int
// Type is the type of this route.
Type RouteType
// Dst is the destination of the route.
Dst RouteDestination
// Gatewayis the gateway address specified for this route.
// This value will be invalid (where !r.Gateway.IsValid()) in cases
// where there is no gateway address for this route.
Gateway netip.Addr
// Interface is the name of the network interface to use when sending
// packets that match this route. This field can be empty.
Interface string
// Sys contains platform-specific information about this route.
Sys any
}
// Format implements the fmt.Formatter interface.
func (r RouteEntry) Format(f fmt.State, verb rune) {
logger.ArgWriter(func(w *bufio.Writer) {
switch r.Family {
case 4:
fmt.Fprintf(w, "{Family: IPv4")
case 6:
fmt.Fprintf(w, "{Family: IPv6")
default:
fmt.Fprintf(w, "{Family: unknown(%d)", r.Family)
}
// Match 'ip route' and other tools by not printing the route
// type if it's a unicast route.
if r.Type != RouteTypeUnicast {
fmt.Fprintf(w, ", Type: %s", r.Type)
}
if r.Dst.IsValid() {
fmt.Fprintf(w, ", Dst: %s", r.Dst)
} else {
w.WriteString(", Dst: invalid")
}
if r.Gateway.IsValid() {
fmt.Fprintf(w, ", Gateway: %s", r.Gateway)
}
if r.Interface != "" {
fmt.Fprintf(w, ", Interface: %s", r.Interface)
}
if r.Sys != nil {
var formatVerb string
switch {
case f.Flag('#'):
formatVerb = "%#v"
case f.Flag('+'):
formatVerb = "%+v"
default:
formatVerb = "%v"
}
fmt.Fprintf(w, ", Sys: "+formatVerb, r.Sys)
}
w.WriteString("}")
}).Format(f, verb)
}
// RouteDestination is the destination of a route.
//
// This is similar to net/netip.Prefix, but also contains an optional IPv6
// zone.
type RouteDestination struct {
netip.Prefix
Zone string
}
func (r RouteDestination) String() string {
ip := r.Prefix.Addr()
if r.Zone != "" {
ip = ip.WithZone(r.Zone)
}
return ip.String() + "/" + strconv.Itoa(r.Prefix.Bits())
}
// RouteType describes the type of a route.
type RouteType int
const (
// RouteTypeUnspecified is the unspecified route type.
RouteTypeUnspecified RouteType = iota
// RouteTypeLocal indicates that the destination of this route is an
// address that belongs to this system.
RouteTypeLocal
// RouteTypeUnicast indicates that the destination of this route is a
// "regular" address--one that neither belongs to this host, nor is a
// broadcast/multicast/etc. address.
RouteTypeUnicast
// RouteTypeBroadcast indicates that the destination of this route is a
// broadcast address.
RouteTypeBroadcast
// RouteTypeMulticast indicates that the destination of this route is a
// multicast address.
RouteTypeMulticast
// RouteTypeOther indicates that the route is of some other valid type;
// see the Sys field for the OS-provided route information to determine
// the exact type.
RouteTypeOther
)
func (r RouteType) String() string {
switch r {
case RouteTypeUnspecified:
return "unspecified"
case RouteTypeLocal:
return "local"
case RouteTypeUnicast:
return "unicast"
case RouteTypeBroadcast:
return "broadcast"
case RouteTypeMulticast:
return "multicast"
case RouteTypeOther:
return "other"
default:
return "invalid"
}
}

@ -0,0 +1,285 @@
// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build darwin || freebsd
// +build darwin freebsd
package routetable
import (
"bufio"
"fmt"
"net"
"net/netip"
"runtime"
"sort"
"strings"
"syscall"
"golang.org/x/net/route"
"golang.org/x/sys/unix"
"tailscale.com/net/interfaces"
"tailscale.com/types/logger"
)
type RouteEntryBSD struct {
// GatewayInterface is the name of the interface specified as a gateway
// for this route, if any.
GatewayInterface string
// GatewayIdx is the index of the interface specified as a gateway for
// this route, if any.
GatewayIdx int
// GatewayAddr is the link-layer address of the gateway for this route,
// if any.
GatewayAddr string
// Flags contains a string representation of common flags for this
// route.
Flags []string
// RawFlags contains the raw flags that were returned by the operating
// system for this route.
RawFlags int
}
// Format implements the fmt.Formatter interface.
func (r RouteEntryBSD) Format(f fmt.State, verb rune) {
logger.ArgWriter(func(w *bufio.Writer) {
var pstart bool
pr := func(format string, args ...any) {
if pstart {
fmt.Fprintf(w, ", "+format, args...)
} else {
fmt.Fprintf(w, format, args...)
pstart = true
}
}
w.WriteString("{")
if r.GatewayInterface != "" {
pr("GatewayInterface: %s", r.GatewayInterface)
}
if r.GatewayIdx > 0 {
pr("GatewayIdx: %d", r.GatewayIdx)
}
if r.GatewayAddr != "" {
pr("GatewayAddr: %s", r.GatewayAddr)
}
pr("Flags: %v", r.Flags)
w.WriteString("}")
}).Format(f, verb)
}
// ipFromRMAddr returns a netip.Addr converted from one of the
// route.Inet{4,6}Addr types.
func ipFromRMAddr(ifs map[int]interfaces.Interface, addr any) netip.Addr {
switch v := addr.(type) {
case *route.Inet4Addr:
return netip.AddrFrom4(v.IP)
case *route.Inet6Addr:
ip := netip.AddrFrom16(v.IP)
if v.ZoneID != 0 {
if iif, ok := ifs[v.ZoneID]; ok {
ip = ip.WithZone(iif.Name)
} else {
ip = ip.WithZone(fmt.Sprint(v.ZoneID))
}
}
return ip
}
return netip.Addr{}
}
// populateGateway populates gateway fields on a RouteEntry/RouteEntryBSD.
func populateGateway(re *RouteEntry, reSys *RouteEntryBSD, ifs map[int]interfaces.Interface, addr any) {
// If the address type has a valid IP, use that.
if ip := ipFromRMAddr(ifs, addr); ip.IsValid() {
re.Gateway = ip
return
}
switch v := addr.(type) {
case *route.LinkAddr:
reSys.GatewayIdx = v.Index
if iif, ok := ifs[v.Index]; ok {
reSys.GatewayInterface = iif.Name
}
var sb strings.Builder
for i, x := range v.Addr {
if i != 0 {
sb.WriteByte(':')
}
fmt.Fprintf(&sb, "%02x", x)
}
reSys.GatewayAddr = sb.String()
}
}
// populateDestination populates the 'Dst' field on a RouteEntry based on the
// RouteMessage's destination and netmask fields.
func populateDestination(re *RouteEntry, ifs map[int]interfaces.Interface, rm *route.RouteMessage) {
dst := rm.Addrs[unix.RTAX_DST]
if dst == nil {
return
}
ip := ipFromRMAddr(ifs, dst)
if !ip.IsValid() {
return
}
if ip.Is4() {
re.Family = 4
} else {
re.Family = 6
}
re.Dst = RouteDestination{
Prefix: netip.PrefixFrom(ip, 32), // default if nothing more specific
}
// If the RTF_HOST flag is set, then this is a host route and there's
// no netmask in this RouteMessage.
if rm.Flags&unix.RTF_HOST != 0 {
return
}
// As above if there's no netmask in the list of addrs
if len(rm.Addrs) < unix.RTAX_NETMASK || rm.Addrs[unix.RTAX_NETMASK] == nil {
return
}
nm := ipFromRMAddr(ifs, rm.Addrs[unix.RTAX_NETMASK])
if !ip.IsValid() {
return
}
// Count the number of bits in the netmask IP and use that to make our prefix.
ones, _ /* bits */ := net.IPMask(nm.AsSlice()).Size()
// Print this ourselves instead of using netip.Prefix so that we don't
// lose the zone (since netip.Prefix strips that).
//
// NOTE(andrew): this doesn't print the same values as the 'netstat' tool
// for some addresses on macOS, and I have no idea why. Specifically,
// 'netstat -rn' will show something like:
// ff00::/8 ::1 UmCI lo0
//
// But we will get:
// destination=ff00::/40 [...]
//
// The netmask that we get back from FetchRIB has 32 more bits in it
// than netstat prints, but only for multicast routes.
//
// For consistency's sake, we're going to do the same here so that we
// get the same values as netstat returns.
if runtime.GOOS == "darwin" && ip.Is6() && ip.IsMulticast() && ones > 32 {
ones -= 32
}
re.Dst = RouteDestination{
Prefix: netip.PrefixFrom(ip, ones),
Zone: ip.Zone(),
}
}
// routeEntryFromMsg returns a RouteEntry from a single route.Message
// returned by the operating system.
func routeEntryFromMsg(ifsByIdx map[int]interfaces.Interface, msg route.Message) (RouteEntry, bool) {
rm, ok := msg.(*route.RouteMessage)
if !ok {
return RouteEntry{}, false
}
// Ignore things that we don't understand
if rm.Version < 3 || rm.Version > 5 {
return RouteEntry{}, false
}
if rm.Type != rmExpectedType {
return RouteEntry{}, false
}
if len(rm.Addrs) < unix.RTAX_GATEWAY {
return RouteEntry{}, false
}
if rm.Flags&skipFlags != 0 {
return RouteEntry{}, false
}
reSys := RouteEntryBSD{
RawFlags: rm.Flags,
}
for fv, fs := range flags {
if rm.Flags&fv == fv {
reSys.Flags = append(reSys.Flags, fs)
}
}
sort.Strings(reSys.Flags)
re := RouteEntry{}
hasFlag := func(f int) bool { return rm.Flags&f != 0 }
switch {
case hasFlag(unix.RTF_LOCAL):
re.Type = RouteTypeLocal
case hasFlag(unix.RTF_BROADCAST):
re.Type = RouteTypeBroadcast
case hasFlag(unix.RTF_MULTICAST):
re.Type = RouteTypeMulticast
// From the manpage: "host entry (net otherwise)"
case !hasFlag(unix.RTF_HOST):
re.Type = RouteTypeUnicast
default:
re.Type = RouteTypeOther
}
populateDestination(&re, ifsByIdx, rm)
if unix.RTAX_GATEWAY < len(rm.Addrs) {
populateGateway(&re, &reSys, ifsByIdx, rm.Addrs[unix.RTAX_GATEWAY])
}
if outif, ok := ifsByIdx[rm.Index]; ok {
re.Interface = outif.Name
}
re.Sys = reSys
return re, true
}
// Get returns route entries from the system route table, limited to at most
// 'max' results.
func Get(max int) ([]RouteEntry, error) {
// Fetching the list of interfaces can race with fetching our route
// table, but we do it anyway since it's helpful for debugging.
ifs, err := interfaces.GetList()
if err != nil {
return nil, err
}
ifsByIdx := make(map[int]interfaces.Interface)
for _, iif := range ifs {
ifsByIdx[iif.Index] = iif
}
rib, err := route.FetchRIB(syscall.AF_UNSPEC, ribType, 0)
if err != nil {
return nil, err
}
msgs, err := route.ParseRIB(parseType, rib)
if err != nil {
return nil, err
}
var ret []RouteEntry
for _, m := range msgs {
re, ok := routeEntryFromMsg(ifsByIdx, m)
if ok {
ret = append(ret, re)
if len(ret) == max {
break
}
}
}
return ret, nil
}

@ -0,0 +1,435 @@
// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build darwin || freebsd
// +build darwin freebsd
package routetable
import (
"fmt"
"net"
"net/netip"
"reflect"
"runtime"
"testing"
"golang.org/x/net/route"
"golang.org/x/sys/unix"
"tailscale.com/net/interfaces"
)
func TestRouteEntryFromMsg(t *testing.T) {
ifs := map[int]interfaces.Interface{
1: {
Interface: &net.Interface{
Name: "iface0",
},
},
2: {
Interface: &net.Interface{
Name: "tailscale0",
},
},
}
ip4 := func(s string) *route.Inet4Addr {
ip := netip.MustParseAddr(s)
return &route.Inet4Addr{IP: ip.As4()}
}
ip6 := func(s string) *route.Inet6Addr {
ip := netip.MustParseAddr(s)
return &route.Inet6Addr{IP: ip.As16()}
}
ip6zone := func(s string, idx int) *route.Inet6Addr {
ip := netip.MustParseAddr(s)
return &route.Inet6Addr{IP: ip.As16(), ZoneID: idx}
}
link := func(idx int, addr string) *route.LinkAddr {
if _, found := ifs[idx]; !found {
panic("index not found")
}
ret := &route.LinkAddr{
Index: idx,
}
if addr != "" {
ret.Addr = make([]byte, 6)
fmt.Sscanf(addr, "%02x:%02x:%02x:%02x:%02x:%02x",
&ret.Addr[0],
&ret.Addr[1],
&ret.Addr[2],
&ret.Addr[3],
&ret.Addr[4],
&ret.Addr[5],
)
}
return ret
}
type testCase struct {
name string
msg *route.RouteMessage
want RouteEntry
fail bool
}
testCases := []testCase{
{
name: "BasicIPv4",
msg: &route.RouteMessage{
Version: 3,
Type: rmExpectedType,
Addrs: []route.Addr{
ip4("1.2.3.4"), // dst
ip4("1.2.3.1"), // gateway
ip4("255.255.255.0"), // netmask
},
},
want: RouteEntry{
Family: 4,
Type: RouteTypeUnicast,
Dst: RouteDestination{Prefix: netip.MustParsePrefix("1.2.3.4/24")},
Gateway: netip.MustParseAddr("1.2.3.1"),
Sys: RouteEntryBSD{},
},
},
{
name: "BasicIPv6",
msg: &route.RouteMessage{
Version: 3,
Type: rmExpectedType,
Addrs: []route.Addr{
ip6("fd7a:115c:a1e0::"), // dst
ip6("1234::"), // gateway
ip6("ffff:ffff:ffff::"), // netmask
},
},
want: RouteEntry{
Family: 6,
Type: RouteTypeUnicast,
Dst: RouteDestination{Prefix: netip.MustParsePrefix("fd7a:115c:a1e0::/48")},
Gateway: netip.MustParseAddr("1234::"),
Sys: RouteEntryBSD{},
},
},
{
name: "IPv6WithZone",
msg: &route.RouteMessage{
Version: 3,
Type: rmExpectedType,
Addrs: []route.Addr{
ip6zone("fe80::", 2), // dst
ip6("1234::"), // gateway
ip6("ffff:ffff:ffff:ffff::"), // netmask
},
},
want: RouteEntry{
Family: 6,
Type: RouteTypeUnicast, // TODO
Dst: RouteDestination{Prefix: netip.MustParsePrefix("fe80::/64"), Zone: "tailscale0"},
Gateway: netip.MustParseAddr("1234::"),
Sys: RouteEntryBSD{},
},
},
{
name: "IPv6WithUnknownZone",
msg: &route.RouteMessage{
Version: 3,
Type: rmExpectedType,
Addrs: []route.Addr{
ip6zone("fe80::", 4), // dst
ip6("1234::"), // gateway
ip6("ffff:ffff:ffff:ffff::"), // netmask
},
},
want: RouteEntry{
Family: 6,
Type: RouteTypeUnicast, // TODO
Dst: RouteDestination{Prefix: netip.MustParsePrefix("fe80::/64"), Zone: "4"},
Gateway: netip.MustParseAddr("1234::"),
Sys: RouteEntryBSD{},
},
},
{
name: "DefaultIPv4",
msg: &route.RouteMessage{
Version: 3,
Type: rmExpectedType,
Addrs: []route.Addr{
ip4("0.0.0.0"), // dst
ip4("1.2.3.4"), // gateway
ip4("0.0.0.0"), // netmask
},
},
want: RouteEntry{
Family: 4,
Type: RouteTypeUnicast,
Dst: defaultRouteIPv4,
Gateway: netip.MustParseAddr("1.2.3.4"),
Sys: RouteEntryBSD{},
},
},
{
name: "DefaultIPv6",
msg: &route.RouteMessage{
Version: 3,
Type: rmExpectedType,
Addrs: []route.Addr{
ip6("0::"), // dst
ip6("1234::"), // gateway
ip6("0::"), // netmask
},
},
want: RouteEntry{
Family: 6,
Type: RouteTypeUnicast,
Dst: defaultRouteIPv6,
Gateway: netip.MustParseAddr("1234::"),
Sys: RouteEntryBSD{},
},
},
{
name: "ShortAddrs",
msg: &route.RouteMessage{
Version: 3,
Type: rmExpectedType,
Addrs: []route.Addr{
ip4("1.2.3.4"), // dst
},
},
want: RouteEntry{
Family: 4,
Type: RouteTypeUnicast,
Dst: RouteDestination{Prefix: netip.MustParsePrefix("1.2.3.4/32")},
Sys: RouteEntryBSD{},
},
},
{
name: "TailscaleIPv4",
msg: &route.RouteMessage{
Version: 3,
Type: rmExpectedType,
Addrs: []route.Addr{
ip4("100.64.0.0"), // dst
link(2, ""),
ip4("255.192.0.0"), // netmask
},
},
want: RouteEntry{
Family: 4,
Type: RouteTypeUnicast,
Dst: RouteDestination{Prefix: netip.MustParsePrefix("100.64.0.0/10")},
Sys: RouteEntryBSD{
GatewayInterface: "tailscale0",
GatewayIdx: 2,
},
},
},
{
name: "Flags",
msg: &route.RouteMessage{
Version: 3,
Type: rmExpectedType,
Addrs: []route.Addr{
ip4("1.2.3.4"), // dst
ip4("1.2.3.1"), // gateway
ip4("255.255.255.0"), // netmask
},
Flags: unix.RTF_STATIC | unix.RTF_GATEWAY | unix.RTF_UP,
},
want: RouteEntry{
Family: 4,
Type: RouteTypeUnicast,
Dst: RouteDestination{Prefix: netip.MustParsePrefix("1.2.3.4/24")},
Gateway: netip.MustParseAddr("1.2.3.1"),
Sys: RouteEntryBSD{
Flags: []string{"gateway", "static", "up"},
RawFlags: unix.RTF_STATIC | unix.RTF_GATEWAY | unix.RTF_UP,
},
},
},
{
name: "SkipNoAddrs",
msg: &route.RouteMessage{
Version: 3,
Type: rmExpectedType,
Addrs: []route.Addr{},
},
fail: true,
},
{
name: "SkipBadVersion",
msg: &route.RouteMessage{
Version: 1,
},
fail: true,
},
{
name: "SkipBadType",
msg: &route.RouteMessage{
Version: 3,
Type: rmExpectedType + 1,
},
fail: true,
},
{
name: "OutputIface",
msg: &route.RouteMessage{
Version: 3,
Type: rmExpectedType,
Index: 1,
Addrs: []route.Addr{
ip4("1.2.3.4"), // dst
},
},
want: RouteEntry{
Family: 4,
Type: RouteTypeUnicast,
Dst: RouteDestination{Prefix: netip.MustParsePrefix("1.2.3.4/32")},
Interface: "iface0",
Sys: RouteEntryBSD{},
},
},
{
name: "GatewayMAC",
msg: &route.RouteMessage{
Version: 3,
Type: rmExpectedType,
Addrs: []route.Addr{
ip4("100.64.0.0"), // dst
link(1, "01:02:03:04:05:06"),
ip4("255.192.0.0"), // netmask
},
},
want: RouteEntry{
Family: 4,
Type: RouteTypeUnicast,
Dst: RouteDestination{Prefix: netip.MustParsePrefix("100.64.0.0/10")},
Sys: RouteEntryBSD{
GatewayAddr: "01:02:03:04:05:06",
GatewayInterface: "iface0",
GatewayIdx: 1,
},
},
},
}
if runtime.GOOS == "darwin" {
testCases = append(testCases,
testCase{
name: "SkipFlags",
msg: &route.RouteMessage{
Version: 3,
Type: rmExpectedType,
Addrs: []route.Addr{
ip4("1.2.3.4"), // dst
ip4("1.2.3.1"), // gateway
ip4("255.255.255.0"), // netmask
},
Flags: unix.RTF_UP | skipFlags,
},
fail: true,
},
testCase{
name: "NetmaskAdjust",
msg: &route.RouteMessage{
Version: 3,
Type: rmExpectedType,
Flags: unix.RTF_MULTICAST,
Addrs: []route.Addr{
ip6("ff00::"), // dst
ip6("1234::"), // gateway
ip6("ffff:ffff:ff00::"), // netmask
},
},
want: RouteEntry{
Family: 6,
Type: RouteTypeMulticast,
Dst: RouteDestination{Prefix: netip.MustParsePrefix("ff00::/8")},
Gateway: netip.MustParseAddr("1234::"),
Sys: RouteEntryBSD{
Flags: []string{"multicast"},
RawFlags: unix.RTF_MULTICAST,
},
},
},
)
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
re, ok := routeEntryFromMsg(ifs, tc.msg)
if wantOk := !tc.fail; ok != wantOk {
t.Fatalf("ok = %v; want %v", ok, wantOk)
}
if !reflect.DeepEqual(re, tc.want) {
t.Fatalf("RouteEntry mismatch:\n got: %+v\nwant: %+v", re, tc.want)
}
})
}
}
func TestRouteEntryFormatting(t *testing.T) {
testCases := []struct {
re RouteEntry
want string
}{
{
re: RouteEntry{
Family: 4,
Type: RouteTypeUnicast,
Dst: RouteDestination{Prefix: netip.MustParsePrefix("1.2.3.0/24")},
Interface: "en0",
Sys: RouteEntryBSD{
GatewayInterface: "en0",
Flags: []string{"static", "up"},
},
},
want: `{Family: IPv4, Dst: 1.2.3.0/24, Interface: en0, Sys: {GatewayInterface: en0, Flags: [static up]}}`,
},
{
re: RouteEntry{
Family: 6,
Type: RouteTypeUnicast,
Dst: RouteDestination{Prefix: netip.MustParsePrefix("fd7a:115c:a1e0::/24")},
Interface: "en0",
Sys: RouteEntryBSD{
GatewayIdx: 3,
Flags: []string{"static", "up"},
},
},
want: `{Family: IPv6, Dst: fd7a:115c:a1e0::/24, Interface: en0, Sys: {GatewayIdx: 3, Flags: [static up]}}`,
},
}
for _, tc := range testCases {
t.Run("", func(t *testing.T) {
got := fmt.Sprint(tc.re)
if got != tc.want {
t.Fatalf("RouteEntry.String() mismatch\n got: %q\nwant: %q", got, tc.want)
}
})
}
}
func TestGetRouteTable(t *testing.T) {
routes, err := Get(1000)
if err != nil {
t.Fatal(err)
}
// Basic assertion: we have at least one 'default' route
var (
hasDefault bool
)
for _, route := range routes {
if route.Dst == defaultRouteIPv4 || route.Dst == defaultRouteIPv6 {
hasDefault = true
}
}
if !hasDefault {
t.Errorf("expected at least one default route; routes=%v", routes)
}
}

@ -0,0 +1,33 @@
// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build darwin
// +build darwin
package routetable
import "golang.org/x/sys/unix"
const (
ribType = unix.NET_RT_DUMP2
parseType = unix.NET_RT_IFLIST2
rmExpectedType = unix.RTM_GET2
// Skip routes that were cloned from a parent
skipFlags = unix.RTF_WASCLONED
)
var flags = map[int]string{
unix.RTF_BLACKHOLE: "blackhole",
unix.RTF_BROADCAST: "broadcast",
unix.RTF_GATEWAY: "gateway",
unix.RTF_GLOBAL: "global",
unix.RTF_HOST: "host",
unix.RTF_IFSCOPE: "ifscope",
unix.RTF_MULTICAST: "multicast",
unix.RTF_REJECT: "reject",
unix.RTF_ROUTER: "router",
unix.RTF_STATIC: "static",
unix.RTF_UP: "up",
}

@ -0,0 +1,30 @@
// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build freebsd
// +build freebsd
package routetable
import "golang.org/x/sys/unix"
const (
ribType = unix.NET_RT_DUMP
parseType = unix.NET_RT_IFLIST
rmExpectedType = unix.RTM_GET
// Nothing to skip
skipFlags = 0
)
var flags = map[int]string{
unix.RTF_BLACKHOLE: "blackhole",
unix.RTF_BROADCAST: "broadcast",
unix.RTF_GATEWAY: "gateway",
unix.RTF_HOST: "host",
unix.RTF_MULTICAST: "multicast",
unix.RTF_REJECT: "reject",
unix.RTF_STATIC: "static",
unix.RTF_UP: "up",
}

@ -0,0 +1,231 @@
// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build linux
// +build linux
package routetable
import (
"bufio"
"fmt"
"net/netip"
"strconv"
"github.com/tailscale/netlink"
"golang.org/x/sys/unix"
"tailscale.com/net/interfaces"
"tailscale.com/net/netaddr"
"tailscale.com/types/logger"
)
// RouteEntryLinux is the structure that makes up the Sys field of the
// RouteEntry structure.
type RouteEntryLinux struct {
// Type is the raw type of the route.
Type int
// Table is the routing table index of this route.
Table int
// Src is the source of the route (if any).
Src netip.Addr
// Proto describes the source of the route--i.e. what caused this route
// to be added to the route table.
Proto netlink.RouteProtocol
// Priority is the route's priority.
Priority int
// Scope is the route's scope.
Scope int
// InputInterfaceIdx is the input interface index.
InputInterfaceIdx int
// InputInterfaceName is the input interface name (if available).
InputInterfaceName string
}
// Format implements the fmt.Formatter interface.
func (r RouteEntryLinux) Format(f fmt.State, verb rune) {
logger.ArgWriter(func(w *bufio.Writer) {
// TODO(andrew): should we skip printing anything if type is unicast?
fmt.Fprintf(w, "{Type: %s", r.TypeName())
// Match 'ip route' behaviour when printing these fields
if r.Table != unix.RT_TABLE_MAIN {
fmt.Fprintf(w, ", Table: %s", r.TableName())
}
if r.Proto != unix.RTPROT_BOOT {
fmt.Fprintf(w, ", Proto: %s", r.Proto)
}
if r.Src.IsValid() {
fmt.Fprintf(w, ", Src: %s", r.Src)
}
if r.Priority != 0 {
fmt.Fprintf(w, ", Priority: %d", r.Priority)
}
if r.Scope != unix.RT_SCOPE_UNIVERSE {
fmt.Fprintf(w, ", Scope: %s", r.ScopeName())
}
if r.InputInterfaceName != "" {
fmt.Fprintf(w, ", InputInterfaceName: %s", r.InputInterfaceName)
} else if r.InputInterfaceIdx != 0 {
fmt.Fprintf(w, ", InputInterfaceIdx: %d", r.InputInterfaceIdx)
}
w.WriteString("}")
}).Format(f, verb)
}
// TypeName returns the string representation of this route's Type.
func (r RouteEntryLinux) TypeName() string {
switch r.Type {
case unix.RTN_UNSPEC:
return "none"
case unix.RTN_UNICAST:
return "unicast"
case unix.RTN_LOCAL:
return "local"
case unix.RTN_BROADCAST:
return "broadcast"
case unix.RTN_ANYCAST:
return "anycast"
case unix.RTN_MULTICAST:
return "multicast"
case unix.RTN_BLACKHOLE:
return "blackhole"
case unix.RTN_UNREACHABLE:
return "unreachable"
case unix.RTN_PROHIBIT:
return "prohibit"
case unix.RTN_THROW:
return "throw"
case unix.RTN_NAT:
return "nat"
case unix.RTN_XRESOLVE:
return "xresolve"
default:
return strconv.Itoa(r.Type)
}
}
// TableName returns the string representation of this route's Table.
func (r RouteEntryLinux) TableName() string {
switch r.Table {
case unix.RT_TABLE_DEFAULT:
return "default"
case unix.RT_TABLE_MAIN:
return "main"
case unix.RT_TABLE_LOCAL:
return "local"
default:
return strconv.Itoa(r.Table)
}
}
// ScopeName returns the string representation of this route's Scope.
func (r RouteEntryLinux) ScopeName() string {
switch r.Scope {
case unix.RT_SCOPE_UNIVERSE:
return "global"
case unix.RT_SCOPE_NOWHERE:
return "nowhere"
case unix.RT_SCOPE_HOST:
return "host"
case unix.RT_SCOPE_LINK:
return "link"
case unix.RT_SCOPE_SITE:
return "site"
default:
return strconv.Itoa(r.Scope)
}
}
// Get returns route entries from the system route table, limited to at most
// max results.
func Get(max int) ([]RouteEntry, error) {
// Fetching the list of interfaces can race with fetching our route
// table, but we do it anyway since it's helpful for debugging.
ifs, err := interfaces.GetList()
if err != nil {
return nil, err
}
ifsByIdx := make(map[int]interfaces.Interface)
for _, iif := range ifs {
ifsByIdx[iif.Index] = iif
}
filter := &netlink.Route{}
routes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, filter, netlink.RT_FILTER_TABLE)
if err != nil {
return nil, err
}
var ret []RouteEntry
for _, route := range routes {
if route.Family != netlink.FAMILY_V4 && route.Family != netlink.FAMILY_V6 {
continue
}
re := RouteEntry{}
if route.Family == netlink.FAMILY_V4 {
re.Family = 4
} else {
re.Family = 6
}
switch route.Type {
case unix.RTN_UNSPEC:
re.Type = RouteTypeUnspecified
case unix.RTN_UNICAST:
re.Type = RouteTypeUnicast
case unix.RTN_LOCAL:
re.Type = RouteTypeLocal
case unix.RTN_BROADCAST:
re.Type = RouteTypeBroadcast
case unix.RTN_MULTICAST:
re.Type = RouteTypeMulticast
default:
re.Type = RouteTypeOther
}
if route.Dst != nil {
if d, ok := netaddr.FromStdIPNet(route.Dst); ok {
re.Dst = RouteDestination{Prefix: d}
}
} else if route.Family == netlink.FAMILY_V4 {
re.Dst = RouteDestination{Prefix: netip.PrefixFrom(netip.IPv4Unspecified(), 0)}
} else {
re.Dst = RouteDestination{Prefix: netip.PrefixFrom(netip.IPv6Unspecified(), 0)}
}
if gw := route.Gw; gw != nil {
if gwa, ok := netip.AddrFromSlice(gw); ok {
re.Gateway = gwa
}
}
if outif, ok := ifsByIdx[route.LinkIndex]; ok {
re.Interface = outif.Name
} else if route.LinkIndex > 0 {
re.Interface = fmt.Sprintf("link#%d", route.LinkIndex)
}
reSys := RouteEntryLinux{
Type: route.Type,
Table: route.Table,
Proto: route.Protocol,
Priority: route.Priority,
Scope: int(route.Scope),
InputInterfaceIdx: route.ILinkIndex,
}
if src, ok := netip.AddrFromSlice(route.Src); ok {
reSys.Src = src
}
if iif, ok := ifsByIdx[route.ILinkIndex]; ok {
reSys.InputInterfaceName = iif.Name
}
re.Sys = reSys
ret = append(ret, re)
// Stop after we've reached the maximum number of routes
if len(ret) == max {
break
}
}
return ret, nil
}

@ -0,0 +1,83 @@
// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build linux
// +build linux
package routetable
import (
"fmt"
"net/netip"
"testing"
"golang.org/x/sys/unix"
)
func TestGetRouteTable(t *testing.T) {
routes, err := Get(1000)
if err != nil {
t.Fatal(err)
}
// Basic assertion: we have at least one 'default' route in the main table
var (
hasDefault bool
)
for _, route := range routes {
if route.Dst == defaultRouteIPv4 && route.Sys.(RouteEntryLinux).Table == unix.RT_TABLE_MAIN {
hasDefault = true
}
}
if !hasDefault {
t.Errorf("expected at least one default route; routes=%v", routes)
}
}
func TestRouteEntryFormatting(t *testing.T) {
testCases := []struct {
re RouteEntry
want string
}{
{
re: RouteEntry{
Family: 4,
Type: RouteTypeMulticast,
Dst: RouteDestination{Prefix: netip.MustParsePrefix("100.64.0.0/10")},
Gateway: netip.MustParseAddr("1.2.3.1"),
Interface: "tailscale0",
Sys: RouteEntryLinux{
Type: unix.RTN_UNICAST,
Table: 52,
Proto: unix.RTPROT_STATIC,
Src: netip.MustParseAddr("1.2.3.4"),
Priority: 555,
},
},
want: `{Family: IPv4, Type: multicast, Dst: 100.64.0.0/10, Gateway: 1.2.3.1, Interface: tailscale0, Sys: {Type: unicast, Table: 52, Proto: static, Src: 1.2.3.4, Priority: 555}}`,
},
{
re: RouteEntry{
Family: 4,
Type: RouteTypeUnicast,
Dst: RouteDestination{Prefix: netip.MustParsePrefix("1.2.3.0/24")},
Gateway: netip.MustParseAddr("1.2.3.1"),
Sys: RouteEntryLinux{
Type: unix.RTN_UNICAST,
Table: unix.RT_TABLE_MAIN,
Proto: unix.RTPROT_BOOT,
},
},
want: `{Family: IPv4, Dst: 1.2.3.0/24, Gateway: 1.2.3.1, Sys: {Type: unicast}}`,
},
}
for _, tc := range testCases {
t.Run("", func(t *testing.T) {
got := fmt.Sprint(tc.re)
if got != tc.want {
t.Fatalf("RouteEntry.String() = %q; want %q", got, tc.want)
}
})
}
}

@ -0,0 +1,18 @@
// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !linux && !darwin && !freebsd
package routetable
import (
"errors"
"runtime"
)
var errUnsupported = errors.New("cannot get route table on platform " + runtime.GOOS)
func Get(max int) ([]RouteEntry, error) {
return nil, errUnsupported
}

@ -138,6 +138,8 @@ var rateFree = []string{
"SetPrefs: %v", "SetPrefs: %v",
"peer keys: %s", "peer keys: %s",
"v%v peers: %v", "v%v peers: %v",
// debug messages printed by 'tailscale bugreport'
"diag: ",
} }
// RateLimitedFn is a wrapper for RateLimitedFnWithClock that includes the // RateLimitedFn is a wrapper for RateLimitedFnWithClock that includes the

Loading…
Cancel
Save