cmd/tailscale, cmd/tailscaled: move portmapper debugging into tailscale CLI

The debug flag on tailscaled isn't available in the macOS App Store
build, since we don't have a tailscaled binary; move it to the
'tailscale debug' CLI that is available on all platforms instead,
accessed over LocalAPI.

Updates #7377

Signed-off-by: Andrew Dunham <andrew@du.nham.ca>
Change-Id: I47bffe4461e036fab577c2e51e173f4003592ff7
pull/7446/head
Andrew Dunham 1 year ago
parent 3e71e0ef68
commit 3f8e8b04fd

@ -367,6 +367,34 @@ func (lc *LocalClient) DebugAction(ctx context.Context, action string) error {
return nil
}
// DebugPortmap invokes the debug-portmap endpoint, and returns an
// io.ReadCloser that can be used to read the logs that are printed during this
// process.
func (lc *LocalClient) DebugPortmap(ctx context.Context, duration time.Duration, ty, gwSelf string) (io.ReadCloser, error) {
vals := make(url.Values)
vals.Set("duration", duration.String())
vals.Set("type", ty)
if gwSelf != "" {
vals.Set("gateway_and_self", gwSelf)
}
req, err := http.NewRequestWithContext(ctx, "GET", "http://"+apitype.LocalAPIHost+"/localapi/v0/debug-portmap?"+vals.Encode(), nil)
if err != nil {
return nil, err
}
res, err := lc.doLocalRequestNiceError(req)
if err != nil {
return nil, err
}
if res.StatusCode != 200 {
body, _ := io.ReadAll(res.Body)
res.Body.Close()
return nil, fmt.Errorf("HTTP %s: %s", res.Status, body)
}
return res.Body, nil
}
// SetDevStoreKeyValue set a statestore key/value. It's only meant for development.
// The schema (including when keys are re-read) is not a stable interface.
func (lc *LocalClient) SetDevStoreKeyValue(ctx context.Context, key, value string) error {

@ -201,6 +201,18 @@ var debugCmd = &ffcli.Command{
return fs
})(),
},
{
Name: "portmap",
Exec: debugPortmap,
ShortHelp: "run portmap debugging debugging",
FlagSet: (func() *flag.FlagSet {
fs := newFlagSet("portmap")
fs.DurationVar(&debugPortmapArgs.duration, "duration", 5*time.Second, "timeout for port mapping")
fs.StringVar(&debugPortmapArgs.ty, "type", "", `portmap debug type (one of "", "pmp", "pcp", or "upnp")`)
fs.StringVar(&debugPortmapArgs.gwSelf, "gw-self", "", `override gateway and self IP (format: "gatewayIP/selfIP")`)
return fs
})(),
},
},
}
@ -789,3 +801,24 @@ func runCapture(ctx context.Context, args []string) error {
_, err = io.Copy(f, stream)
return err
}
var debugPortmapArgs struct {
duration time.Duration
gwSelf string
ty string
}
func debugPortmap(ctx context.Context, args []string) error {
rc, err := localClient.DebugPortmap(ctx,
debugPortmapArgs.duration,
debugPortmapArgs.ty,
debugPortmapArgs.gwSelf,
)
if err != nil {
return err
}
defer rc.Close()
_, err = io.Copy(os.Stdout, rc)
return err
}

@ -47,7 +47,7 @@ var netcheckArgs struct {
func runNetcheck(ctx context.Context, args []string) error {
c := &netcheck.Client{
UDPBindAddr: envknob.String("TS_DEBUG_NETCHECK_UDP_BIND"),
PortMapper: portmapper.NewClient(logger.WithPrefix(log.Printf, "portmap: "), nil),
PortMapper: portmapper.NewClient(logger.WithPrefix(log.Printf, "portmap: "), nil, nil),
}
if netcheckArgs.verbose {
c.Logf = logger.WithPrefix(log.Printf, "netcheck: ")

@ -14,24 +14,18 @@ import (
"fmt"
"io"
"log"
"net"
"net/http"
"net/http/httptrace"
"net/netip"
"net/url"
"os"
"strings"
"time"
"tailscale.com/derp/derphttp"
"tailscale.com/envknob"
"tailscale.com/ipn"
"tailscale.com/net/interfaces"
"tailscale.com/net/portmapper"
"tailscale.com/net/tshttpproxy"
"tailscale.com/tailcfg"
"tailscale.com/types/key"
"tailscale.com/types/logger"
"tailscale.com/wgengine/monitor"
)
@ -229,95 +223,5 @@ func checkDerp(ctx context.Context, derpRegion string) (err error) {
}
func debugPortmap(ctx context.Context) error {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
portmapper.VerboseLogs = true
switch envknob.String("TS_DEBUG_PORTMAP_TYPE") {
case "":
case "pmp":
portmapper.DisablePCP = true
portmapper.DisableUPnP = true
case "pcp":
portmapper.DisablePMP = true
portmapper.DisableUPnP = true
case "upnp":
portmapper.DisablePCP = true
portmapper.DisablePMP = true
default:
log.Fatalf("TS_DEBUG_PORTMAP_TYPE must be one of pmp,pcp,upnp")
}
done := make(chan bool, 1)
var c *portmapper.Client
logf := log.Printf
c = portmapper.NewClient(logger.WithPrefix(logf, "portmapper: "), func() {
logf("portmapping changed.")
logf("have mapping: %v", c.HaveMapping())
if ext, ok := c.GetCachedMappingOrStartCreatingOne(); ok {
logf("cb: mapping: %v", ext)
select {
case done <- true:
default:
}
return
}
logf("cb: no mapping")
})
linkMon, err := monitor.New(logger.WithPrefix(logf, "monitor: "))
if err != nil {
return err
}
gatewayAndSelfIP := func() (gw, self netip.Addr, ok bool) {
if v := os.Getenv("TS_DEBUG_GW_SELF"); strings.Contains(v, "/") {
i := strings.Index(v, "/")
gw = netip.MustParseAddr(v[:i])
self = netip.MustParseAddr(v[i+1:])
return gw, self, true
}
return linkMon.GatewayAndSelfIP()
}
c.SetGatewayLookupFunc(gatewayAndSelfIP)
gw, selfIP, ok := gatewayAndSelfIP()
if !ok {
logf("no gateway or self IP; %v", linkMon.InterfaceState())
return nil
}
logf("gw=%v; self=%v", gw, selfIP)
uc, err := net.ListenPacket("udp", "0.0.0.0:0")
if err != nil {
return err
}
defer uc.Close()
c.SetLocalPort(uint16(uc.LocalAddr().(*net.UDPAddr).Port))
res, err := c.Probe(ctx)
if err != nil {
return fmt.Errorf("Probe: %v", err)
}
logf("Probe: %+v", res)
if !res.PCP && !res.PMP && !res.UPnP {
logf("no portmapping services available")
return nil
}
if ext, ok := c.GetCachedMappingOrStartCreatingOne(); ok {
logf("mapping: %v", ext)
} else {
logf("no mapping")
}
select {
case <-done:
return nil
case <-ctx.Done():
return ctx.Err()
}
return fmt.Errorf("this flag has been deprecated in favour of 'tailscale debug portmap'")
}

@ -35,6 +35,7 @@ import (
"tailscale.com/ipn/ipnstate"
"tailscale.com/logtail"
"tailscale.com/net/netutil"
"tailscale.com/net/portmapper"
"tailscale.com/tailcfg"
"tailscale.com/tka"
"tailscale.com/types/key"
@ -44,6 +45,7 @@ import (
"tailscale.com/util/httpm"
"tailscale.com/util/mak"
"tailscale.com/version"
"tailscale.com/wgengine/monitor"
)
type localAPIHandler func(*Handler, http.ResponseWriter, *http.Request)
@ -68,6 +70,7 @@ var handler = map[string]localAPIHandler{
"debug-derp-region": (*Handler).serveDebugDERPRegion,
"debug-packet-filter-matches": (*Handler).serveDebugPacketFilterMatches,
"debug-packet-filter-rules": (*Handler).serveDebugPacketFilterRules,
"debug-portmap": (*Handler).serveDebugPortmap,
"debug-capture": (*Handler).serveDebugCapture,
"derpmap": (*Handler).serveDERPMap,
"dev-set-state-store": (*Handler).serveDevSetStateStore,
@ -600,6 +603,135 @@ func (h *Handler) serveDebugPacketFilterMatches(w http.ResponseWriter, r *http.R
enc.Encode(nm.PacketFilter)
}
func (h *Handler) serveDebugPortmap(w http.ResponseWriter, r *http.Request) {
if !h.PermitWrite {
http.Error(w, "debug access denied", http.StatusForbidden)
return
}
w.Header().Set("Content-Type", "text/plain")
dur, err := time.ParseDuration(r.FormValue("duration"))
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
gwSelf := r.FormValue("gateway_and_self")
// Update portmapper debug flags
debugKnobs := &portmapper.DebugKnobs{VerboseLogs: true}
switch r.FormValue("type") {
case "":
case "pmp":
debugKnobs.DisablePCP = true
debugKnobs.DisableUPnP = true
case "pcp":
debugKnobs.DisablePMP = true
debugKnobs.DisableUPnP = true
case "upnp":
debugKnobs.DisablePCP = true
debugKnobs.DisablePMP = true
default:
http.Error(w, "unknown portmap debug type", http.StatusBadRequest)
return
}
var logLock sync.Mutex
logf := func(format string, args ...any) {
if !strings.HasSuffix(format, "\n") {
format = format + "\n"
}
logLock.Lock()
defer logLock.Unlock()
// Write and flush each line to the client so that output is streamed
fmt.Fprintf(w, format, args...)
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
}
ctx, cancel := context.WithTimeout(r.Context(), dur)
defer cancel()
done := make(chan bool, 1)
var c *portmapper.Client
c = portmapper.NewClient(logger.WithPrefix(logf, "portmapper: "), debugKnobs, func() {
logf("portmapping changed.")
logf("have mapping: %v", c.HaveMapping())
if ext, ok := c.GetCachedMappingOrStartCreatingOne(); ok {
logf("cb: mapping: %v", ext)
select {
case done <- true:
default:
}
return
}
logf("cb: no mapping")
})
linkMon, err := monitor.New(logger.WithPrefix(logf, "monitor: "))
if err != nil {
logf("error creating monitor: %v", err)
return
}
gatewayAndSelfIP := func() (gw, self netip.Addr, ok bool) {
if a, b, ok := strings.Cut(gwSelf, "/"); ok {
gw = netip.MustParseAddr(a)
self = netip.MustParseAddr(b)
return gw, self, true
}
return linkMon.GatewayAndSelfIP()
}
c.SetGatewayLookupFunc(gatewayAndSelfIP)
gw, selfIP, ok := gatewayAndSelfIP()
if !ok {
logf("no gateway or self IP; %v", linkMon.InterfaceState())
return
}
logf("gw=%v; self=%v", gw, selfIP)
uc, err := net.ListenPacket("udp", "0.0.0.0:0")
if err != nil {
return
}
defer uc.Close()
c.SetLocalPort(uint16(uc.LocalAddr().(*net.UDPAddr).Port))
res, err := c.Probe(ctx)
if err != nil {
logf("error in Probe: %v", err)
return
}
logf("Probe: %+v", res)
if !res.PCP && !res.PMP && !res.UPnP {
logf("no portmapping services available")
return
}
if ext, ok := c.GetCachedMappingOrStartCreatingOne(); ok {
logf("mapping: %v", ext)
} else {
logf("no mapping")
}
select {
case <-done:
case <-ctx.Done():
if r.Context().Err() == nil {
logf("serveDebugPortmap: context done: %v", ctx.Err())
} else {
h.logf("serveDebugPortmap: context done: %v", ctx.Err())
}
}
}
func (h *Handler) serveComponentDebugLogging(w http.ResponseWriter, r *http.Request) {
if !h.PermitWrite {
http.Error(w, "debug access denied", http.StatusForbidden)

@ -249,7 +249,7 @@ func (d *TestIGD) handlePCPQuery(pkt []byte, src netip.AddrPort) {
func newTestClient(t *testing.T, igd *TestIGD) *Client {
var c *Client
c = NewClient(t.Logf, func() {
c = NewClient(t.Logf, nil, func() {
t.Logf("port map changed")
t.Logf("have mapping: %v", c.HaveMapping())
})

@ -28,16 +28,18 @@ import (
"tailscale.com/util/clientmetric"
)
// Debug knobs for "tailscaled debug --portmap".
var (
// DebugKnobs contains debug configuration that can be provided when creating a
// Client. The zero value is valid for use.
type DebugKnobs struct {
// VerboseLogs tells the Client to print additional debug information
// to its logger.
VerboseLogs bool
// Disable* disables a specific service from mapping.
DisableUPnP bool
DisablePMP bool
DisablePCP bool
)
}
// References:
//
@ -59,6 +61,7 @@ type Client struct {
logf logger.Logf
ipAndGateway func() (gw, ip netip.Addr, ok bool)
onChange func() // or nil
debug DebugKnobs
testPxPPort uint16 // if non-zero, pxpPort to use for tests
testUPnPPort uint16 // if non-zero, uPnPPort to use for tests
@ -150,15 +153,22 @@ func (m *pmpMapping) Release(ctx context.Context) {
// NewClient returns a new portmapping client.
//
// The debug argument allows configuring the behaviour of the portmapper for
// debugging; if nil, a sensible set of defaults will be used.
//
// The optional onChange argument specifies a func to run in a new
// goroutine whenever the port mapping status has changed. If nil,
// it doesn't make a callback.
func NewClient(logf logger.Logf, onChange func()) *Client {
return &Client{
func NewClient(logf logger.Logf, debug *DebugKnobs, onChange func()) *Client {
ret := &Client{
logf: logf,
ipAndGateway: interfaces.LikelyHomeRouterIP,
onChange: onChange,
}
if debug != nil {
ret.debug = *debug
}
return ret
}
// SetGatewayLookupFunc set the func that returns the machine's default gateway IP, and
@ -407,7 +417,7 @@ var wildcardIP = netip.MustParseAddr("0.0.0.0")
// If no mapping is available, the error will be of type
// NoMappingError; see IsNoMappingError.
func (c *Client) createOrGetMapping(ctx context.Context) (external netip.AddrPort, err error) {
if DisableUPnP && DisablePCP && DisablePMP {
if c.debug.DisableUPnP && c.debug.DisablePCP && c.debug.DisablePMP {
return netip.AddrPort{}, NoMappingError{ErrNoPortMappingServices}
}
gw, myIP, ok := c.gatewayAndSelfIP()
@ -437,7 +447,7 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netip.AddrPor
prevPort = m.External().Port()
}
if DisablePCP && DisablePMP {
if c.debug.DisablePCP && c.debug.DisablePMP {
c.mu.Unlock()
if external, ok := c.getUPnPPortMapping(ctx, gw, internalAddr, prevPort); ok {
return external, nil
@ -486,7 +496,7 @@ func (c *Client) createOrGetMapping(ctx context.Context) (external netip.AddrPor
pxpAddr := netip.AddrPortFrom(gw, c.pxpPort())
preferPCP := !DisablePCP && (DisablePMP || (!haveRecentPMP && haveRecentPCP))
preferPCP := !c.debug.DisablePCP && (c.debug.DisablePMP || (!haveRecentPMP && haveRecentPCP))
// Create a mapping, defaulting to PMP unless only PCP was seen recently.
if preferPCP {
@ -712,19 +722,19 @@ func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
// https://github.com/tailscale/tailscale/issues/1001
if c.sawPMPRecently() {
res.PMP = true
} else if !DisablePMP {
} else if !c.debug.DisablePMP {
metricPMPSent.Add(1)
uc.WriteToUDPAddrPort(pmpReqExternalAddrPacket, pxpAddr)
}
if c.sawPCPRecently() {
res.PCP = true
} else if !DisablePCP {
} else if !c.debug.DisablePCP {
metricPCPSent.Add(1)
uc.WriteToUDPAddrPort(pcpAnnounceRequest(myIP), pxpAddr)
}
if c.sawUPnPRecently() {
res.UPnP = true
} else if !DisableUPnP {
} else if !c.debug.DisableUPnP {
// Strictly speaking, you discover UPnP services by sending an
// SSDP query (which uPnPPacket is) to udp/1900 on the SSDP
// multicast address, and then get a flood of responses back

@ -16,7 +16,7 @@ func TestCreateOrGetMapping(t *testing.T) {
if v, _ := strconv.ParseBool(os.Getenv("HIT_NETWORK")); !v {
t.Skip("skipping test without HIT_NETWORK=1")
}
c := NewClient(t.Logf, nil)
c := NewClient(t.Logf, nil, nil)
defer c.Close()
c.SetLocalPort(1234)
for i := 0; i < 2; i++ {
@ -32,7 +32,7 @@ func TestClientProbe(t *testing.T) {
if v, _ := strconv.ParseBool(os.Getenv("HIT_NETWORK")); !v {
t.Skip("skipping test without HIT_NETWORK=1")
}
c := NewClient(t.Logf, nil)
c := NewClient(t.Logf, nil, nil)
defer c.Close()
for i := 0; i < 3; i++ {
if i > 0 {
@ -47,7 +47,7 @@ func TestClientProbeThenMap(t *testing.T) {
if v, _ := strconv.ParseBool(os.Getenv("HIT_NETWORK")); !v {
t.Skip("skipping test without HIT_NETWORK=1")
}
c := NewClient(t.Logf, nil)
c := NewClient(t.Logf, nil, nil)
defer c.Close()
c.SetLocalPort(1234)
res, err := c.Probe(context.Background())

@ -152,8 +152,8 @@ func addAnyPortMapping(
//
// The provided ctx is not retained in the returned upnpClient, but
// its associated HTTP client is (if set via goupnp.WithHTTPClient).
func getUPnPClient(ctx context.Context, logf logger.Logf, gw netip.Addr, meta uPnPDiscoResponse) (client upnpClient, err error) {
if controlknobs.DisableUPnP() || DisableUPnP {
func getUPnPClient(ctx context.Context, logf logger.Logf, debug DebugKnobs, gw netip.Addr, meta uPnPDiscoResponse) (client upnpClient, err error) {
if controlknobs.DisableUPnP() || debug.DisableUPnP {
return nil, nil
}
@ -161,7 +161,7 @@ func getUPnPClient(ctx context.Context, logf logger.Logf, gw netip.Addr, meta uP
return nil, nil
}
if VerboseLogs {
if debug.VerboseLogs {
logf("fetching %v", meta.Location)
}
u, err := url.Parse(meta.Location)
@ -237,9 +237,10 @@ func (c *Client) getUPnPPortMapping(
internal netip.AddrPort,
prevPort uint16,
) (external netip.AddrPort, ok bool) {
if controlknobs.DisableUPnP() || DisableUPnP {
if controlknobs.DisableUPnP() || c.debug.DisableUPnP {
return netip.AddrPort{}, false
}
now := time.Now()
upnp := &upnpMapping{
gw: gw,
@ -257,8 +258,8 @@ func (c *Client) getUPnPPortMapping(
client = oldMapping.client
} else {
ctx := goupnp.WithHTTPClient(ctx, httpClient)
client, err = getUPnPClient(ctx, c.logf, gw, meta)
if VerboseLogs {
client, err = getUPnPClient(ctx, c.logf, c.debug, gw, meta)
if c.debug.VerboseLogs {
c.logf("getUPnPClient: %T, %v", client, err)
}
if err != nil {
@ -278,15 +279,15 @@ func (c *Client) getUPnPPortMapping(
internal.Addr().String(),
time.Second*pmpMapLifetimeSec,
)
if VerboseLogs {
c.logf("addAnyPortMapping: %v, %v", newPort, err)
if c.debug.VerboseLogs {
c.logf("addAnyPortMapping: %v, err=%q", newPort, err)
}
if err != nil {
return netip.AddrPort{}, false
}
// TODO cache this ip somewhere?
extIP, err := client.GetExternalIPAddress(ctx)
if VerboseLogs {
if c.debug.VerboseLogs {
c.logf("client.GetExternalIPAddress: %v, %v", extIP, err)
}
if err != nil {

@ -112,7 +112,7 @@ func TestGetUPnPClient(t *testing.T) {
gw, _ := netip.AddrFromSlice(ts.Listener.Addr().(*net.TCPAddr).IP)
gw = gw.Unmap()
var logBuf tstest.MemLogger
c, err := getUPnPClient(context.Background(), logBuf.Logf, gw, uPnPDiscoResponse{
c, err := getUPnPClient(context.Background(), logBuf.Logf, DebugKnobs{}, gw, uPnPDiscoResponse{
Location: ts.URL + "/rootDesc.xml",
})
if err != nil {

@ -25,7 +25,6 @@ import (
_ "tailscale.com/net/dnsfallback"
_ "tailscale.com/net/interfaces"
_ "tailscale.com/net/netns"
_ "tailscale.com/net/portmapper"
_ "tailscale.com/net/proxymux"
_ "tailscale.com/net/socks5"
_ "tailscale.com/net/tsdial"

@ -25,7 +25,6 @@ import (
_ "tailscale.com/net/dnsfallback"
_ "tailscale.com/net/interfaces"
_ "tailscale.com/net/netns"
_ "tailscale.com/net/portmapper"
_ "tailscale.com/net/proxymux"
_ "tailscale.com/net/socks5"
_ "tailscale.com/net/tsdial"

@ -25,7 +25,6 @@ import (
_ "tailscale.com/net/dnsfallback"
_ "tailscale.com/net/interfaces"
_ "tailscale.com/net/netns"
_ "tailscale.com/net/portmapper"
_ "tailscale.com/net/proxymux"
_ "tailscale.com/net/socks5"
_ "tailscale.com/net/tsdial"

@ -25,7 +25,6 @@ import (
_ "tailscale.com/net/dnsfallback"
_ "tailscale.com/net/interfaces"
_ "tailscale.com/net/netns"
_ "tailscale.com/net/portmapper"
_ "tailscale.com/net/proxymux"
_ "tailscale.com/net/socks5"
_ "tailscale.com/net/tsdial"

@ -33,7 +33,6 @@ import (
_ "tailscale.com/net/dnsfallback"
_ "tailscale.com/net/interfaces"
_ "tailscale.com/net/netns"
_ "tailscale.com/net/portmapper"
_ "tailscale.com/net/proxymux"
_ "tailscale.com/net/socks5"
_ "tailscale.com/net/tsdial"

@ -632,7 +632,7 @@ func NewConn(opts Options) (*Conn, error) {
c.idleFunc = opts.IdleFunc
c.testOnlyPacketListener = opts.TestOnlyPacketListener
c.noteRecvActivity = opts.NoteRecvActivity
c.portMapper = portmapper.NewClient(logger.WithPrefix(c.logf, "portmapper: "), c.onPortMapChanged)
c.portMapper = portmapper.NewClient(logger.WithPrefix(c.logf, "portmapper: "), nil, c.onPortMapChanged)
if opts.LinkMonitor != nil {
c.portMapper.SetGatewayLookupFunc(opts.LinkMonitor.GatewayAndSelfIP)
}

Loading…
Cancel
Save