diff --git a/control/controlclient/debug.go b/control/controlclient/debug.go new file mode 100644 index 000000000..fe9d3450a --- /dev/null +++ b/control/controlclient/debug.go @@ -0,0 +1,69 @@ +// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package controlclient + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "log" + "net/http" + "regexp" + "runtime" + "strconv" + "time" +) + +func dumpGoroutinesToURL(c *http.Client, targetURL string) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + zbuf := new(bytes.Buffer) + zw := gzip.NewWriter(zbuf) + zw.Write(scrubbedGoroutineDump()) + zw.Close() + + req, err := http.NewRequestWithContext(ctx, "PUT", targetURL, zbuf) + if err != nil { + log.Printf("dumpGoroutinesToURL: %v", err) + return + } + req.Header.Set("Content-Encoding", "gzip") + t0 := time.Now() + _, err = c.Do(req) + d := time.Since(t0).Round(time.Millisecond) + if err != nil { + log.Printf("dumpGoroutinesToURL error: %v to %v (after %v)", err, targetURL, d) + } else { + log.Printf("dumpGoroutinesToURL complete to %v (after %v)", targetURL, d) + } +} + +var reHexArgs = regexp.MustCompile(`\b0x[0-9a-f]+\b`) + +// scrubbedGoroutineDump returns the list of all current goroutines, but with the actual +// values of arguments scrubbed out, lest it contain some private key material. +func scrubbedGoroutineDump() []byte { + buf := make([]byte, 1<<20) + buf = buf[:runtime.Stack(buf, true)] + + saw := map[string][]byte{} // "0x123" => "v1%3" (unique value 1 and its value mod 8) + return reHexArgs.ReplaceAllFunc(buf, func(in []byte) []byte { + if string(in) == "0x0" { + return in + } + if v, ok := saw[string(in)]; ok { + return v + } + u64, err := strconv.ParseUint(string(in[2:]), 16, 64) + if err != nil { + return []byte("??") + } + v := []byte(fmt.Sprintf("v%d%%%d", len(saw)+1, u64%8)) + saw[string(in)] = v + return v + }) +} diff --git a/control/controlclient/debug_test.go b/control/controlclient/debug_test.go new file mode 100644 index 000000000..8e928789a --- /dev/null +++ b/control/controlclient/debug_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package controlclient + +import "testing" + +func TestScrubbedGoroutineDump(t *testing.T) { + t.Logf("Got:\n%s\n", scrubbedGoroutineDump()) +} diff --git a/control/controlclient/direct.go b/control/controlclient/direct.go index 6aab70760..b38552e2d 100644 --- a/control/controlclient/direct.go +++ b/control/controlclient/direct.go @@ -702,6 +702,9 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm if resp.Debug.LogHeapPprof { go logheap.LogHeap(resp.Debug.LogHeapURL) } + if resp.Debug.GoroutineDumpURL != "" { + go dumpGoroutinesToURL(c.httpc, resp.Debug.GoroutineDumpURL) + } setControlAtomic(&controlUseDERPRoute, resp.Debug.DERPRoute) setControlAtomic(&controlTrimWGConfig, resp.Debug.TrimWGConfig) } diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 6ad7b0c19..4907feb51 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -824,6 +824,10 @@ type Debug struct { // DisableSubnetsIfPAC controls whether subnet routers should be // disabled if WPAD is present on the network. DisableSubnetsIfPAC opt.Bool `json:",omitempty"` + + // GoroutineDumpURL, if non-empty, requests that the client do + // a one-time dump of its active goroutines to the given URL. + GoroutineDumpURL string `json:",omitempty"` } func (k MachineKey) String() string { return fmt.Sprintf("mkey:%x", k[:]) }