cmd/tailscale/cli: add --watch flag to "debug metrics" subcommand

This adds a new --watch flag that prints out a block of metric changes
every second, if anything changed.

Example output:

magicsock_disco_recv_ping    +1 => 254
magicsock_disco_recv_pong    +1 => 218
magicsock_disco_recv_udp     +2 => 472
magicsock_disco_send_udp     +2 => 536
magicsock_disco_sent_udp     +2 => 536
magicsock_recv_data_ipv6     +1 => 82
magicsock_send_data          +1 => 86
magicsock_send_udp           +3 => 620

magicsock_recv_data_ipv6    +1 => 83
magicsock_send_data         +1 => 87
magicsock_send_udp          +1 => 621

magicsock_disco_recv_ping    +1 => 255
magicsock_disco_recv_pong    +1 => 219
magicsock_disco_recv_udp     +2 => 474
magicsock_disco_send_udp     +2 => 538
magicsock_disco_sent_udp     +2 => 538
magicsock_recv_data_ipv6     +1 => 84
magicsock_send_data          +1 => 88
magicsock_send_udp           +3 => 624

magicsock_recv_data_ipv6    +1 => 85
magicsock_send_data         +1 => 89
magicsock_send_udp          +1 => 625

controlclient_map_response_map          +1 => 207
controlclient_map_response_map_delta    +1 => 204
controlclient_map_response_message      +1 => 275
magicsock_disco_recv_ping               +3 => 258
magicsock_disco_recv_pong               +2 => 221
magicsock_disco_recv_udp                +5 => 479
magicsock_disco_send_derp               +1 => 6
magicsock_disco_send_udp                +7 => 545
magicsock_disco_sent_derp               +1 => 6
magicsock_disco_sent_udp                +7 => 545
magicsock_recv_data_ipv6                +1 => 86
magicsock_send_data                     +1 => 90
magicsock_send_derp                     +1 => 12
magicsock_send_derp_queued              +1 => 12
magicsock_send_udp                      +8 => 633

Updates #3307

Change-Id: I5ac2511e3ad24fa1e6ea958c3946fecebe4f79a7
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

cmd/tailscale/cli/debug.go

@@ -5,6 +5,8 @@
 package cli
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -14,7 +16,9 @@ import (
 	"log"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"tailscale.com/client/tailscale"
@@ -50,6 +54,11 @@ var debugCmd = &ffcli.Command{
 			Name:      "metrics",
 			Exec:      runDaemonMetrics,
 			ShortHelp: "print tailscaled's metrics",
+			FlagSet: (func() *flag.FlagSet {
+				fs := newFlagSet("metrics")
+				fs.BoolVar(&metricsArgs.watch, "watch", false, "print JSON dump of delta values")
+				return fs
+			})(),
 		},
 		{
 			Name:      "env",
@@ -251,11 +260,59 @@ func runDaemonGoroutines(ctx context.Context, args []string) error {
 	return nil
 }
 
+var metricsArgs struct {
+	watch bool
+}
+
 func runDaemonMetrics(ctx context.Context, args []string) error {
-	out, err := tailscale.DaemonMetrics(ctx)
-	if err != nil {
-		return err
+	last := map[string]int64{}
+	for {
+		out, err := tailscale.DaemonMetrics(ctx)
+		if err != nil {
+			return err
+		}
+		if !metricsArgs.watch {
+			Stdout.Write(out)
+			return nil
+		}
+		bs := bufio.NewScanner(bytes.NewReader(out))
+		type change struct {
+			name     string
+			from, to int64
+		}
+		var changes []change
+		var maxNameLen int
+		for bs.Scan() {
+			line := bytes.TrimSpace(bs.Bytes())
+			if len(line) == 0 || line[0] == '#' {
+				continue
+			}
+			f := strings.Fields(string(line))
+			if len(f) != 2 {
+				continue
+			}
+			name := f[0]
+			n, _ := strconv.ParseInt(f[1], 10, 64)
+			prev, ok := last[name]
+			if ok && prev == n {
+				continue
+			}
+			last[name] = n
+			if !ok {
+				continue
+			}
+			changes = append(changes, change{name, prev, n})
+			if len(name) > maxNameLen {
+				maxNameLen = len(name)
+			}
+		}
+		if len(changes) > 0 {
+			format := fmt.Sprintf("%%-%ds %%+5d => %%v\n", maxNameLen)
+			for _, c := range changes {
+				fmt.Fprintf(Stdout, format, c.name, c.to-c.from, c.to)
+			}
+			io.WriteString(Stdout, "\n")
+		}
+		time.Sleep(time.Second)
 	}
-	Stdout.Write(out)
-	return nil
 }