cmd/tailscale/cli: add 'tailscale metrics' command

- `tailscale metrics print`: to show metric values in console
- `tailscale metrics write`: to write metrics to a file (with a tempfile
  & rename dance, which is atomic on Unix).

Also, remove the `TS_DEBUG_USER_METRICS` envknob as we are getting
more confident in these metrics.

Updates tailscale/corp#22075

Signed-off-by: Anton Tolchanov <anton@tailscale.com>

cmd/tailscale/cli/cli.go

@@ -189,6 +189,7 @@ change in the future.
 			ipCmd,
 			dnsCmd,
 			statusCmd,
+			metricsCmd,
 			pingCmd,
 			ncCmd,
 			sshCmd,

cmd/tailscale/cli/metrics.go

@@ -0,0 +1,88 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package cli
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/peterbourgon/ff/v3/ffcli"
+	"tailscale.com/atomicfile"
+)
+
+var metricsCmd = &ffcli.Command{
+	Name:      "metrics",
+	ShortHelp: "Show Tailscale metrics",
+	LongHelp: strings.TrimSpace(`
+
+The 'tailscale metrics' command shows Tailscale user-facing metrics (as opposed
+to internal metrics printed by 'tailscale debug metrics').
+
+For more information about Tailscale metrics, refer to
+https://tailscale.com/s/client-metrics
+
+`),
+	ShortUsage: "tailscale metrics <subcommand> [flags]",
+	UsageFunc:  usageFuncNoDefaultValues,
+	Exec:       runMetricsNoSubcommand,
+	Subcommands: []*ffcli.Command{
+		{
+			Name:       "print",
+			ShortUsage: "tailscale metrics print",
+			Exec:       runMetricsPrint,
+			ShortHelp:  "Prints current metric values in the Prometheus text exposition format",
+		},
+		{
+			Name:       "write",
+			ShortUsage: "tailscale metrics write <path>",
+			Exec:       runMetricsWrite,
+			ShortHelp:  "Writes metric values to a file",
+			LongHelp: strings.TrimSpace(`
+
+The 'tailscale metrics write' command writes metric values to a text file provided as its
+only argument. It's meant to be used alongside Prometheus node exporter, allowing Tailscale
+metrics to be consumed and exported by the textfile collector.
+
+As an example, to export Tailscale metrics on an Ubuntu system running node exporter, you
+can regularly run 'tailscale metrics write /var/lib/prometheus/node-exporter/tailscaled.prom'
+using cron or a systemd timer.
+
+	`),
+		},
+	},
+}
+
+// runMetricsNoSubcommand prints metric values if no subcommand is specified.
+func runMetricsNoSubcommand(ctx context.Context, args []string) error {
+	if len(args) > 0 {
+		return fmt.Errorf("tailscale metrics: unknown subcommand: %s", args[0])
+	}
+
+	return runMetricsPrint(ctx, args)
+}
+
+// runMetricsPrint prints metric values to stdout.
+func runMetricsPrint(ctx context.Context, args []string) error {
+	out, err := localClient.UserMetrics(ctx)
+	if err != nil {
+		return err
+	}
+	Stdout.Write(out)
+	return nil
+}
+
+// runMetricsWrite writes metric values to a file.
+func runMetricsWrite(ctx context.Context, args []string) error {
+	if len(args) != 1 {
+		return errors.New("usage: tailscale metrics write <path>")
+	}
+	path := args[0]
+	out, err := localClient.UserMetrics(ctx)
+	if err != nil {
+		return err
+	}
+	return atomicfile.WriteFile(path, out, 0644)
+}

ipn/localapi/localapi.go

@@ -62,7 +62,6 @@ import (
 	"tailscale.com/util/osdiag"
 	"tailscale.com/util/progresstracking"
 	"tailscale.com/util/rands"
-	"tailscale.com/util/testenv"
 	"tailscale.com/version"
 	"tailscale.com/wgengine/magicsock"
 )
@@ -570,15 +569,9 @@ func (h *Handler) serveMetrics(w http.ResponseWriter, r *http.Request) {
 	clientmetric.WritePrometheusExpositionFormat(w)
 }
 
-// TODO(kradalby): Remove this once we have landed on a final set of
+// serveUserMetrics returns user-facing metrics in Prometheus text
-// metrics to export to clients and consider the metrics stable.
+// exposition format.
-var debugUsermetricsEndpoint = envknob.RegisterBool("TS_DEBUG_USER_METRICS")
-
 func (h *Handler) serveUserMetrics(w http.ResponseWriter, r *http.Request) {
-	if !testenv.InTest() && !debugUsermetricsEndpoint() {
-		http.Error(w, "usermetrics debug flag not enabled", http.StatusForbidden)
-		return
-	}
 	h.b.UserMetricsRegistry().Handler(w, r)
 }