mirror of https://github.com/tailscale/tailscale/
cmd/{containerboot,k8s-operator},k8s-operator: new options to expose user metrics (#14035)
containerboot:
Adds 3 new environment variables for containerboot, `TS_LOCAL_ADDR_PORT` (default
`"${POD_IP}:9002"`), `TS_METRICS_ENABLED` (default `false`), and `TS_DEBUG_ADDR_PORT`
(default `""`), to configure metrics and debug endpoints. In a follow-up PR, the
health check endpoint will be updated to use the `TS_LOCAL_ADDR_PORT` if
`TS_HEALTHCHECK_ADDR_PORT` hasn't been set.
Users previously only had access to internal debug metrics (which are unstable
and not recommended) via passing the `--debug` flag to tailscaled, but can now
set `TS_METRICS_ENABLED=true` to expose the stable metrics documented at
https://tailscale.com/kb/1482/client-metrics at `/metrics` on the addr/port
specified by `TS_LOCAL_ADDR_PORT`.
Users can also now configure a debug endpoint more directly via the
`TS_DEBUG_ADDR_PORT` environment variable. This is not recommended for production
use, but exposes an internal set of debug metrics and pprof endpoints.
operator:
The `ProxyClass` CRD's `.spec.metrics.enable` field now enables serving the
stable user metrics documented at https://tailscale.com/kb/1482/client-metrics
at `/metrics` on the same "metrics" container port that debug metrics were
previously served on. To smooth the transition for anyone relying on the way the
operator previously consumed this field, we also _temporarily_ serve tailscaled's
internal debug metrics on the same `/debug/metrics` path as before, until 1.82.0
when debug metrics will be turned off by default even if `.spec.metrics.enable`
is set. At that point, anyone who wishes to continue using the internal debug
metrics (not recommended) will need to set the new `ProxyClass` field
`.spec.statefulSet.pod.tailscaleContainer.debug.enable`.
Users who wish to opt out of the transitional behaviour, where enabling
`.spec.metrics.enable` also enables debug metrics, can set
`.spec.statefulSet.pod.tailscaleContainer.debug.enable` to false (recommended).
Separately but related, the operator will no longer specify a host port for the
"metrics" container port definition. This caused scheduling conflicts when k8s
needs to schedule more than one proxy per node, and was not necessary for allowing
the pod's port to be exposed to prometheus scrapers.
Updates #11292
---------
Co-authored-by: Kristoffer Dalby <kristoffer@tailscale.com>
Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>
pull/14122/merge
Tom Proctor
1 day ago
committed by
GitHub
parent
c59ab6baac
commit
74d4652144
@ -0,0 +1,91 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build linux
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
|
||||
"tailscale.com/client/tailscale"
|
||||
"tailscale.com/client/tailscale/apitype"
|
||||
)
|
||||
|
||||
// metrics is a simple metrics HTTP server, if enabled it forwards requests to
|
||||
// the tailscaled's LocalAPI usermetrics endpoint at /localapi/v0/usermetrics.
|
||||
type metrics struct {
|
||||
debugEndpoint string
|
||||
lc *tailscale.LocalClient
|
||||
}
|
||||
|
||||
func proxy(w http.ResponseWriter, r *http.Request, url string, do func(*http.Request) (*http.Response, error)) {
|
||||
req, err := http.NewRequestWithContext(r.Context(), r.Method, url, r.Body)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("failed to construct request: %s", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
req.Header = r.Header.Clone()
|
||||
|
||||
resp, err := do(req)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("failed to proxy request: %s", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
for key, val := range resp.Header {
|
||||
for _, v := range val {
|
||||
w.Header().Add(key, v)
|
||||
}
|
||||
}
|
||||
if _, err := io.Copy(w, resp.Body); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *metrics) handleMetrics(w http.ResponseWriter, r *http.Request) {
|
||||
localAPIURL := "http://" + apitype.LocalAPIHost + "/localapi/v0/usermetrics"
|
||||
proxy(w, r, localAPIURL, m.lc.DoLocalRequest)
|
||||
}
|
||||
|
||||
func (m *metrics) handleDebug(w http.ResponseWriter, r *http.Request) {
|
||||
if m.debugEndpoint == "" {
|
||||
http.Error(w, "debug endpoint not configured", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
debugURL := "http://" + m.debugEndpoint + r.URL.Path
|
||||
proxy(w, r, debugURL, http.DefaultClient.Do)
|
||||
}
|
||||
|
||||
// runMetrics runs a simple HTTP metrics endpoint at <addr>/metrics, forwarding
|
||||
// requests to tailscaled's /localapi/v0/usermetrics API.
|
||||
//
|
||||
// In 1.78.x and 1.80.x, it also proxies debug paths to tailscaled's debug
|
||||
// endpoint if configured to ease migration for a breaking change serving user
|
||||
// metrics instead of debug metrics on the "metrics" port.
|
||||
func runMetrics(addr string, m *metrics) {
|
||||
ln, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
log.Fatalf("error listening on the provided metrics endpoint address %q: %v", addr, err)
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /metrics", m.handleMetrics)
|
||||
mux.HandleFunc("/debug/", m.handleDebug) // TODO(tomhjp): Remove for 1.82.0 release.
|
||||
|
||||
log.Printf("Running metrics endpoint at %s/metrics", addr)
|
||||
ms := &http.Server{Handler: mux}
|
||||
|
||||
go func() {
|
||||
if err := ms.Serve(ln); err != nil {
|
||||
log.Fatalf("failed running metrics endpoint: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
Loading…
Reference in New Issue