tailscale/cmd/derpprobe/derpprobe.go

// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause

// The derpprobe binary probes derpers.
package main

import (
	"flag"
	"fmt"
	"html"
	"io"
	"log"
	"net/http"
	"sort"
	"time"

	"tailscale.com/prober"
	"tailscale.com/tsweb"
)

var (
	derpMapURL   = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://)")
	listen       = flag.String("listen", ":8030", "HTTP listen address")
	probeOnce    = flag.Bool("once", false, "probe once and print results, then exit; ignores the listen flag")
	spread       = flag.Bool("spread", true, "whether to spread probing over time")
	interval     = flag.Duration("interval", 15*time.Second, "probe interval")
	meshInterval = flag.Duration("mesh-interval", 15*time.Second, "mesh probe interval")
	stunInterval = flag.Duration("stun-interval", 15*time.Second, "STUN probe interval")
	tlsInterval  = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval")
	bwInterval   = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)")
	bwSize       = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size")
)

func main() {
	flag.Parse()

	p := prober.New().WithSpread(*spread).WithOnce(*probeOnce).WithMetricNamespace("derpprobe")
	opts := []prober.DERPOpt{
		prober.WithMeshProbing(*meshInterval),
		prober.WithSTUNProbing(*stunInterval),
		prober.WithTLSProbing(*tlsInterval),
	}
	if *bwInterval > 0 {
		opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize))
	}
	dp, err := prober.DERP(p, *derpMapURL, opts...)
	if err != nil {
		log.Fatal(err)
	}
	p.Run("derpmap-probe", *interval, nil, dp.ProbeMap)

	if *probeOnce {
		log.Printf("Waiting for all probes (may take up to 1m)")
		p.Wait()

		st := getOverallStatus(p)
		for _, s := range st.good {
			log.Printf("good: %s", s)
		}
		for _, s := range st.bad {
			log.Printf("bad: %s", s)
		}
		return
	}

	mux := http.NewServeMux()
	tsweb.Debugger(mux)
	mux.HandleFunc("/", http.HandlerFunc(serveFunc(p)))
	log.Printf("Listening on %s", *listen)
	log.Fatal(http.ListenAndServe(*listen, mux))
}

type overallStatus struct {
	good, bad []string
}

func (st *overallStatus) addBadf(format string, a ...any) {
	st.bad = append(st.bad, fmt.Sprintf(format, a...))
}

func (st *overallStatus) addGoodf(format string, a ...any) {
	st.good = append(st.good, fmt.Sprintf(format, a...))
}

func getOverallStatus(p *prober.Prober) (o overallStatus) {
	for p, i := range p.ProbeInfo() {
		if i.End.IsZero() {
			// Do not show probes that have not finished yet.
			continue
		}
		if i.Result {
			o.addGoodf("%s: %s", p, i.Latency)
		} else {
			o.addBadf("%s: %s", p, i.Error)
		}
	}

	sort.Strings(o.bad)
	sort.Strings(o.good)
	return
}

func serveFunc(p *prober.Prober) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		st := getOverallStatus(p)
		summary := "All good"
		if (float64(len(st.bad)) / float64(len(st.bad)+len(st.good))) > 0.25 {
			// Returning a 500 allows monitoring this server externally and configuring
			// an alert on HTTP response code.
			w.WriteHeader(500)
			summary = fmt.Sprintf("%d problems", len(st.bad))
		}

		io.WriteString(w, "<html><head><style>.bad { font-weight: bold; color: #700; }</style></head>\n")
		fmt.Fprintf(w, "<body><h1>derp probe</h1>\n%s:<ul>", summary)
		for _, s := range st.bad {
			fmt.Fprintf(w, "<li class=bad>%s</li>\n", html.EscapeString(s))
		}
		for _, s := range st.good {
			fmt.Fprintf(w, "<li>%s</li>\n", html.EscapeString(s))
		}
		io.WriteString(w, "</ul></body></html>\n")
	}
}
all: update copyright and license headers This updates all source files to use a new standard header for copyright and license declaration. Notably, copyright no longer includes a date, and we now use the standard SPDX-License-Identifier header. This commit was done almost entirely mechanically with perl, and then some minimal manual fixes. Updates #6865 Signed-off-by: Will Norris <will@tailscale.com> 2 years ago			`// Copyright (c) Tailscale Inc & AUTHORS`
			`// SPDX-License-Identifier: BSD-3-Clause`
cmd/derper/derpprobe: add derp prober Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com> 3 years ago
			`// The derpprobe binary probes derpers.`
cmd/derpprobe: migrate to the prober framework `prober.DERP` was created in #5988 based on derpprobe. Having used it instead of derpprobe for a few months, I think we have enough confidence that it works and can now migrate derpprobe to use the prober framework and get rid of code duplication. A few notable changes in behaviour: - results of STUN probes over IPv4 and IPv6 are now reported separately; - TLS probing now includes OCSP verification; - probe names in the output have changed; - ability to send Slack notification from the prober has been removed. Instead, the prober now exports metrics in Expvar (/debug/vars) and Prometheus (/debug/varz) formats. Fixes https://github.com/tailscale/corp/issues/8497 Signed-off-by: Anton Tolchanov <anton@tailscale.com> 2 years ago			`package main`
cmd/derper/derpprobe: add derp prober Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com> 3 years ago
			`import (`
			`"flag"`
			`"fmt"`
			`"html"`
			`"io"`
			`"log"`
			`"net/http"`
			`"sort"`
			`"time"`

cmd/derpprobe: migrate to the prober framework `prober.DERP` was created in #5988 based on derpprobe. Having used it instead of derpprobe for a few months, I think we have enough confidence that it works and can now migrate derpprobe to use the prober framework and get rid of code duplication. A few notable changes in behaviour: - results of STUN probes over IPv4 and IPv6 are now reported separately; - TLS probing now includes OCSP verification; - probe names in the output have changed; - ability to send Slack notification from the prober has been removed. Instead, the prober now exports metrics in Expvar (/debug/vars) and Prometheus (/debug/varz) formats. Fixes https://github.com/tailscale/corp/issues/8497 Signed-off-by: Anton Tolchanov <anton@tailscale.com> 2 years ago			`"tailscale.com/prober"`
			`"tailscale.com/tsweb"`
cmd/derper/derpprobe: add derp prober Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com> 3 years ago			`)`

			`var (`
prober: add a DERP bandwidth probe Updates tailscale/corp#17912 Signed-off-by: Anton Tolchanov <anton@tailscale.com> 9 months ago			`derpMapURL = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://)")`
			`listen = flag.String("listen", ":8030", "HTTP listen address")`
			`probeOnce = flag.Bool("once", false, "probe once and print results, then exit; ignores the listen flag")`
			`spread = flag.Bool("spread", true, "whether to spread probing over time")`
			`interval = flag.Duration("interval", 15*time.Second, "probe interval")`
			`meshInterval = flag.Duration("mesh-interval", 15*time.Second, "mesh probe interval")`
			`stunInterval = flag.Duration("stun-interval", 15*time.Second, "STUN probe interval")`
			`tlsInterval = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval")`
			`bwInterval = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)")`
			`bwSize = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size")`
cmd/derper/derpprobe: add derp prober Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com> 3 years ago			`)`

			`func main() {`
			`flag.Parse()`
cmd/derpprobe: don't alert for smaller failures. There is a Cosmic Background level of DERP Unreachability, with individual nodes or regions becoming unreachable briefly and returning a short time later. This is due to hosting provider outages or just the Internet sloshing about. Returning a 500 error pages a human. Being awoken at 3am for a transient error is annoying. For relatively small levels of badness don't page a human, just post to Slack. If the outage impacts a significant fraction of the DERP fleet, then page a human. Signed-off-by: Denton Gentry <dgentry@tailscale.com> 3 years ago
prober: migrate to Prometheus metric library This provides an example of using native Prometheus metrics with tsweb. Prober library seems to be the only user of PrometheusVar, so I am removing support for it in tsweb. Updates https://github.com/tailscale/corp/issues/10205 Signed-off-by: Anton Tolchanov <anton@tailscale.com> 2 years ago			`p := prober.New().WithSpread(spread).WithOnce(probeOnce).WithMetricNamespace("derpprobe")`
prober: add a DERP bandwidth probe Updates tailscale/corp#17912 Signed-off-by: Anton Tolchanov <anton@tailscale.com> 9 months ago			`opts := []prober.DERPOpt{`
			`prober.WithMeshProbing(*meshInterval),`
			`prober.WithSTUNProbing(*stunInterval),`
			`prober.WithTLSProbing(*tlsInterval),`
			`}`
			`if *bwInterval > 0 {`
			`opts = append(opts, prober.WithBandwidthProbing(bwInterval, bwSize))`
			`}`
			`dp, err := prober.DERP(p, *derpMapURL, opts...)`
cmd/derpprobe: migrate to the prober framework `prober.DERP` was created in #5988 based on derpprobe. Having used it instead of derpprobe for a few months, I think we have enough confidence that it works and can now migrate derpprobe to use the prober framework and get rid of code duplication. A few notable changes in behaviour: - results of STUN probes over IPv4 and IPv6 are now reported separately; - TLS probing now includes OCSP verification; - probe names in the output have changed; - ability to send Slack notification from the prober has been removed. Instead, the prober now exports metrics in Expvar (/debug/vars) and Prometheus (/debug/varz) formats. Fixes https://github.com/tailscale/corp/issues/8497 Signed-off-by: Anton Tolchanov <anton@tailscale.com> 2 years ago			`if err != nil {`
			`log.Fatal(err)`
			`}`
			`p.Run("derpmap-probe", *interval, nil, dp.ProbeMap)`
cmd/derpprobe: don't alert for smaller failures. There is a Cosmic Background level of DERP Unreachability, with individual nodes or regions becoming unreachable briefly and returning a short time later. This is due to hosting provider outages or just the Internet sloshing about. Returning a 500 error pages a human. Being awoken at 3am for a transient error is annoying. For relatively small levels of badness don't page a human, just post to Slack. If the outage impacts a significant fraction of the DERP fleet, then page a human. Signed-off-by: Denton Gentry <dgentry@tailscale.com> 3 years ago
cmd/derpprobe: add -once flag for one-off CLI diagnostics Updates #6478 Signed-off-by: James Tucker <james@tailscale.com> 2 years ago			`if *probeOnce {`
cmd/derpprobe: migrate to the prober framework `prober.DERP` was created in #5988 based on derpprobe. Having used it instead of derpprobe for a few months, I think we have enough confidence that it works and can now migrate derpprobe to use the prober framework and get rid of code duplication. A few notable changes in behaviour: - results of STUN probes over IPv4 and IPv6 are now reported separately; - TLS probing now includes OCSP verification; - probe names in the output have changed; - ability to send Slack notification from the prober has been removed. Instead, the prober now exports metrics in Expvar (/debug/vars) and Prometheus (/debug/varz) formats. Fixes https://github.com/tailscale/corp/issues/8497 Signed-off-by: Anton Tolchanov <anton@tailscale.com> 2 years ago			`log.Printf("Waiting for all probes (may take up to 1m)")`
			`p.Wait()`

			`st := getOverallStatus(p)`
cmd/derpprobe: add -once flag for one-off CLI diagnostics Updates #6478 Signed-off-by: James Tucker <james@tailscale.com> 2 years ago			`for _, s := range st.good {`
			`log.Printf("good: %s", s)`
			`}`
			`for _, s := range st.bad {`
			`log.Printf("bad: %s", s)`
			`}`
			`return`
			`}`

cmd/derpprobe: migrate to the prober framework `prober.DERP` was created in #5988 based on derpprobe. Having used it instead of derpprobe for a few months, I think we have enough confidence that it works and can now migrate derpprobe to use the prober framework and get rid of code duplication. A few notable changes in behaviour: - results of STUN probes over IPv4 and IPv6 are now reported separately; - TLS probing now includes OCSP verification; - probe names in the output have changed; - ability to send Slack notification from the prober has been removed. Instead, the prober now exports metrics in Expvar (/debug/vars) and Prometheus (/debug/varz) formats. Fixes https://github.com/tailscale/corp/issues/8497 Signed-off-by: Anton Tolchanov <anton@tailscale.com> 2 years ago			`mux := http.NewServeMux()`
			`tsweb.Debugger(mux)`
			`mux.HandleFunc("/", http.HandlerFunc(serveFunc(p)))`
prober: add a DERP bandwidth probe Updates tailscale/corp#17912 Signed-off-by: Anton Tolchanov <anton@tailscale.com> 9 months ago			`log.Printf("Listening on %s", *listen)`
cmd/derpprobe: migrate to the prober framework `prober.DERP` was created in #5988 based on derpprobe. Having used it instead of derpprobe for a few months, I think we have enough confidence that it works and can now migrate derpprobe to use the prober framework and get rid of code duplication. A few notable changes in behaviour: - results of STUN probes over IPv4 and IPv6 are now reported separately; - TLS probing now includes OCSP verification; - probe names in the output have changed; - ability to send Slack notification from the prober has been removed. Instead, the prober now exports metrics in Expvar (/debug/vars) and Prometheus (/debug/varz) formats. Fixes https://github.com/tailscale/corp/issues/8497 Signed-off-by: Anton Tolchanov <anton@tailscale.com> 2 years ago			`log.Fatal(http.ListenAndServe(*listen, mux))`
cmd/derpprobe: check derper TLS certs too Change-Id: If8c48e012b294570ebbb1a46bacdc58fafbfbcc5 Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com> 3 years ago			`}`

cmd/derper/derpprobe: add derp prober Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com> 3 years ago			`type overallStatus struct {`
			`good, bad []string`
			`}`

all: use any instead of interface{} My favorite part of generics. Signed-off-by: Josh Bleecher Snyder <josh@tailscale.com> 3 years ago			`func (st *overallStatus) addBadf(format string, a ...any) {`
cmd/derper/derpprobe: add derp prober Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com> 3 years ago			`st.bad = append(st.bad, fmt.Sprintf(format, a...))`
			`}`

all: use any instead of interface{} My favorite part of generics. Signed-off-by: Josh Bleecher Snyder <josh@tailscale.com> 3 years ago			`func (st *overallStatus) addGoodf(format string, a ...any) {`
cmd/derper/derpprobe: add derp prober Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com> 3 years ago			`st.good = append(st.good, fmt.Sprintf(format, a...))`
			`}`

cmd/derpprobe: migrate to the prober framework `prober.DERP` was created in #5988 based on derpprobe. Having used it instead of derpprobe for a few months, I think we have enough confidence that it works and can now migrate derpprobe to use the prober framework and get rid of code duplication. A few notable changes in behaviour: - results of STUN probes over IPv4 and IPv6 are now reported separately; - TLS probing now includes OCSP verification; - probe names in the output have changed; - ability to send Slack notification from the prober has been removed. Instead, the prober now exports metrics in Expvar (/debug/vars) and Prometheus (/debug/varz) formats. Fixes https://github.com/tailscale/corp/issues/8497 Signed-off-by: Anton Tolchanov <anton@tailscale.com> 2 years ago			`func getOverallStatus(p *prober.Prober) (o overallStatus) {`
			`for p, i := range p.ProbeInfo() {`
			`if i.End.IsZero() {`
			`// Do not show probes that have not finished yet.`
cmd/derpprobe: check derper TLS certs too Change-Id: If8c48e012b294570ebbb1a46bacdc58fafbfbcc5 Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com> 3 years ago			`continue`
			`}`
cmd/derpprobe: migrate to the prober framework `prober.DERP` was created in #5988 based on derpprobe. Having used it instead of derpprobe for a few months, I think we have enough confidence that it works and can now migrate derpprobe to use the prober framework and get rid of code duplication. A few notable changes in behaviour: - results of STUN probes over IPv4 and IPv6 are now reported separately; - TLS probing now includes OCSP verification; - probe names in the output have changed; - ability to send Slack notification from the prober has been removed. Instead, the prober now exports metrics in Expvar (/debug/vars) and Prometheus (/debug/varz) formats. Fixes https://github.com/tailscale/corp/issues/8497 Signed-off-by: Anton Tolchanov <anton@tailscale.com> 2 years ago			`if i.Result {`
			`o.addGoodf("%s: %s", p, i.Latency)`
			`} else {`
			`o.addBadf("%s: %s", p, i.Error)`
cmd/derpprobe: check derper TLS certs too Change-Id: If8c48e012b294570ebbb1a46bacdc58fafbfbcc5 Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com> 3 years ago			`}`
			`}`

cmd/derpprobe: migrate to the prober framework `prober.DERP` was created in #5988 based on derpprobe. Having used it instead of derpprobe for a few months, I think we have enough confidence that it works and can now migrate derpprobe to use the prober framework and get rid of code duplication. A few notable changes in behaviour: - results of STUN probes over IPv4 and IPv6 are now reported separately; - TLS probing now includes OCSP verification; - probe names in the output have changed; - ability to send Slack notification from the prober has been removed. Instead, the prober now exports metrics in Expvar (/debug/vars) and Prometheus (/debug/varz) formats. Fixes https://github.com/tailscale/corp/issues/8497 Signed-off-by: Anton Tolchanov <anton@tailscale.com> 2 years ago			`sort.Strings(o.bad)`
			`sort.Strings(o.good)`
cmd/derper/derpprobe: add derp prober Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com> 3 years ago			`return`
			`}`

cmd/derpprobe: migrate to the prober framework `prober.DERP` was created in #5988 based on derpprobe. Having used it instead of derpprobe for a few months, I think we have enough confidence that it works and can now migrate derpprobe to use the prober framework and get rid of code duplication. A few notable changes in behaviour: - results of STUN probes over IPv4 and IPv6 are now reported separately; - TLS probing now includes OCSP verification; - probe names in the output have changed; - ability to send Slack notification from the prober has been removed. Instead, the prober now exports metrics in Expvar (/debug/vars) and Prometheus (/debug/varz) formats. Fixes https://github.com/tailscale/corp/issues/8497 Signed-off-by: Anton Tolchanov <anton@tailscale.com> 2 years ago			`func serveFunc(p prober.Prober) func(w http.ResponseWriter, r http.Request) {`
			`return func(w http.ResponseWriter, r *http.Request) {`
			`st := getOverallStatus(p)`
			`summary := "All good"`
			`if (float64(len(st.bad)) / float64(len(st.bad)+len(st.good))) > 0.25 {`
			`// Returning a 500 allows monitoring this server externally and configuring`
			`// an alert on HTTP response code.`
			`w.WriteHeader(500)`
			`summary = fmt.Sprintf("%d problems", len(st.bad))`
cmd/derper/derpprobe: add derp prober Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com> 3 years ago			`}`
cmd/derpprobe: check derper TLS certs too Change-Id: If8c48e012b294570ebbb1a46bacdc58fafbfbcc5 Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com> 3 years ago
cmd/derpprobe: migrate to the prober framework `prober.DERP` was created in #5988 based on derpprobe. Having used it instead of derpprobe for a few months, I think we have enough confidence that it works and can now migrate derpprobe to use the prober framework and get rid of code duplication. A few notable changes in behaviour: - results of STUN probes over IPv4 and IPv6 are now reported separately; - TLS probing now includes OCSP verification; - probe names in the output have changed; - ability to send Slack notification from the prober has been removed. Instead, the prober now exports metrics in Expvar (/debug/vars) and Prometheus (/debug/varz) formats. Fixes https://github.com/tailscale/corp/issues/8497 Signed-off-by: Anton Tolchanov <anton@tailscale.com> 2 years ago			`io.WriteString(w, "<html><head><style>.bad { font-weight: bold; color: #700; }</style></head>\n")`
			`fmt.Fprintf(w, "<body><h1>derp probe</h1>\n%s:<ul>", summary)`
			`for _, s := range st.bad {`
			`fmt.Fprintf(w, "<li class=bad>%s</li>\n", html.EscapeString(s))`
cmd/derper/derpprobe: add derp prober Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com> 3 years ago			`}`
cmd/derpprobe: migrate to the prober framework `prober.DERP` was created in #5988 based on derpprobe. Having used it instead of derpprobe for a few months, I think we have enough confidence that it works and can now migrate derpprobe to use the prober framework and get rid of code duplication. A few notable changes in behaviour: - results of STUN probes over IPv4 and IPv6 are now reported separately; - TLS probing now includes OCSP verification; - probe names in the output have changed; - ability to send Slack notification from the prober has been removed. Instead, the prober now exports metrics in Expvar (/debug/vars) and Prometheus (/debug/varz) formats. Fixes https://github.com/tailscale/corp/issues/8497 Signed-off-by: Anton Tolchanov <anton@tailscale.com> 2 years ago			`for _, s := range st.good {`
			`fmt.Fprintf(w, "<li>%s</li>\n", html.EscapeString(s))`
cmd/derper/derpprobe: add derp prober Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com> 3 years ago			`}`
cmd/derpprobe: migrate to the prober framework `prober.DERP` was created in #5988 based on derpprobe. Having used it instead of derpprobe for a few months, I think we have enough confidence that it works and can now migrate derpprobe to use the prober framework and get rid of code duplication. A few notable changes in behaviour: - results of STUN probes over IPv4 and IPv6 are now reported separately; - TLS probing now includes OCSP verification; - probe names in the output have changed; - ability to send Slack notification from the prober has been removed. Instead, the prober now exports metrics in Expvar (/debug/vars) and Prometheus (/debug/varz) formats. Fixes https://github.com/tailscale/corp/issues/8497 Signed-off-by: Anton Tolchanov <anton@tailscale.com> 2 years ago			`io.WriteString(w, "</ul></body></html>\n")`
cmd/derper/derpprobe: add derp prober Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com> 3 years ago			`}`
			`}`