From cc1761e8d272f5ddf326d35de8a647c6cbf6a8c7 Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 22 Sep 2025 13:55:16 +0100 Subject: [PATCH] cmd/k8s-operator: send operator logs to tailscale (#17110) This commit modifies the k8s operator to wrap its logger using the logtail logger provided via the tsnet server. This causes any logs written by the operator to make their way to Tailscale in the same fashion as wireguard logs to be used by support. This functionality can also be opted-out of entirely using the "TS_NO_LOGS_NO_SUPPORT" environment variable. Updates https://github.com/tailscale/corp/issues/32037 Signed-off-by: David Bond --- cmd/k8s-operator/logger.go | 26 ++++++++++++++++++++++++++ cmd/k8s-operator/operator.go | 9 +++++++++ cmd/k8s-operator/sts.go | 19 +++++++++---------- tsnet/tsnet.go | 8 ++++---- 4 files changed, 48 insertions(+), 14 deletions(-) create mode 100644 cmd/k8s-operator/logger.go diff --git a/cmd/k8s-operator/logger.go b/cmd/k8s-operator/logger.go new file mode 100644 index 000000000..46b1fc0c8 --- /dev/null +++ b/cmd/k8s-operator/logger.go @@ -0,0 +1,26 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "io" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + kzap "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +// wrapZapCore returns a zapcore.Core implementation that splits the core chain using zapcore.NewTee. This causes +// logs to be simultaneously written to both the original core and the provided io.Writer implementation. +func wrapZapCore(core zapcore.Core, writer io.Writer) zapcore.Core { + encoder := &kzap.KubeAwareEncoder{ + Encoder: zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + } + + // We use a tee logger here so that logs are written to stdout/stderr normally while at the same time being + // sent upstream. + return zapcore.NewTee(core, zapcore.NewCore(encoder, zapcore.AddSync(writer), zap.DebugLevel)) +} diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 76d2df51d..1d988eb03 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -44,6 +44,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager/signals" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/envknob" "tailscale.com/client/local" "tailscale.com/client/tailscale" @@ -133,6 +134,14 @@ func main() { } }() } + + // Operator log uploads can be opted-out using the "TS_NO_LOGS_NO_SUPPORT" environment variable. + if !envknob.NoLogsNoSupport() { + zlog = zlog.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { + return wrapZapCore(core, s.LogtailWriter()) + })) + } + rOpts := reconcilerOpts{ log: zlog, tsServer: s, diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 9a87d2643..80c9ca806 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -439,12 +439,12 @@ func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, logger *z } if orig != nil && !apiequality.Semantic.DeepEqual(latest, orig) { - logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig)) + logger.With("config", sanitizeConfig(latestConfig)).Debugf("patching the existing proxy Secret") if err = a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil { return nil, err } } else { - logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig)) + logger.With("config", sanitizeConfig(latestConfig)).Debugf("creating a new Secret for the proxy") if err = a.Create(ctx, secret); err != nil { return nil, err } @@ -494,17 +494,16 @@ func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, logger *z return secretNames, nil } -// sanitizeConfigBytes returns ipn.ConfigVAlpha in string form with redacted -// auth key. -func sanitizeConfigBytes(c ipn.ConfigVAlpha) string { +// sanitizeConfig returns an ipn.ConfigVAlpha with sensitive fields redacted. Since we pump everything +// into JSON-encoded logs it's easier to read this with a .With method than converting it to a string. +func sanitizeConfig(c ipn.ConfigVAlpha) ipn.ConfigVAlpha { + // Explicitly redact AuthKey because we never want it appearing in logs. Never populate this with the + // actual auth key. if c.AuthKey != nil { c.AuthKey = ptr.To("**redacted**") } - sanitizedBytes, err := json.Marshal(c) - if err != nil { - return "invalid config" - } - return string(sanitizedBytes) + + return c } // DeviceInfo returns the device ID, hostname, IPs and capver for the Tailscale device that acts as an operator proxy. diff --git a/tsnet/tsnet.go b/tsnet/tsnet.go index 978819519..08f08281a 100644 --- a/tsnet/tsnet.go +++ b/tsnet/tsnet.go @@ -495,14 +495,14 @@ func (s *Server) TailscaleIPs() (ip4, ip6 netip.Addr) { return ip4, ip6 } -// Logtailf returns a [logger.Logf] that outputs to Tailscale's logging service and will be only visible to Tailscale's +// LogtailWriter returns an [io.Writer] that writes to Tailscale's logging service and will be only visible to Tailscale's // support team. Logs written there cannot be retrieved by the user. This method always returns a non-nil value. -func (s *Server) Logtailf() logger.Logf { +func (s *Server) LogtailWriter() io.Writer { if s.logtail == nil { - return logger.Discard + return io.Discard } - return s.logtail.Logf + return s.logtail } func (s *Server) getAuthKey() string {