cmd/k8s-operator,k8s-operator/sessionrecording,sessionrecording,ssh/tailssh: refactor session recording functionality (#12945)

cmd/k8s-operator,k8s-operator/sessionrecording,sessionrecording,ssh/tailssh: refactor session recording functionality

Refactor SSH session recording functionality (mostly the bits related to
Kubernetes API server proxy 'kubectl exec' session recording):

- move the session recording bits used by both Tailscale SSH
and the Kubernetes API server proxy into a shared sessionrecording package,
to avoid having the operator to import ssh/tailssh

- move the Kubernetes API server proxy session recording functionality
into a k8s-operator/sessionrecording package, add some abstractions
in preparation for adding support for a second streaming protocol (WebSockets)

Updates tailscale/corp#19821

Signed-off-by: Irbe Krumina <irbe@tailscale.com>
pull/12951/head
Irbe Krumina 4 months ago committed by GitHub
parent 1bf7ed0348
commit a21bf100f3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -5,7 +5,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
W 💣 github.com/alexbrainman/sspi from github.com/alexbrainman/sspi/internal/common+
W github.com/alexbrainman/sspi/internal/common from github.com/alexbrainman/sspi/negotiate
W 💣 github.com/alexbrainman/sspi/negotiate from tailscale.com/net/tshttpproxy
LD github.com/anmitsu/go-shlex from tailscale.com/tempfork/gliderlabs/ssh
L github.com/aws/aws-sdk-go-v2/aws from github.com/aws/aws-sdk-go-v2/aws/defaults+
L github.com/aws/aws-sdk-go-v2/aws/arn from tailscale.com/ipn/store/awsstore
L github.com/aws/aws-sdk-go-v2/aws/defaults from github.com/aws/aws-sdk-go-v2/service/ssm+
@ -82,7 +81,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
github.com/bits-and-blooms/bitset from github.com/gaissmai/bart
💣 github.com/cespare/xxhash/v2 from github.com/prometheus/client_golang/prometheus
L github.com/coreos/go-iptables/iptables from tailscale.com/util/linuxfw
LD 💣 github.com/creack/pty from tailscale.com/ssh/tailssh
💣 github.com/davecgh/go-spew/spew from k8s.io/apimachinery/pkg/util/dump
W 💣 github.com/dblohm7/wingoes from github.com/dblohm7/wingoes/com+
W 💣 github.com/dblohm7/wingoes/com from tailscale.com/util/osdiag+
@ -113,7 +111,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
github.com/go-openapi/jsonreference from k8s.io/kube-openapi/pkg/internal+
github.com/go-openapi/jsonreference/internal from github.com/go-openapi/jsonreference
github.com/go-openapi/swag from github.com/go-openapi/jsonpointer+
L 💣 github.com/godbus/dbus/v5 from tailscale.com/net/dns+
L 💣 github.com/godbus/dbus/v5 from tailscale.com/net/dns
💣 github.com/gogo/protobuf/proto from k8s.io/api/admission/v1+
github.com/gogo/protobuf/sortkeys from k8s.io/api/admission/v1+
github.com/golang/groupcache/lru from k8s.io/client-go/tools/record+
@ -161,7 +159,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe
github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd
github.com/kortschak/wol from tailscale.com/ipn/ipnlocal
LD github.com/kr/fs from github.com/pkg/sftp
github.com/mailru/easyjson/buffer from github.com/mailru/easyjson/jwriter
💣 github.com/mailru/easyjson/jlexer from github.com/go-openapi/swag
github.com/mailru/easyjson/jwriter from github.com/go-openapi/swag
@ -183,8 +180,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
L github.com/pierrec/lz4/v4/internal/lz4stream from github.com/pierrec/lz4/v4
L github.com/pierrec/lz4/v4/internal/xxh32 from github.com/pierrec/lz4/v4/internal/lz4stream
github.com/pkg/errors from github.com/evanphx/json-patch/v5+
LD github.com/pkg/sftp from tailscale.com/ssh/tailssh
LD github.com/pkg/sftp/internal/encoding/ssh/filexfer from github.com/pkg/sftp
D github.com/prometheus-community/pro-bing from tailscale.com/wgengine/netstack
💣 github.com/prometheus/client_golang/prometheus from github.com/prometheus/client_golang/prometheus/collectors+
github.com/prometheus/client_golang/prometheus/collectors from sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics
@ -207,7 +202,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
W github.com/tailscale/go-winio/pkg/guid from github.com/tailscale/go-winio+
github.com/tailscale/golang-x-crypto/acme from tailscale.com/ipn/ipnlocal
LD github.com/tailscale/golang-x-crypto/internal/poly1305 from github.com/tailscale/golang-x-crypto/ssh
LD github.com/tailscale/golang-x-crypto/ssh from tailscale.com/ipn/ipnlocal+
LD github.com/tailscale/golang-x-crypto/ssh from tailscale.com/ipn/ipnlocal
LD github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf from github.com/tailscale/golang-x-crypto/ssh
github.com/tailscale/goupnp from github.com/tailscale/goupnp/dcps/internetgateway2+
github.com/tailscale/goupnp/dcps/internetgateway2 from tailscale.com/net/portmapper
@ -230,7 +225,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
github.com/tailscale/wireguard-go/tai64n from github.com/tailscale/wireguard-go/device
💣 github.com/tailscale/wireguard-go/tun from github.com/tailscale/wireguard-go/device+
github.com/tcnksm/go-httpstat from tailscale.com/net/netcheck
LD github.com/u-root/u-root/pkg/termios from tailscale.com/ssh/tailssh
L github.com/u-root/uio/rand from github.com/insomniacslk/dhcp/dhcpv4
L github.com/u-root/uio/uio from github.com/insomniacslk/dhcp/dhcpv4+
L 💣 github.com/vishvananda/netlink/nl from github.com/tailscale/netlink
@ -660,7 +654,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
tailscale.com/client/web from tailscale.com/ipn/ipnlocal
tailscale.com/clientupdate from tailscale.com/client/web+
tailscale.com/clientupdate/distsign from tailscale.com/clientupdate
LD tailscale.com/cmd/tailscaled/childproc from tailscale.com/ssh/tailssh
tailscale.com/control/controlbase from tailscale.com/control/controlhttp+
tailscale.com/control/controlclient from tailscale.com/ipn/ipnlocal+
tailscale.com/control/controlhttp from tailscale.com/control/controlclient
@ -692,6 +685,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
tailscale.com/k8s-operator from tailscale.com/cmd/k8s-operator
tailscale.com/k8s-operator/apis from tailscale.com/k8s-operator/apis/v1alpha1
tailscale.com/k8s-operator/apis/v1alpha1 from tailscale.com/cmd/k8s-operator+
tailscale.com/k8s-operator/sessionrecording from tailscale.com/cmd/k8s-operator
tailscale.com/k8s-operator/sessionrecording/conn from tailscale.com/k8s-operator/sessionrecording/spdy
tailscale.com/k8s-operator/sessionrecording/spdy from tailscale.com/k8s-operator/sessionrecording
tailscale.com/k8s-operator/sessionrecording/tsrecorder from tailscale.com/k8s-operator/sessionrecording+
tailscale.com/kube from tailscale.com/cmd/k8s-operator+
tailscale.com/licenses from tailscale.com/client/web
tailscale.com/log/filelogger from tailscale.com/logpolicy
@ -744,16 +741,15 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
tailscale.com/posture from tailscale.com/ipn/ipnlocal
tailscale.com/proxymap from tailscale.com/tsd+
💣 tailscale.com/safesocket from tailscale.com/client/tailscale+
💣 tailscale.com/ssh/tailssh from tailscale.com/cmd/k8s-operator
tailscale.com/sessionrecording from tailscale.com/cmd/k8s-operator+
tailscale.com/syncs from tailscale.com/control/controlknobs+
tailscale.com/tailcfg from tailscale.com/client/tailscale+
tailscale.com/taildrop from tailscale.com/ipn/ipnlocal+
LD tailscale.com/tempfork/gliderlabs/ssh from tailscale.com/ssh/tailssh
tailscale.com/tempfork/heap from tailscale.com/wgengine/magicsock
tailscale.com/tka from tailscale.com/client/tailscale+
W tailscale.com/tsconst from tailscale.com/net/netmon+
tailscale.com/tsd from tailscale.com/ipn/ipnlocal+
tailscale.com/tsnet from tailscale.com/cmd/k8s-operator
tailscale.com/tsnet from tailscale.com/cmd/k8s-operator+
tailscale.com/tstime from tailscale.com/cmd/k8s-operator+
tailscale.com/tstime/mono from tailscale.com/net/tstun+
tailscale.com/tstime/rate from tailscale.com/derp+
@ -838,7 +834,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
golang.org/x/crypto/argon2 from tailscale.com/tka
golang.org/x/crypto/blake2b from golang.org/x/crypto/argon2+
golang.org/x/crypto/blake2s from github.com/tailscale/wireguard-go/device+
LD golang.org/x/crypto/blowfish from github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf+
LD golang.org/x/crypto/blowfish from github.com/tailscale/golang-x-crypto/ssh/internal/bcrypt_pbkdf
golang.org/x/crypto/chacha20 from github.com/tailscale/golang-x-crypto/ssh+
golang.org/x/crypto/chacha20poly1305 from crypto/tls+
golang.org/x/crypto/cryptobyte from crypto/ecdsa+
@ -849,7 +845,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
golang.org/x/crypto/nacl/secretbox from golang.org/x/crypto/nacl/box
golang.org/x/crypto/poly1305 from github.com/tailscale/wireguard-go/device+
golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+
LD golang.org/x/crypto/ssh from github.com/pkg/sftp+
golang.org/x/exp/constraints from github.com/dblohm7/wingoes/pe+
golang.org/x/exp/maps from sigs.k8s.io/controller-runtime/pkg/cache+
golang.org/x/exp/slices from tailscale.com/cmd/k8s-operator+
@ -954,7 +949,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
log/internal from log+
log/slog from github.com/go-logr/logr+
log/slog/internal from log/slog
LD log/syslog from tailscale.com/ssh/tailssh
maps from sigs.k8s.io/controller-runtime/pkg/predicate+
math from archive/tar+
math/big from crypto/dsa+

@ -22,8 +22,9 @@ import (
"k8s.io/client-go/transport"
"tailscale.com/client/tailscale"
"tailscale.com/client/tailscale/apitype"
kubesessionrecording "tailscale.com/k8s-operator/sessionrecording"
tskube "tailscale.com/kube"
"tailscale.com/ssh/tailssh"
"tailscale.com/sessionrecording"
"tailscale.com/tailcfg"
"tailscale.com/tsnet"
"tailscale.com/util/clientmetric"
@ -36,12 +37,6 @@ var whoIsKey = ctxkey.New("", (*apitype.WhoIsResponse)(nil))
var (
// counterNumRequestsproxies counts the number of API server requests proxied via this proxy.
counterNumRequestsProxied = clientmetric.NewCounter("k8s_auth_proxy_requests_proxied")
// counterSessionRecordingsAttempted counts the number of session recording attempts.
counterSessionRecordingsAttempted = clientmetric.NewCounter("k8s_auth_proxy__session_recordings_attempted")
// counterSessionRecordingsUploaded counts the number of successfully uploaded session recordings.
counterSessionRecordingsUploaded = clientmetric.NewCounter("k8s_auth_proxy_session_recordings_uploaded")
)
type apiServerProxyMode int
@ -232,7 +227,7 @@ func (ap *apiserverProxy) serveExec(w http.ResponseWriter, r *http.Request) {
ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
return
}
counterSessionRecordingsAttempted.Add(1) // at this point we know that users intended for this session to be recorded
kubesessionrecording.CounterSessionRecordingsAttempted.Add(1) // at this point we know that users intended for this session to be recorded
if !failOpen && len(addrs) == 0 {
msg := "forbidden: 'kubectl exec' session must be recorded, but no recorders are available."
ap.log.Error(msg)
@ -252,18 +247,7 @@ func (ap *apiserverProxy) serveExec(w http.ResponseWriter, r *http.Request) {
http.Error(w, msg, http.StatusForbidden)
return
}
spdyH := &spdyHijacker{
ts: ap.ts,
req: r,
who: who,
ResponseWriter: w,
log: ap.log,
pod: r.PathValue("pod"),
ns: r.PathValue("namespace"),
addrs: addrs,
failOpen: failOpen,
connectToRecorder: tailssh.ConnectToRecorder,
}
spdyH := kubesessionrecording.New(ap.ts, r, who, w, r.PathValue("pod"), r.PathValue("namespace"), kubesessionrecording.SPDYProtocol, addrs, failOpen, sessionrecording.ConnectToRecorder, ap.log)
ap.rp.ServeHTTP(spdyH, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
}

@ -330,6 +330,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
tailscale.com/posture from tailscale.com/ipn/ipnlocal
tailscale.com/proxymap from tailscale.com/tsd+
💣 tailscale.com/safesocket from tailscale.com/client/tailscale+
LD tailscale.com/sessionrecording from tailscale.com/ssh/tailssh
LD 💣 tailscale.com/ssh/tailssh from tailscale.com/cmd/tailscaled
tailscale.com/syncs from tailscale.com/cmd/tailscaled+
tailscale.com/tailcfg from tailscale.com/client/tailscale+

@ -0,0 +1,20 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
// Package conn contains shared interface for the hijacked
// connection of a 'kubectl exec' session that is being recorded.
package conn
import "net"
type Conn interface {
net.Conn
// Fail can be called to set connection state to failed. By default any
// bytes left over in write buffer are forwarded to the intended
// destination when the connection is being closed except for when the
// connection state is failed- so set the state to failed when erroring
// out and failure policy is to fail closed.
Fail()
}

@ -0,0 +1,118 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
// Package fakes contains mocks used for testing 'kubectl exec' session
// recording functionality.
package fakes
import (
"bytes"
"encoding/json"
"net"
"sync"
"testing"
"tailscale.com/sessionrecording"
"tailscale.com/tstime"
)
func New(conn net.Conn, wb bytes.Buffer, rb bytes.Buffer, closed bool) net.Conn {
return &TestConn{
Conn: conn,
writeBuf: wb,
readBuf: rb,
closed: closed,
}
}
type TestConn struct {
net.Conn
// writeBuf contains whatever was send to the conn via Write.
writeBuf bytes.Buffer
// readBuf contains whatever was sent to the conn via Read.
readBuf bytes.Buffer
sync.RWMutex // protects the following
closed bool
}
var _ net.Conn = &TestConn{}
func (tc *TestConn) Read(b []byte) (int, error) {
return tc.readBuf.Read(b)
}
func (tc *TestConn) Write(b []byte) (int, error) {
return tc.writeBuf.Write(b)
}
func (tc *TestConn) Close() error {
tc.Lock()
defer tc.Unlock()
tc.closed = true
return nil
}
func (tc *TestConn) IsClosed() bool {
tc.Lock()
defer tc.Unlock()
return tc.closed
}
func (tc *TestConn) WriteBufBytes() []byte {
return tc.writeBuf.Bytes()
}
func (tc *TestConn) ResetReadBuf() {
tc.readBuf.Reset()
}
func (tc *TestConn) WriteReadBufBytes(b []byte) error {
_, err := tc.readBuf.Write(b)
return err
}
type TestSessionRecorder struct {
// buf holds data that was sent to the session recorder.
buf bytes.Buffer
}
func (t *TestSessionRecorder) Write(b []byte) (int, error) {
return t.buf.Write(b)
}
func (t *TestSessionRecorder) Close() error {
t.buf.Reset()
return nil
}
func (t *TestSessionRecorder) Bytes() []byte {
return t.buf.Bytes()
}
func CastLine(t *testing.T, p []byte, clock tstime.Clock) []byte {
t.Helper()
j, err := json.Marshal([]any{
clock.Now().Sub(clock.Now()).Seconds(),
"o",
string(p),
})
if err != nil {
t.Fatalf("error marshalling cast line: %v", err)
}
return append(j, '\n')
}
func AsciinemaResizeMsg(t *testing.T, width, height int) []byte {
t.Helper()
ch := sessionrecording.CastHeader{
Width: width,
Height: height,
}
bs, err := json.Marshal(ch)
if err != nil {
t.Fatalf("error marshalling CastHeader: %v", err)
}
return append(bs, '\n')
}

@ -3,7 +3,9 @@
//go:build !plan9
package main
// Package sessionrecording contains functionality for recording Kubernetes API
// server proxy 'kubectl exec' sessions.
package sessionrecording
import (
"bufio"
@ -19,17 +21,51 @@ import (
"github.com/pkg/errors"
"go.uber.org/zap"
"tailscale.com/client/tailscale/apitype"
"tailscale.com/k8s-operator/sessionrecording/spdy"
"tailscale.com/k8s-operator/sessionrecording/tsrecorder"
"tailscale.com/sessionrecording"
"tailscale.com/tailcfg"
"tailscale.com/tsnet"
"tailscale.com/tstime"
"tailscale.com/util/clientmetric"
"tailscale.com/util/multierr"
)
// spdyHijacker implements [net/http.Hijacker] interface.
const SPDYProtocol protocol = "SPDY"
// protocol is the streaming protocol of the hijacked session. Supported
// protocols are SPDY.
type protocol string
var (
// CounterSessionRecordingsAttempted counts the number of session recording attempts.
CounterSessionRecordingsAttempted = clientmetric.NewCounter("k8s_auth_proxy_session_recordings_attempted")
// counterSessionRecordingsUploaded counts the number of successfully uploaded session recordings.
counterSessionRecordingsUploaded = clientmetric.NewCounter("k8s_auth_proxy_session_recordings_uploaded")
)
func New(ts *tsnet.Server, req *http.Request, who *apitype.WhoIsResponse, w http.ResponseWriter, pod, ns string, proto protocol, addrs []netip.AddrPort, failOpen bool, connFunc RecorderDialFn, log *zap.SugaredLogger) *Hijacker {
return &Hijacker{
ts: ts,
req: req,
who: who,
ResponseWriter: w,
pod: pod,
ns: ns,
addrs: addrs,
failOpen: failOpen,
connectToRecorder: connFunc,
proto: proto,
log: log,
}
}
// Hijacker implements [net/http.Hijacker] interface.
// It must be configured with an http request for a 'kubectl exec' session that
// needs to be recorded. It knows how to hijack the connection and configure for
// the session contents to be sent to a tsrecorder instance.
type spdyHijacker struct {
type Hijacker struct {
http.ResponseWriter
ts *tsnet.Server
req *http.Request
@ -40,6 +76,7 @@ type spdyHijacker struct {
addrs []netip.AddrPort // tsrecorder addresses
failOpen bool // whether to fail open if recording fails
connectToRecorder RecorderDialFn
proto protocol // streaming protocol
}
// RecorderDialFn dials the specified netip.AddrPorts that should be tsrecorder
@ -51,7 +88,7 @@ type RecorderDialFn func(context.Context, []netip.AddrPort, func(context.Context
// Hijack hijacks a 'kubectl exec' session and configures for the session
// contents to be sent to a recorder.
func (h *spdyHijacker) Hijack() (net.Conn, *bufio.ReadWriter, error) {
func (h *Hijacker) Hijack() (net.Conn, *bufio.ReadWriter, error) {
h.log.Infof("recorder addrs: %v, failOpen: %v", h.addrs, h.failOpen)
reqConn, brw, err := h.ResponseWriter.(http.Hijacker).Hijack()
if err != nil {
@ -69,7 +106,7 @@ func (h *spdyHijacker) Hijack() (net.Conn, *bufio.ReadWriter, error) {
// spdyHijacker.addrs. Returns conn from provided opts, wrapped in recording
// logic. If connecting to the recorder fails or an error is received during the
// session and spdyHijacker.failOpen is false, connection will be closed.
func (h *spdyHijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn, error) {
func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn, error) {
const (
// https://docs.asciinema.org/manual/asciicast/v2/
asciicastv2 = 2
@ -96,25 +133,15 @@ func (h *spdyHijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.C
h.log.Info("successfully connected to a session recorder")
wc = rw
cl := tstime.DefaultClock{}
lc := &spdyRemoteConnRecorder{
log: h.log,
Conn: conn,
rec: &recorder{
start: cl.Now(),
clock: cl,
failOpen: h.failOpen,
conn: wc,
},
}
rec := tsrecorder.New(wc, cl, cl.Now(), h.failOpen)
qp := h.req.URL.Query()
ch := CastHeader{
ch := sessionrecording.CastHeader{
Version: asciicastv2,
Timestamp: lc.rec.start.Unix(),
Timestamp: cl.Now().Unix(),
Command: strings.Join(qp["command"], " "),
SrcNode: strings.TrimSuffix(h.who.Node.Name, "."),
SrcNodeID: h.who.Node.StableID,
Kubernetes: &Kubernetes{
Kubernetes: &sessionrecording.Kubernetes{
PodName: h.pod,
Namespace: h.ns,
Container: strings.Join(qp["container"], " "),
@ -126,7 +153,7 @@ func (h *spdyHijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.C
} else {
ch.SrcNodeTags = h.who.Node.Tags
}
lc.ch = ch
lc := spdy.New(conn, rec, ch, h.log)
go func() {
var err error
select {
@ -147,7 +174,7 @@ func (h *spdyHijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.C
}
msg += "; failure mode set to 'fail closed'; closing connection"
h.log.Error(msg)
lc.failed = true
lc.Fail()
// TODO (irbekrm): write a message to the client
if err := lc.Close(); err != nil {
h.log.Infof("error closing recorder connections: %v", err)
@ -157,52 +184,6 @@ func (h *spdyHijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.C
return lc, nil
}
// CastHeader is the asciicast header to be sent to the recorder at the start of
// the recording of a session.
// https://docs.asciinema.org/manual/asciicast/v2/#header
type CastHeader struct {
// Version is the asciinema file format version.
Version int `json:"version"`
// Width is the terminal width in characters.
Width int `json:"width"`
// Height is the terminal height in characters.
Height int `json:"height"`
// Timestamp is the unix timestamp of when the recording started.
Timestamp int64 `json:"timestamp"`
// Tailscale-specific fields: SrcNode is the full MagicDNS name of the
// tailnet node originating the connection, without the trailing dot.
SrcNode string `json:"srcNode"`
// SrcNodeID is the node ID of the tailnet node originating the connection.
SrcNodeID tailcfg.StableNodeID `json:"srcNodeID"`
// SrcNodeTags is the list of tags on the node originating the connection (if any).
SrcNodeTags []string `json:"srcNodeTags,omitempty"`
// SrcNodeUserID is the user ID of the node originating the connection (if not tagged).
SrcNodeUserID tailcfg.UserID `json:"srcNodeUserID,omitempty"` // if not tagged
// SrcNodeUser is the LoginName of the node originating the connection (if not tagged).
SrcNodeUser string `json:"srcNodeUser,omitempty"`
Command string
// Kubernetes-specific fields:
Kubernetes *Kubernetes `json:"kubernetes,omitempty"`
}
// Kubernetes contains 'kubectl exec' session specific information for
// tsrecorder.
type Kubernetes struct {
PodName string
Namespace string
Container string
}
func closeConnWithWarning(conn net.Conn, msg string) error {
b := io.NopCloser(bytes.NewBuffer([]byte(msg)))
resp := http.Response{Status: http.StatusText(http.StatusForbidden), StatusCode: http.StatusForbidden, Body: b}

@ -3,7 +3,7 @@
//go:build !plan9
package main
package sessionrecording
import (
"context"
@ -19,12 +19,13 @@ import (
"go.uber.org/zap"
"tailscale.com/client/tailscale/apitype"
"tailscale.com/k8s-operator/sessionrecording/fakes"
"tailscale.com/tailcfg"
"tailscale.com/tsnet"
"tailscale.com/tstest"
)
func Test_SPDYHijacker(t *testing.T) {
func Test_Hijacker(t *testing.T) {
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
@ -64,9 +65,9 @@ func Test_SPDYHijacker(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tc := &testConn{}
tc := &fakes.TestConn{}
ch := make(chan error)
h := &spdyHijacker{
h := &Hijacker{
connectToRecorder: func(context.Context, []netip.AddrPort, func(context.Context, string, string) (net.Conn, error)) (wc io.WriteCloser, rec []*tailcfg.SSHRecordingAttempt, _ <-chan error, err error) {
if tt.failRecorderConnect {
err = errors.New("test")
@ -98,8 +99,8 @@ func Test_SPDYHijacker(t *testing.T) {
// (test that connection remains open over some period
// of time).
if err := tstest.WaitFor(timeout, func() (err error) {
if tt.wantsConnClosed != tc.isClosed() {
return fmt.Errorf("got connection state: %t, wants connection state: %t", tc.isClosed(), tt.wantsConnClosed)
if tt.wantsConnClosed != tc.IsClosed() {
return fmt.Errorf("got connection state: %t, wants connection state: %t", tc.IsClosed(), tt.wantsConnClosed)
}
return nil
}); err != nil {

@ -3,7 +3,9 @@
//go:build !plan9
package main
// Package spdy contains functionality for parsing SPDY streaming sessions. This
// is used for 'kubectl exec' session recording.
package spdy
import (
"bytes"
@ -17,16 +19,29 @@ import (
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
srconn "tailscale.com/k8s-operator/sessionrecording/conn"
"tailscale.com/k8s-operator/sessionrecording/tsrecorder"
"tailscale.com/sessionrecording"
)
// spdyRemoteConnRecorder is a wrapper around net.Conn. It reads the bytestream
// for a 'kubectl exec' session, sends session recording data to the configured
// recorder and forwards the raw bytes to the original destination.
type spdyRemoteConnRecorder struct {
func New(nc net.Conn, rec *tsrecorder.Client, ch sessionrecording.CastHeader, log *zap.SugaredLogger) srconn.Conn {
return &conn{
Conn: nc,
rec: rec,
ch: ch,
log: log,
}
}
// conn is a wrapper around net.Conn. It reads the bytestream for a 'kubectl
// exec' session streamed using SPDY protocol, sends session recording data to
// the configured recorder and forwards the raw bytes to the original
// destination.
type conn struct {
net.Conn
// rec knows how to send data written to it to a tsrecorder instance.
rec *recorder
ch CastHeader
rec *tsrecorder.Client
ch sessionrecording.CastHeader
stdoutStreamID atomic.Uint32
stderrStreamID atomic.Uint32
@ -53,7 +68,7 @@ type spdyRemoteConnRecorder struct {
// If the frame is a data frame for resize stream, sends resize message to the
// recorder. If the frame is a SYN_STREAM control frame that starts stdout,
// stderr or resize stream, store the stream ID.
func (c *spdyRemoteConnRecorder) Read(b []byte) (int, error) {
func (c *conn) Read(b []byte) (int, error) {
c.rmu.Lock()
defer c.rmu.Unlock()
n, err := c.Conn.Read(b)
@ -103,7 +118,7 @@ func (c *spdyRemoteConnRecorder) Read(b []byte) (int, error) {
// Write forwards the raw data of the latest parsed SPDY frame to the original
// destination. If the frame is an SPDY data frame, it also sends the payload to
// the connected session recorder.
func (c *spdyRemoteConnRecorder) Write(b []byte) (int, error) {
func (c *conn) Write(b []byte) (int, error) {
c.wmu.Lock()
defer c.wmu.Unlock()
c.writeBuf.Write(b)
@ -133,7 +148,7 @@ func (c *spdyRemoteConnRecorder) Write(b []byte) (int, error) {
return
}
j = append(j, '\n')
err = c.rec.writeCastLine(j)
err = c.rec.WriteCastLine(j)
if err != nil {
c.log.Errorf("received error from recorder: %v", err)
}
@ -151,7 +166,7 @@ func (c *spdyRemoteConnRecorder) Write(b []byte) (int, error) {
return len(b), err
}
func (c *spdyRemoteConnRecorder) Close() error {
func (c *conn) Close() error {
c.wmu.Lock()
defer c.wmu.Unlock()
if c.closed {
@ -167,13 +182,19 @@ func (c *spdyRemoteConnRecorder) Close() error {
return err
}
// parseSynStream parses SYN_STREAM SPDY control frame and updates
func (s *conn) Fail() {
s.wmu.Lock()
s.failed = true
s.wmu.Unlock()
}
// storeStreamID parses SYN_STREAM SPDY control frame and updates
// spdyRemoteConnRecorder to store the newly created stream's ID if it is one of
// the stream types we care about. Storing stream_id:stream_type mapping allows
// us to parse received data frames (that have stream IDs) differently depening
// on which stream they belong to (i.e send data frame payload for stdout stream
// to session recorder).
func (c *spdyRemoteConnRecorder) storeStreamID(sf spdyFrame, header http.Header) {
func (c *conn) storeStreamID(sf spdyFrame, header http.Header) {
const (
streamTypeHeaderKey = "Streamtype"
)

@ -3,19 +3,18 @@
//go:build !plan9
package main
package spdy
import (
"bytes"
"encoding/json"
"net"
"reflect"
"sync"
"testing"
"go.uber.org/zap"
"tailscale.com/k8s-operator/sessionrecording/fakes"
"tailscale.com/k8s-operator/sessionrecording/tsrecorder"
"tailscale.com/sessionrecording"
"tailscale.com/tstest"
"tailscale.com/tstime"
)
// Test_Writes tests that 1 or more Write calls to spdyRemoteConnRecorder
@ -56,13 +55,13 @@ func Test_Writes(t *testing.T) {
name: "single_write_stdout_data_frame_with_payload",
inputs: [][]byte{{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}},
wantForwarded: []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5},
wantRecorded: castLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl),
wantRecorded: fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl),
},
{
name: "single_write_stderr_data_frame_with_payload",
inputs: [][]byte{{0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}},
wantForwarded: []byte{0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5},
wantRecorded: castLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl),
wantRecorded: fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl),
},
{
name: "single_data_frame_unknow_stream_with_payload",
@ -73,13 +72,13 @@ func Test_Writes(t *testing.T) {
name: "control_frame_and_data_frame_split_across_two_writes",
inputs: [][]byte{{0x80, 0x3, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, {0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}},
wantForwarded: []byte{0x80, 0x3, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5},
wantRecorded: castLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl),
wantRecorded: fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl),
},
{
name: "single_first_write_stdout_data_frame_with_payload",
inputs: [][]byte{{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}},
wantForwarded: []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5},
wantRecorded: append(asciinemaResizeMsg(t, 10, 20), castLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl)...),
wantRecorded: append(fakes.AsciinemaResizeMsg(t, 10, 20), fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl)...),
width: 10,
height: 20,
firstWrite: true,
@ -87,19 +86,15 @@ func Test_Writes(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tc := &testConn{}
sr := &testSessionRecorder{}
rec := &recorder{
conn: sr,
clock: cl,
start: cl.Now(),
}
tc := &fakes.TestConn{}
sr := &fakes.TestSessionRecorder{}
rec := tsrecorder.New(sr, cl, cl.Now(), true)
c := &spdyRemoteConnRecorder{
c := &conn{
Conn: tc,
log: zl.Sugar(),
rec: rec,
ch: CastHeader{
ch: sessionrecording.CastHeader{
Width: tt.width,
Height: tt.height,
},
@ -118,13 +113,13 @@ func Test_Writes(t *testing.T) {
}
// Assert that the expected bytes have been forwarded to the original destination.
gotForwarded := tc.writeBuf.Bytes()
gotForwarded := tc.WriteBufBytes()
if !reflect.DeepEqual(gotForwarded, tt.wantForwarded) {
t.Errorf("expected bytes not forwarded, wants\n%v\ngot\n%v", tt.wantForwarded, gotForwarded)
}
// Assert that the expected bytes have been forwarded to the session recorder.
gotRecorded := sr.buf.Bytes()
gotRecorded := sr.Bytes()
if !reflect.DeepEqual(gotRecorded, tt.wantRecorded) {
t.Errorf("expected bytes not recorded, wants\n%v\ngot\n%v", tt.wantRecorded, gotRecorded)
}
@ -197,14 +192,10 @@ func Test_Reads(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tc := &testConn{}
sr := &testSessionRecorder{}
rec := &recorder{
conn: sr,
clock: cl,
start: cl.Now(),
}
c := &spdyRemoteConnRecorder{
tc := &fakes.TestConn{}
sr := &fakes.TestSessionRecorder{}
rec := tsrecorder.New(sr, cl, cl.Now(), true)
c := &conn{
Conn: tc,
log: zl.Sugar(),
rec: rec,
@ -213,9 +204,8 @@ func Test_Reads(t *testing.T) {
for i, input := range tt.inputs {
c.zlibReqReader = reader
tc.readBuf.Reset()
_, err := tc.readBuf.Write(input)
if err != nil {
tc.ResetReadBuf()
if err := tc.WriteReadBufBytes(input); err != nil {
t.Fatalf("writing bytes to test conn: %v", err)
}
_, err = c.Read(make([]byte, len(input)))
@ -244,19 +234,6 @@ func Test_Reads(t *testing.T) {
}
}
func castLine(t *testing.T, p []byte, clock tstime.Clock) []byte {
t.Helper()
j, err := json.Marshal([]any{
clock.Now().Sub(clock.Now()).Seconds(),
"o",
string(p),
})
if err != nil {
t.Fatalf("error marshalling cast line: %v", err)
}
return append(j, '\n')
}
func resizeMsgBytes(t *testing.T, width, height int) []byte {
t.Helper()
bs, err := json.Marshal(spdyResizeMsg{Width: width, Height: height})
@ -265,62 +242,3 @@ func resizeMsgBytes(t *testing.T, width, height int) []byte {
}
return bs
}
func asciinemaResizeMsg(t *testing.T, width, height int) []byte {
t.Helper()
ch := CastHeader{
Width: width,
Height: height,
}
bs, err := json.Marshal(ch)
if err != nil {
t.Fatalf("error marshalling CastHeader: %v", err)
}
return append(bs, '\n')
}
type testConn struct {
net.Conn
// writeBuf contains whatever was send to the conn via Write.
writeBuf bytes.Buffer
// readBuf contains whatever was sent to the conn via Read.
readBuf bytes.Buffer
sync.RWMutex // protects the following
closed bool
}
var _ net.Conn = &testConn{}
func (tc *testConn) Read(b []byte) (int, error) {
return tc.readBuf.Read(b)
}
func (tc *testConn) Write(b []byte) (int, error) {
return tc.writeBuf.Write(b)
}
func (tc *testConn) Close() error {
tc.Lock()
defer tc.Unlock()
tc.closed = true
return nil
}
func (tc *testConn) isClosed() bool {
tc.Lock()
defer tc.Unlock()
return tc.closed
}
type testSessionRecorder struct {
// buf holds data that was sent to the session recorder.
buf bytes.Buffer
}
func (t *testSessionRecorder) Write(b []byte) (int, error) {
return t.buf.Write(b)
}
func (t *testSessionRecorder) Close() error {
t.buf.Reset()
return nil
}

@ -3,7 +3,7 @@
//go:build !plan9
package main
package spdy
import (
"bytes"

@ -3,7 +3,7 @@
//go:build !plan9
package main
package spdy
import (
"bytes"

@ -3,7 +3,7 @@
//go:build !plan9
package main
package spdy
import (
"bytes"

@ -3,7 +3,8 @@
//go:build !plan9
package main
// Package tsrecorder contains functionality for connecting to a tsrecorder instance.
package tsrecorder
import (
"encoding/json"
@ -16,9 +17,18 @@ import (
"tailscale.com/tstime"
)
func New(conn io.WriteCloser, clock tstime.Clock, start time.Time, failOpen bool) *Client {
return &Client{
start: start,
clock: clock,
conn: conn,
failOpen: failOpen,
}
}
// recorder knows how to send the provided bytes to the configured tsrecorder
// instance in asciinema format.
type recorder struct {
type Client struct {
start time.Time
clock tstime.Clock
@ -36,7 +46,7 @@ type recorder struct {
// Write appends timestamp to the provided bytes and sends them to the
// configured tsrecorder.
func (rec *recorder) Write(p []byte) (err error) {
func (rec *Client) Write(p []byte) (err error) {
if len(p) == 0 {
return nil
}
@ -52,7 +62,7 @@ func (rec *recorder) Write(p []byte) (err error) {
return fmt.Errorf("error marhalling payload: %w", err)
}
j = append(j, '\n')
if err := rec.writeCastLine(j); err != nil {
if err := rec.WriteCastLine(j); err != nil {
if !rec.failOpen {
return fmt.Errorf("error writing payload to recorder: %w", err)
}
@ -61,7 +71,7 @@ func (rec *recorder) Write(p []byte) (err error) {
return nil
}
func (rec *recorder) Close() error {
func (rec *Client) Close() error {
rec.mu.Lock()
defer rec.mu.Unlock()
if rec.conn == nil {
@ -74,15 +84,20 @@ func (rec *recorder) Close() error {
// writeCastLine sends bytes to the tsrecorder. The bytes should be in
// asciinema format.
func (rec *recorder) writeCastLine(j []byte) error {
rec.mu.Lock()
defer rec.mu.Unlock()
if rec.conn == nil {
func (c *Client) WriteCastLine(j []byte) error {
c.mu.Lock()
defer c.mu.Unlock()
if c.conn == nil {
return errors.New("recorder closed")
}
_, err := rec.conn.Write(j)
_, err := c.conn.Write(j)
if err != nil {
return fmt.Errorf("recorder write error: %w", err)
}
return nil
}
type ResizeMsg struct {
Width int `json:"width"`
Height int `json:"height"`
}

@ -1,7 +1,9 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package tailssh
// Package sessionrecording contains session recording utils shared amongst
// Tailscale SSH and Kubernetes API server proxy session recording.
package sessionrecording
import (
"context"

@ -0,0 +1,78 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package sessionrecording
import "tailscale.com/tailcfg"
// CastHeader is the header of an asciinema file.
type CastHeader struct {
// Version is the asciinema file format version.
Version int `json:"version"`
// Width is the terminal width in characters.
// It is non-zero for Pty sessions.
Width int `json:"width"`
// Height is the terminal height in characters.
// It is non-zero for Pty sessions.
Height int `json:"height"`
// Timestamp is the unix timestamp of when the recording started.
Timestamp int64 `json:"timestamp"`
// Command is the command that was executed.
// Typically empty for shell sessions.
Command string `json:"command,omitempty"`
// SrcNode is the FQDN of the node originating the connection.
// It is also the MagicDNS name for the node.
// It does not have a trailing dot.
// e.g. "host.tail-scale.ts.net"
SrcNode string `json:"srcNode"`
// SrcNodeID is the node ID of the node originating the connection.
SrcNodeID tailcfg.StableNodeID `json:"srcNodeID"`
// Tailscale-specific fields:
// SrcNodeTags is the list of tags on the node originating the connection (if any).
SrcNodeTags []string `json:"srcNodeTags,omitempty"`
// SrcNodeUserID is the user ID of the node originating the connection (if not tagged).
SrcNodeUserID tailcfg.UserID `json:"srcNodeUserID,omitempty"` // if not tagged
// SrcNodeUser is the LoginName of the node originating the connection (if not tagged).
SrcNodeUser string `json:"srcNodeUser,omitempty"`
// Fields that are only set for Tailscale SSH session recordings:
// Env is the environment variables of the session.
// Only "TERM" is set (2023-03-22).
Env map[string]string `json:"env"`
// SSHUser is the username as presented by the client.
SSHUser string `json:"sshUser"` // as presented by the client
// LocalUser is the effective username on the server.
LocalUser string `json:"localUser"`
// ConnectionID uniquely identifies a connection made to the SSH server.
// It may be shared across multiple sessions over the same connection in
// case of SSH multiplexing.
ConnectionID string `json:"connectionID"`
// Fields that are only set for Kubernetes API server proxy session recordings:
Kubernetes *Kubernetes `json:"kubernetes,omitempty"`
}
// Kubernetes contains 'kubectl exec' session specific information for
// tsrecorder.
type Kubernetes struct {
// PodName is the name of the Pod being exec-ed.
PodName string
// Namespace is the namespace in which is the Pod that is being exec-ed.
Namespace string
// Container is the container being exec-ed.
Container string
}

@ -36,6 +36,7 @@ import (
"tailscale.com/logtail/backoff"
"tailscale.com/net/tsaddr"
"tailscale.com/net/tsdial"
"tailscale.com/sessionrecording"
"tailscale.com/tailcfg"
"tailscale.com/tempfork/gliderlabs/ssh"
"tailscale.com/types/key"
@ -1428,61 +1429,6 @@ func randBytes(n int) []byte {
return b
}
// CastHeader is the header of an asciinema file.
type CastHeader struct {
// Version is the asciinema file format version.
Version int `json:"version"`
// Width is the terminal width in characters.
// It is non-zero for Pty sessions.
Width int `json:"width"`
// Height is the terminal height in characters.
// It is non-zero for Pty sessions.
Height int `json:"height"`
// Timestamp is the unix timestamp of when the recording started.
Timestamp int64 `json:"timestamp"`
// Env is the environment variables of the session.
// Only "TERM" is set (2023-03-22).
Env map[string]string `json:"env"`
// Command is the command that was executed.
// Typically empty for shell sessions.
Command string `json:"command,omitempty"`
// Tailscale-specific fields:
// SrcNode is the FQDN of the node originating the connection.
// It is also the MagicDNS name for the node.
// It does not have a trailing dot.
// e.g. "host.tail-scale.ts.net"
SrcNode string `json:"srcNode"`
// SrcNodeID is the node ID of the node originating the connection.
SrcNodeID tailcfg.StableNodeID `json:"srcNodeID"`
// SrcNodeTags is the list of tags on the node originating the connection (if any).
SrcNodeTags []string `json:"srcNodeTags,omitempty"`
// SrcNodeUserID is the user ID of the node originating the connection (if not tagged).
SrcNodeUserID tailcfg.UserID `json:"srcNodeUserID,omitempty"` // if not tagged
// SrcNodeUser is the LoginName of the node originating the connection (if not tagged).
SrcNodeUser string `json:"srcNodeUser,omitempty"`
// SSHUser is the username as presented by the client.
SSHUser string `json:"sshUser"` // as presented by the client
// LocalUser is the effective username on the server.
LocalUser string `json:"localUser"`
// ConnectionID uniquely identifies a connection made to the SSH server.
// It may be shared across multiple sessions over the same connection in
// case of SSH multiplexing.
ConnectionID string `json:"connectionID"`
}
func (ss *sshSession) openFileForRecording(now time.Time) (_ io.WriteCloser, err error) {
varRoot := ss.conn.srv.lb.TailscaleVarRoot()
if varRoot == "" {
@ -1548,7 +1494,7 @@ func (ss *sshSession) startNewRecording() (_ *recording, err error) {
} else {
var errChan <-chan error
var attempts []*tailcfg.SSHRecordingAttempt
rec.out, attempts, errChan, err = ConnectToRecorder(ctx, recorders, ss.conn.srv.lb.Dialer().UserDial)
rec.out, attempts, errChan, err = sessionrecording.ConnectToRecorder(ctx, recorders, ss.conn.srv.lb.Dialer().UserDial)
if err != nil {
if onFailure != nil && onFailure.NotifyURL != "" && len(attempts) > 0 {
eventType := tailcfg.SSHSessionRecordingFailed
@ -1598,7 +1544,7 @@ func (ss *sshSession) startNewRecording() (_ *recording, err error) {
}()
}
ch := CastHeader{
ch := sessionrecording.CastHeader{
Version: 2,
Width: w.Width,
Height: w.Height,

@ -36,6 +36,7 @@ import (
"tailscale.com/ipn/store/mem"
"tailscale.com/net/memnet"
"tailscale.com/net/tsdial"
"tailscale.com/sessionrecording"
"tailscale.com/tailcfg"
"tailscale.com/tempfork/gliderlabs/ssh"
"tailscale.com/tsd"
@ -630,7 +631,7 @@ func TestSSHRecordingNonInteractive(t *testing.T) {
wg.Wait()
<-ctx.Done() // wait for recording to finish
var ch CastHeader
var ch sessionrecording.CastHeader
if err := json.NewDecoder(bytes.NewReader(recording)).Decode(&ch); err != nil {
t.Fatal(err)
}

Loading…
Cancel
Save