From 2ca6dd1f1d339e3c28f1113d7b50e188b4e5f5a6 Mon Sep 17 00:00:00 2001 From: Tom DNetto Date: Wed, 8 Feb 2023 15:48:27 -0800 Subject: [PATCH] wgengine: start logging DISCO frames to pcap stream Signed-off-by: Tom DNetto --- net/tstun/wrap.go | 10 +-- wgengine/capture/capture.go | 10 +++ wgengine/capture/ts-dissector.lua | 130 +++++++++++++++++++++++++++++- wgengine/magicsock/magicsock.go | 47 +++++++++++ wgengine/userspace.go | 6 +- wgengine/watchdog.go | 3 +- wgengine/wgengine.go | 8 +- 7 files changed, 193 insertions(+), 21 deletions(-) diff --git a/net/tstun/wrap.go b/net/tstun/wrap.go index 4848730fd..6725c7c81 100644 --- a/net/tstun/wrap.go +++ b/net/tstun/wrap.go @@ -69,12 +69,6 @@ var parsedPacketPool = sync.Pool{New: func() any { return new(packet.Parsed) }} // It must not hold onto the packet struct, as its backing storage will be reused. type FilterFunc func(*packet.Parsed, *Wrapper) filter.Response -// CaptureFunc describes a callback to record packets when -// debugging packet-capture. Such callbacks must not take -// ownership of the provided data slice: it may only copy -// out of it within the lifetime of the function. -type CaptureFunc func(capture.Path, time.Time, []byte) - // Wrapper augments a tun.Device with packet filtering and injection. type Wrapper struct { logf logger.Logf @@ -181,7 +175,7 @@ type Wrapper struct { // stats maintains per-connection counters. stats atomic.Pointer[connstats.Statistics] - captureHook syncs.AtomicValue[CaptureFunc] + captureHook syncs.AtomicValue[capture.Callback] } // tunInjectedRead is an injected packet pretending to be a tun.Read(). @@ -942,6 +936,6 @@ var ( metricPacketOutDropSelfDisco = clientmetric.NewCounter("tstun_out_to_wg_drop_self_disco") ) -func (t *Wrapper) InstallCaptureHook(cb CaptureFunc) { +func (t *Wrapper) InstallCaptureHook(cb capture.Callback) { t.captureHook.Store(cb) } diff --git a/wgengine/capture/capture.go b/wgengine/capture/capture.go index 1d3f50fcf..ef1bbde3f 100644 --- a/wgengine/capture/capture.go +++ b/wgengine/capture/capture.go @@ -21,6 +21,13 @@ import ( //go:embed ts-dissector.lua var DissectorLua string +// Callback describes a function which is called to +// record packets when debugging packet-capture. +// Such callbacks must not take ownership of the +// provided data slice: it may only copy out of it +// within the lifetime of the function. +type Callback func(Path, time.Time, []byte) + var bufferPool = sync.Pool{ New: func() any { return new(bytes.Buffer) @@ -65,6 +72,9 @@ const ( // SynthesizedToPeer indicates the packet was generated from within tailscaled, // and is being routed to a remote Wireguard peer. SynthesizedToPeer Path = 3 + + // PathDisco indicates the packet is information about a disco frame. + PathDisco Path = 254 ) // New creates a new capture sink. diff --git a/wgengine/capture/ts-dissector.lua b/wgengine/capture/ts-dissector.lua index 7779f84cc..0ff20bab5 100644 --- a/wgengine/capture/ts-dissector.lua +++ b/wgengine/capture/ts-dissector.lua @@ -1,3 +1,7 @@ +function hasbit(x, p) + return x % (p + p) >= p +end + tsdebug_ll = Proto("tsdebug", "Tailscale debug") PATH = ProtoField.string("tsdebug.PATH","PATH", base.ASCII) tsdebug_ll.fields = {PATH} @@ -14,14 +18,134 @@ function tsdebug_ll.dissector(buffer, pinfo, tree) elseif path_id == 1 then subtree:add(PATH, "FromPeer") elseif path_id == 2 then subtree:add(PATH, "Synthesized (Inbound / ToLocal)") elseif path_id == 3 then subtree:add(PATH, "Synthesized (Outbound / ToPeer)") + elseif path_id == 254 then subtree:add(PATH, "Disco frame") end offset = offset + 2 - -- -- Handover rest of data to ip dissector + -- -- Handover rest of data to lower-level dissector local data_buffer = buffer:range(offset, packet_length-offset):tvb() - Dissector.get("ip"):call(data_buffer, pinfo, tree) + if path_id == 254 then + Dissector.get("tsdisco"):call(data_buffer, pinfo, tree) + else + Dissector.get("ip"):call(data_buffer, pinfo, tree) + end end -- Install the dissector on link-layer ID 147 (User-defined protocol 0) local eth_table = DissectorTable.get("wtap_encap") -eth_table:add(wtap.USER0, tsdebug_ll) \ No newline at end of file +eth_table:add(wtap.USER0, tsdebug_ll) + + +local ts_dissectors = DissectorTable.new("ts.proto", "Tailscale-specific dissectors", ftypes.STRING, base.NONE) + + +-- +-- DISCO metadata dissector +-- +tsdisco_meta = Proto("tsdisco", "Tailscale DISCO metadata") +DISCO_IS_DERP = ProtoField.bool("tsdisco.IS_DERP","From DERP") +DISCO_SRC_IP_4 = ProtoField.ipv4("tsdisco.SRC_IP_4", "Source IPv4 address") +DISCO_SRC_IP_6 = ProtoField.ipv4("tsdisco.SRC_IP_6", "Source IPv6 address") +DISCO_SRC_PORT = ProtoField.uint16("tsdisco.SRC_PORT","Source port", base.DEC) +DISCO_DERP_PUB = ProtoField.bytes("tsdisco.DERP_PUB", "DERP public key", base.SPACE) +tsdisco_meta.fields = {DISCO_IS_DERP, DISCO_SRC_PORT, DISCO_DERP_PUB, DISCO_SRC_IP_4, DISCO_SRC_IP_6} + +function tsdisco_meta.dissector(buffer, pinfo, tree) + pinfo.cols.protocol = tsdisco_meta.name + packet_length = buffer:len() + local offset = 0 + local subtree = tree:add(tsdisco_meta, buffer(), "DISCO metadata") + + -- Parse flags + local from_derp = hasbit(buffer(offset, 1):le_uint(), 0) + subtree:add(DISCO_IS_DERP, from_derp) -- Flag bit 0 + offset = offset + 1 + -- Parse DERP public key + if from_derp then + subtree:add(DISCO_DERP_PUB, buffer(offset, 32)) + end + offset = offset + 32 + + -- Parse source port + subtree:add(DISCO_SRC_PORT, buffer:range(offset, 2):le_uint()) + offset = offset + 2 + + -- Parse source address + local addr_len = buffer:range(offset, 2):le_uint() + offset = offset + 2 + if addr_len == 4 then subtree:add(DISCO_SRC_IP_4, buffer:range(offset, addr_len)) + else subtree:add(DISCO_SRC_IP_6, buffer:range(offset, addr_len)) + end + offset = offset + addr_len + + -- Handover to the actual disco frame dissector + offset = offset + 2 -- skip over payload len + local data_buffer = buffer:range(offset, packet_length-offset):tvb() + Dissector.get("disco"):call(data_buffer, pinfo, tree) +end + +ts_dissectors:add(1, tsdisco_meta) + +-- +-- DISCO frame dissector +-- +tsdisco_frame = Proto("disco", "Tailscale DISCO frame") +DISCO_TYPE = ProtoField.string("disco.TYPE", "Message type", base.ASCII) +DISCO_VERSION = ProtoField.uint8("disco.VERSION","Protocol version", base.DEC) +DISCO_TXID = ProtoField.bytes("disco.TXID", "Transaction ID", base.SPACE) +DISCO_NODEKEY = ProtoField.bytes("disco.NODE_KEY", "Node key", base.SPACE) +DISCO_PONG_SRC = ProtoField.ipv6("disco.PONG_SRC", "Pong source") +DISCO_PONG_SRC_PORT = ProtoField.uint16("disco.PONG_SRC_PORT","Source port", base.DEC) +DISCO_UNKNOWN = ProtoField.bytes("disco.UNKNOWN_DATA", "Trailing data", base.SPACE) +tsdisco_frame.fields = {DISCO_TYPE, DISCO_VERSION, DISCO_TXID, DISCO_NODEKEY, DISCO_PONG_SRC, DISCO_PONG_SRC_PORT, DISCO_UNKNOWN} + +function tsdisco_frame.dissector(buffer, pinfo, tree) + packet_length = buffer:len() + local offset = 0 + local subtree = tree:add(tsdisco_frame, buffer(), "DISCO frame") + + -- Message type + local message_type = buffer(offset, 1):le_uint() + offset = offset + 1 + if message_type == 1 then subtree:add(DISCO_TYPE, "Ping") + elseif message_type == 2 then subtree:add(DISCO_TYPE, "Pong") + elseif message_type == 3 then subtree:add(DISCO_TYPE, "Call me maybe") + end + + -- Message version + local message_version = buffer(offset, 1):le_uint() + offset = offset + 1 + subtree:add(DISCO_VERSION, message_version) + + -- TXID (Ping / Pong) + if message_type == 1 or message_type == 2 then + subtree:add(DISCO_TXID, buffer(offset, 12)) + offset = offset + 12 + end + + -- NodeKey (Ping) + if message_type == 1 then + subtree:add(DISCO_NODEKEY, buffer(offset, 32)) + offset = offset + 32 + end + + -- Src (Pong) + if message_type == 2 then + subtree:add(DISCO_PONG_SRC, buffer:range(offset, 16)) + offset = offset + 16 + end + -- Src port (Pong) + if message_type == 2 then + subtree:add(DISCO_PONG_SRC_PORT, buffer(offset, 2):le_uint()) + offset = offset + 2 + end + + -- TODO(tom): Parse CallMeMaybe.MyNumber + + local trailing = buffer:range(offset, packet_length-offset) + if trailing:len() > 0 then + subtree:add(DISCO_UNKNOWN, trailing) + end +end + +ts_dissectors:add(2, tsdisco_frame) \ No newline at end of file diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 4f2ec7825..881be0e96 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -7,6 +7,7 @@ package magicsock import ( "bufio" + "bytes" "context" crand "crypto/rand" "encoding/binary" @@ -62,6 +63,7 @@ import ( "tailscale.com/util/mak" "tailscale.com/util/uniq" "tailscale.com/version" + "tailscale.com/wgengine/capture" "tailscale.com/wgengine/monitor" ) @@ -348,6 +350,9 @@ type Conn struct { // stats maintains per-connection counters. stats atomic.Pointer[connstats.Statistics] + // captureHook, if non-nil, is the pcap logging callback when capturing. + captureHook syncs.AtomicValue[capture.Callback] + // ============================================================ // mu guards all following fields; see userspaceEngine lock // ordering rules against the engine. For derphttp, mu must @@ -664,6 +669,14 @@ func NewConn(opts Options) (*Conn, error) { return c, nil } +// InstallCaptureHook installs a callback which is called to +// log debug information into the pcap stream. This function +// can be called with a nil argument to uninstall the capture +// hook. +func (c *Conn) InstallCaptureHook(cb capture.Callback) { + c.captureHook.Store(cb) +} + // ignoreSTUNPackets sets a STUN packet processing func that does nothing. func (c *Conn) ignoreSTUNPackets() { c.stunReceiveFunc.Store(func([]byte, netip.AddrPort) {}) @@ -2017,6 +2030,34 @@ func (c *Conn) sendDiscoMessage(dst netip.AddrPort, dstKey key.NodePublic, dstDi return sent, err } +// discoPcapFrame marshals the bytes for a pcap record that describe a +// disco frame. +// +// Warning: Alloc garbage. Acceptable while capturing. +func discoPcapFrame(src netip.AddrPort, derpNodeSrc key.NodePublic, payload []byte) []byte { + var ( + b bytes.Buffer + flag uint8 + ) + b.Grow(128) // Most disco frames will probably be smaller than this. + + if src.Addr() == derpMagicIPAddr { + flag |= 0x01 + } + b.WriteByte(flag) // 1b: flag + + derpSrc := derpNodeSrc.Raw32() + b.Write(derpSrc[:]) // 32b: derp public key + binary.Write(&b, binary.LittleEndian, uint16(src.Port())) // 2b: port + addr, _ := src.Addr().MarshalBinary() + binary.Write(&b, binary.LittleEndian, uint16(len(addr))) // 2b: len(addr) + b.Write(addr) // Xb: addr + binary.Write(&b, binary.LittleEndian, uint16(len(payload))) // 2b: len(payload) + b.Write(payload) // Xb: payload + + return b.Bytes() +} + // handleDiscoMessage handles a discovery message and reports whether // msg was a Tailscale inter-node discovery message. // @@ -2099,6 +2140,12 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke return } + // Emit information about the disco frame into the pcap stream + // if a capture hook is installed. + if cb := c.captureHook.Load(); cb != nil { + cb(capture.PathDisco, time.Now(), discoPcapFrame(src, derpNodeSrc, payload)) + } + dm, err := disco.Parse(payload) if debugDisco() { c.logf("magicsock: disco: disco.Parse = %T, %v", dm, err) diff --git a/wgengine/userspace.go b/wgengine/userspace.go index f08bbda2b..c31288229 100644 --- a/wgengine/userspace.go +++ b/wgengine/userspace.go @@ -44,6 +44,7 @@ import ( "tailscale.com/util/clientmetric" "tailscale.com/util/deephash" "tailscale.com/util/mak" + "tailscale.com/wgengine/capture" "tailscale.com/wgengine/filter" "tailscale.com/wgengine/magicsock" "tailscale.com/wgengine/monitor" @@ -1580,6 +1581,7 @@ var ( metricNumMinorChanges = clientmetric.NewCounter("wgengine_minor_changes") ) -func (e *userspaceEngine) InstallCaptureHook(cb CaptureCallback) { - e.tundev.InstallCaptureHook(tstun.CaptureFunc(cb)) +func (e *userspaceEngine) InstallCaptureHook(cb capture.Callback) { + e.tundev.InstallCaptureHook(cb) + e.magicConn.InstallCaptureHook(cb) } diff --git a/wgengine/watchdog.go b/wgengine/watchdog.go index d2bc7f2fc..c0129bd14 100644 --- a/wgengine/watchdog.go +++ b/wgengine/watchdog.go @@ -22,6 +22,7 @@ import ( "tailscale.com/tailcfg" "tailscale.com/types/key" "tailscale.com/types/netmap" + "tailscale.com/wgengine/capture" "tailscale.com/wgengine/filter" "tailscale.com/wgengine/magicsock" "tailscale.com/wgengine/monitor" @@ -201,6 +202,6 @@ func (e *watchdogEngine) Wait() { e.wrap.Wait() } -func (e *watchdogEngine) InstallCaptureHook(cb CaptureCallback) { +func (e *watchdogEngine) InstallCaptureHook(cb capture.Callback) { e.wrap.InstallCaptureHook(cb) } diff --git a/wgengine/wgengine.go b/wgengine/wgengine.go index 914b4102a..4178ec619 100644 --- a/wgengine/wgengine.go +++ b/wgengine/wgengine.go @@ -43,12 +43,6 @@ type NetInfoCallback func(*tailcfg.NetInfo) // into network map updates. type NetworkMapCallback func(*netmap.NetworkMap) -// CaptureCallback is the type used to record packets when -// debugging packet-capture. This function must not take -// ownership of the provided data slice: it may only copy -// out of it within the lifetime of the function. -type CaptureCallback func(capture.Path, time.Time, []byte) - // someHandle is allocated so its pointer address acts as a unique // map key handle. (It needs to have non-zero size for Go to guarantee // the pointer is unique.) @@ -182,5 +176,5 @@ type Engine interface { // InstallCaptureHook registers a function to be called to capture // packets traversing the data path. The hook can be uninstalled by // calling this function with a nil value. - InstallCaptureHook(CaptureCallback) + InstallCaptureHook(capture.Callback) }