net/packet, wgengine, tstun: add inter-node TSMP protocol for connect errors

This adds a new IP Protocol type, TSMP on protocol number 99 for
sending inter-tailscale messages over WireGuard, currently just for
why a peer rejects TCP SYNs (ACL rejection, shields up, and in the
future: nothing listening, something listening on that port but wrong
interface, etc)

Updates #1094
Updates tailscale/corp#1185

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>
pull/1135/head
Brad Fitzpatrick 3 years ago committed by Brad Fitzpatrick
parent 01e8b7fb7e
commit b560386c1a

@ -564,8 +564,7 @@ func (b *LocalBackend) updateFilter(netMap *controlclient.NetworkMap, prefs *Pre
if shieldsUp {
b.logf("netmap packet filter: (shields up)")
var prevFilter *filter.Filter // don't reuse old filter state
b.e.SetFilter(filter.New(nil, localNets, prevFilter, b.logf))
b.e.SetFilter(filter.NewShieldsUpFilter(b.logf))
} else {
b.logf("netmap packet filter: %v", packetFilter)
b.e.SetFilter(filter.New(packetFilter, localNets, b.e.GetFilter(), b.logf))

@ -36,9 +36,6 @@ type Header interface {
// purpose of computing length and checksum fields. Marshal
// implementations must not allocate memory.
Marshal(buf []byte) error
// ToResponse transforms the header into one for a response packet.
// For instance, this swaps the source and destination IPs.
ToResponse()
}
// Generate generates a new packet with the given Header and

@ -24,6 +24,17 @@ const (
TCP IPProto = 0x06
UDP IPProto = 0x11
// TSMP is the Tailscale Message Protocol (our ICMP-ish
// thing), an IP protocol used only between Tailscale nodes
// (still encrypted by WireGuard) that communicates why things
// failed, etc.
//
// Proto number 99 is reserved for "any private encryption
// scheme". We never accept these from the host OS stack nor
// send them to the host network stack. It's only used between
// nodes.
TSMP IPProto = 99
// Fragment represents any non-first IP fragment, for which we
// don't have the sub-protocol header (and therefore can't
// figure out what the sub-protocol is).
@ -47,6 +58,8 @@ func (p IPProto) String() string {
return "UDP"
case TCP:
return "TCP"
case TSMP:
return "TSMP"
default:
return "Unknown"
}

@ -204,6 +204,10 @@ func (q *Parsed) decode4(b []byte) {
q.Dst.Port = binary.BigEndian.Uint16(sub[2:4])
q.dataofs = q.subofs + udpHeaderLength
return
case TSMP:
// Inter-tailscale messages.
q.dataofs = q.subofs
return
default:
q.IPProto = Unknown
return
@ -291,6 +295,10 @@ func (q *Parsed) decode6(b []byte) {
q.Src.Port = binary.BigEndian.Uint16(sub[0:2])
q.Dst.Port = binary.BigEndian.Uint16(sub[2:4])
q.dataofs = q.subofs + udpHeaderLength
case TSMP:
// Inter-tailscale messages.
q.dataofs = q.subofs
return
default:
q.IPProto = Unknown
return

@ -274,7 +274,38 @@ var igmpPacketDecode = Parsed{
Dst: mustIPPort("224.0.0.251:0"),
}
func TestParsed(t *testing.T) {
var ipv4TSMPBuffer = []byte{
// IPv4 header:
0x45, 0x00,
0x00, 0x1b, // 20 + 7 bytes total
0x00, 0x00, // ID
0x00, 0x00, // Fragment
0x40, // TTL
byte(TSMP),
0x5f, 0xc3, // header checksum (wrong here)
// source IP:
0x64, 0x5e, 0x0c, 0x0e,
// dest IP:
0x64, 0x4a, 0x46, 0x03,
byte(TSMPTypeRejectedConn),
byte(TCP),
byte(RejectedDueToACLs),
0x00, 123, // src port
0x00, 80, // dst port
}
var ipv4TSMPDecode = Parsed{
b: ipv4TSMPBuffer,
subofs: 20,
dataofs: 20,
length: 27,
IPVersion: 4,
IPProto: TSMP,
Src: mustIPPort("100.94.12.14:0"),
Dst: mustIPPort("100.74.70.3:0"),
}
func TestParsedString(t *testing.T) {
tests := []struct {
name string
qdecode Parsed
@ -288,6 +319,7 @@ func TestParsed(t *testing.T) {
{"icmp6", icmp6PacketDecode, "ICMPv6{[fe80::fb57:1dea:9c39:8fb7]:0 > [ff02::2]:0}"},
{"igmp", igmpPacketDecode, "IGMP{192.168.1.82:0 > 224.0.0.251:0}"},
{"unknown", unknownPacketDecode, "Unknown{???}"},
{"ipv4_tsmp", ipv4TSMPDecode, "TSMP{100.94.12.14:0 > 100.74.70.3:0}"},
}
for _, tt := range tests {
@ -324,6 +356,7 @@ func TestDecode(t *testing.T) {
{"igmp", igmpPacketBuffer, igmpPacketDecode},
{"unknown", unknownPacketBuffer, unknownPacketDecode},
{"invalid4", invalid4RequestBuffer, invalid4RequestDecode},
{"ipv4_tsmp", ipv4TSMPBuffer, ipv4TSMPDecode},
}
for _, tt := range tests {
@ -331,7 +364,7 @@ func TestDecode(t *testing.T) {
var got Parsed
got.Decode(tt.buf)
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("mismatch\n got: %#v\nwant: %#v", got, tt.want)
t.Errorf("mismatch\n got: %s %#v\nwant: %s %#v", got.String(), got, tt.want.String(), tt.want)
}
})
}
@ -416,9 +449,16 @@ func TestMarshalResponse(t *testing.T) {
icmpHeader := icmp4RequestDecode.ICMP4Header()
udpHeader := udp4RequestDecode.UDP4Header()
type HeaderToResponser interface {
Header
// ToResponse transforms the header into one for a response packet.
// For instance, this swaps the source and destination IPs.
ToResponse()
}
tests := []struct {
name string
header Header
header HeaderToResponser
want []byte
}{
{"icmp", &icmpHeader, icmp4ReplyBuffer},

@ -0,0 +1,140 @@
// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// TSMP is our ICMP-like "Tailscale Message Protocol" for signaling
// Tailscale-specific messages between nodes. It uses IP protocol 99
// (reserved for "any private encryption scheme") within
// Wireguard's normal encryption between peers and never hits the host
// network stack.
package packet
import (
"encoding/binary"
"errors"
"fmt"
"inet.af/netaddr"
"tailscale.com/net/flowtrack"
)
// TailscaleRejectedHeader is a TSMP message that says that one
// Tailscale node has rejected the connection from another. Unlike a
// TCP RST, this includes a reason.
//
// On the wire, after the IP header, it's currently 7 bytes:
// * '!'
// * IPProto byte (IANA protocol number: TCP or UDP)
// * 'A' or 'S' (RejectedDueToACLs, RejectedDueToShieldsUp)
// * srcPort big endian uint16
// * dstPort big endian uint16
//
// In the future it might also accept 16 byte IP flow src/dst IPs
// after the header, if they're different than the IP-level ones.
type TailscaleRejectedHeader struct {
IPSrc netaddr.IP // IPv4 or IPv6 header's src IP
IPDst netaddr.IP // IPv4 or IPv6 header's dst IP
Src netaddr.IPPort // rejected flow's src
Dst netaddr.IPPort // rejected flow's dst
Proto IPProto // proto that was rejected (TCP or UDP)
Reason TailscaleRejectReason // why the connection was rejected
}
func (rh TailscaleRejectedHeader) Flow() flowtrack.Tuple {
return flowtrack.Tuple{Src: rh.Src, Dst: rh.Dst}
}
func (rh TailscaleRejectedHeader) String() string {
return fmt.Sprintf("TSMP-reject-flow{%s %s > %s}: %s", rh.Proto, rh.Src, rh.Dst, rh.Reason)
}
type TSMPType uint8
const (
TSMPTypeRejectedConn TSMPType = '!'
)
type TailscaleRejectReason byte
const (
RejectedDueToACLs TailscaleRejectReason = 'A'
RejectedDueToShieldsUp TailscaleRejectReason = 'S'
)
func (r TailscaleRejectReason) String() string {
switch r {
case RejectedDueToACLs:
return "acl"
case RejectedDueToShieldsUp:
return "shields"
}
return fmt.Sprintf("0x%02x", byte(r))
}
func (h TailscaleRejectedHeader) Len() int {
var ipHeaderLen int
if h.IPSrc.Is4() {
ipHeaderLen = ip4HeaderLength
} else if h.IPSrc.Is6() {
ipHeaderLen = ip6HeaderLength
}
return ipHeaderLen +
1 + // TSMPType byte
1 + // IPProto byte
1 + // TailscaleRejectReason byte
2*2 // 2 uint16 ports
}
func (h TailscaleRejectedHeader) Marshal(buf []byte) error {
if len(buf) < h.Len() {
return errSmallBuffer
}
if len(buf) > maxPacketLength {
return errLargePacket
}
if h.Src.IP.Is4() {
iph := IP4Header{
IPProto: TSMP,
Src: h.IPSrc,
Dst: h.IPDst,
}
iph.Marshal(buf)
buf = buf[ip4HeaderLength:]
} else if h.Src.IP.Is6() {
iph := IP6Header{
IPProto: TSMP,
Src: h.IPSrc,
Dst: h.IPDst,
}
iph.Marshal(buf)
buf = buf[ip6HeaderLength:]
} else {
return errors.New("bogus src IP")
}
buf[0] = byte(TSMPTypeRejectedConn)
buf[1] = byte(h.Proto)
buf[2] = byte(h.Reason)
binary.BigEndian.PutUint16(buf[3:5], h.Src.Port)
binary.BigEndian.PutUint16(buf[5:7], h.Dst.Port)
return nil
}
// AsTailscaleRejectedHeader parses pp as an incoming rejection
// connection TSMP message.
//
// ok reports whether pp was a valid TSMP rejection packet.
func (pp *Parsed) AsTailscaleRejectedHeader() (h TailscaleRejectedHeader, ok bool) {
p := pp.Payload()
if len(p) < 7 || p[0] != byte(TSMPTypeRejectedConn) {
return
}
return TailscaleRejectedHeader{
Proto: IPProto(p[1]),
Reason: TailscaleRejectReason(p[2]),
IPSrc: pp.Src.IP,
IPDst: pp.Dst.IP,
Src: netaddr.IPPort{IP: pp.Dst.IP, Port: binary.BigEndian.Uint16(p[3:5])},
Dst: netaddr.IPPort{IP: pp.Src.IP, Port: binary.BigEndian.Uint16(p[5:7])},
}, true
}

@ -0,0 +1,63 @@
// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package packet
import (
"testing"
"inet.af/netaddr"
)
func TestTailscaleRejectedHeader(t *testing.T) {
tests := []struct {
h TailscaleRejectedHeader
wantStr string
}{
{
h: TailscaleRejectedHeader{
IPSrc: netaddr.MustParseIP("5.5.5.5"),
IPDst: netaddr.MustParseIP("1.2.3.4"),
Src: netaddr.MustParseIPPort("1.2.3.4:567"),
Dst: netaddr.MustParseIPPort("5.5.5.5:443"),
Proto: TCP,
Reason: RejectedDueToACLs,
},
wantStr: "TSMP-reject-flow{TCP 1.2.3.4:567 > 5.5.5.5:443}: acl",
},
{
h: TailscaleRejectedHeader{
IPSrc: netaddr.MustParseIP("2::2"),
IPDst: netaddr.MustParseIP("1::1"),
Src: netaddr.MustParseIPPort("[1::1]:567"),
Dst: netaddr.MustParseIPPort("[2::2]:443"),
Proto: UDP,
Reason: RejectedDueToShieldsUp,
},
wantStr: "TSMP-reject-flow{UDP [1::1]:567 > [2::2]:443}: shields",
},
}
for i, tt := range tests {
gotStr := tt.h.String()
if gotStr != tt.wantStr {
t.Errorf("%v. String = %q; want %q", i, gotStr, tt.wantStr)
continue
}
pkt := make([]byte, tt.h.Len())
tt.h.Marshal(pkt)
var p Parsed
p.Decode(pkt)
t.Logf("Parsed: %+v", p)
t.Logf("Parsed: %s", p.String())
back, ok := p.AsTailscaleRejectedHeader()
if !ok {
t.Errorf("%v. %q (%02x) didn't parse back", i, gotStr, pkt)
continue
}
if back != tt.h {
t.Errorf("%v. %q parsed back as %q", i, tt.h, back)
}
}
}

@ -39,6 +39,8 @@ type Filter struct {
// to an outbound connection that this node made, even if those
// incoming packets don't get accepted by matches above.
state *filterState
shieldsUp bool
}
// filterState is a state cache of past seen packets.
@ -54,15 +56,18 @@ const lruMax = 512
type Response int
const (
Drop Response = iota // do not continue processing packet.
Accept // continue processing packet.
noVerdict // no verdict yet, continue running filter
Drop Response = iota // do not continue processing packet.
DropSilently // do not continue processing packet, but also don't log
Accept // continue processing packet.
noVerdict // no verdict yet, continue running filter
)
func (r Response) String() string {
switch r {
case Drop:
return "Drop"
case DropSilently:
return "DropSilently"
case Accept:
return "Accept"
case noVerdict:
@ -72,6 +77,10 @@ func (r Response) String() string {
}
}
func (r Response) IsDrop() bool {
return r == Drop || r == DropSilently
}
// RunFlags controls the filter's debug log verbosity at runtime.
type RunFlags int
@ -123,6 +132,12 @@ func NewAllowNone(logf logger.Logf) *Filter {
return New(nil, nil, nil, logf)
}
func NewShieldsUpFilter(logf logger.Logf) *Filter {
f := New(nil, nil, nil, logf)
f.shieldsUp = true
return f
}
// New creates a new packet filter. The filter enforces that incoming
// packets must be destined to an IP in localNets, and must be allowed
// by matches. If shareStateWith is non-nil, the returned filter
@ -253,6 +268,10 @@ func (f *Filter) CheckTCP(srcIP, dstIP netaddr.IP, dstPort uint16) Response {
return f.RunIn(pkt, 0)
}
// ShieldsUp reports whether this is a "shields up" (block everything
// incoming) filter.
func (f *Filter) ShieldsUp() bool { return f.shieldsUp }
// RunIn determines whether this node is allowed to receive q from a
// Tailscale peer.
func (f *Filter) RunIn(q *packet.Parsed, rf RunFlags) Response {
@ -339,6 +358,8 @@ func (f *Filter) runIn4(q *packet.Parsed) (r Response, why string) {
if f.matches4.match(q) {
return Accept, "udp ok"
}
case packet.TSMP:
return Accept, "tsmp ok"
default:
return Drop, "Unknown proto"
}

@ -32,35 +32,47 @@ type pendingOpenFlow struct {
timer *time.Timer // until giving up on the flow
}
func (e *userspaceEngine) removeFlow(f flowtrack.Tuple) (removed bool) {
e.mu.Lock()
defer e.mu.Unlock()
of, ok := e.pendOpen[f]
if !ok {
// Not a tracked flow (likely already removed)
return false
}
of.timer.Stop()
delete(e.pendOpen, f)
return true
}
func (e *userspaceEngine) trackOpenPreFilterIn(pp *packet.Parsed, t *tstun.TUN) (res filter.Response) {
res = filter.Accept // always
if pp.IPProto == packet.TSMP {
res = filter.DropSilently
rh, ok := pp.AsTailscaleRejectedHeader()
if !ok {
return
}
if f := rh.Flow(); e.removeFlow(f) {
e.logf("open-conn-track: flow %v %v > %v rejected due to %v", rh.Proto, rh.Src, rh.Dst, rh.Reason)
}
return
}
if pp.IPVersion == 0 ||
pp.IPProto != packet.TCP ||
pp.TCPFlags&(packet.TCPSyn|packet.TCPRst) == 0 {
return
}
flow := flowtrack.Tuple{Dst: pp.Src, Src: pp.Dst} // src/dst reversed
// Either a SYN or a RST came back. Remove it in either case.
e.mu.Lock()
defer e.mu.Unlock()
of, ok := e.pendOpen[flow]
if !ok {
// Not a tracked flow.
return
f := flowtrack.Tuple{Dst: pp.Src, Src: pp.Dst} // src/dst reversed
removed := e.removeFlow(f)
if removed && pp.TCPFlags&packet.TCPRst != 0 {
e.logf("open-conn-track: flow TCP %v got RST by peer", f)
}
of.timer.Stop()
delete(e.pendOpen, flow)
if pp.TCPFlags&packet.TCPRst != 0 {
// TODO(bradfitz): have peer send a IP proto 99 "why"
// packet first with details and log that instead
// (e.g. ACL prohibited, shields up, etc).
e.logf("open-conn-track: flow %v got RST by peer", flow)
return
}
return
}

@ -218,8 +218,8 @@ func (t *TUN) poll() {
func (t *TUN) filterOut(p *packet.Parsed) filter.Response {
if t.PreFilterOut != nil {
if t.PreFilterOut(p, t) == filter.Drop {
return filter.Drop
if res := t.PreFilterOut(p, t); res.IsDrop() {
return res
}
}
@ -234,8 +234,8 @@ func (t *TUN) filterOut(p *packet.Parsed) filter.Response {
}
if t.PostFilterOut != nil {
if t.PostFilterOut(p, t) == filter.Drop {
return filter.Drop
if res := t.PostFilterOut(p, t); res.IsDrop() {
return res
}
}
@ -264,12 +264,12 @@ func (t *TUN) Read(buf []byte, offset int) (int, error) {
return 0, io.EOF
case err := <-t.errors:
return 0, err
case packet := <-t.outbound:
n = copy(buf[offset:], packet)
case pkt := <-t.outbound:
n = copy(buf[offset:], pkt)
// t.buffer has a fixed location in memory,
// so this is the easiest way to tell when it has been consumed.
// &packet[0] can be used because empty packets do not reach t.outbound.
if &packet[0] == &t.buffer[PacketStartOffset] {
// &pkt[0] can be used because empty packets do not reach t.outbound.
if &pkt[0] == &t.buffer[PacketStartOffset] {
t.bufferConsumed <- struct{}{}
} else {
// If the packet is not from t.buffer, then it is an injected packet.
@ -307,8 +307,8 @@ func (t *TUN) filterIn(buf []byte) filter.Response {
p.Decode(buf)
if t.PreFilterIn != nil {
if t.PreFilterIn(p, t) == filter.Drop {
return filter.Drop
if res := t.PreFilterIn(p, t); res.IsDrop() {
return res
}
}
@ -319,6 +319,29 @@ func (t *TUN) filterIn(buf []byte) filter.Response {
}
if filt.RunIn(p, t.filterFlags) != filter.Accept {
// Tell them, via TSMP, we're dropping them due to the ACL.
// Their host networking stack can translate this into ICMP
// or whatnot as required. But notably, their GUI or tailscale CLI
// can show them a rejection history with reasons.
if p.IPVersion == 4 && p.IPProto == packet.TCP && p.TCPFlags&packet.TCPSyn != 0 {
rj := packet.TailscaleRejectedHeader{
IPSrc: p.Dst.IP,
IPDst: p.Src.IP,
Src: p.Src,
Dst: p.Dst,
Proto: p.IPProto,
Reason: packet.RejectedDueToACLs,
}
if filt.ShieldsUp() {
rj.Reason = packet.RejectedDueToShieldsUp
}
pkt := packet.Generate(rj, nil)
t.InjectOutbound(pkt)
// TODO(bradfitz): also send a TCP RST, after the TSMP message.
}
return filter.Drop
}
@ -331,10 +354,15 @@ func (t *TUN) filterIn(buf []byte) filter.Response {
return filter.Accept
}
// Write accepts an incoming packet. The packet begins at buf[offset:],
// like wireguard-go/tun.Device.Write.
func (t *TUN) Write(buf []byte, offset int) (int, error) {
if !t.disableFilter {
response := t.filterIn(buf[offset:])
if response != filter.Accept {
res := t.filterIn(buf[offset:])
if res == filter.DropSilently {
return len(buf), nil
}
if res != filter.Accept {
return 0, ErrFiltered
}
}

Loading…
Cancel
Save