From b560386c1a607674f1baa83dfc2a406c05fb5d46 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 12 Jan 2021 12:03:41 -0800 Subject: [PATCH] net/packet, wgengine, tstun: add inter-node TSMP protocol for connect errors This adds a new IP Protocol type, TSMP on protocol number 99 for sending inter-tailscale messages over WireGuard, currently just for why a peer rejects TCP SYNs (ACL rejection, shields up, and in the future: nothing listening, something listening on that port but wrong interface, etc) Updates #1094 Updates tailscale/corp#1185 Signed-off-by: Brad Fitzpatrick --- ipn/local.go | 3 +- net/packet/header.go | 3 - net/packet/ip.go | 13 ++++ net/packet/packet.go | 8 +++ net/packet/packet_test.go | 46 ++++++++++++- net/packet/tsmp.go | 140 ++++++++++++++++++++++++++++++++++++++ net/packet/tsmp_test.go | 63 +++++++++++++++++ wgengine/filter/filter.go | 27 +++++++- wgengine/pendopen.go | 48 ++++++++----- wgengine/tstun/tun.go | 52 ++++++++++---- 10 files changed, 362 insertions(+), 41 deletions(-) create mode 100644 net/packet/tsmp.go create mode 100644 net/packet/tsmp_test.go diff --git a/ipn/local.go b/ipn/local.go index ba2733b16..7e2c60909 100644 --- a/ipn/local.go +++ b/ipn/local.go @@ -564,8 +564,7 @@ func (b *LocalBackend) updateFilter(netMap *controlclient.NetworkMap, prefs *Pre if shieldsUp { b.logf("netmap packet filter: (shields up)") - var prevFilter *filter.Filter // don't reuse old filter state - b.e.SetFilter(filter.New(nil, localNets, prevFilter, b.logf)) + b.e.SetFilter(filter.NewShieldsUpFilter(b.logf)) } else { b.logf("netmap packet filter: %v", packetFilter) b.e.SetFilter(filter.New(packetFilter, localNets, b.e.GetFilter(), b.logf)) diff --git a/net/packet/header.go b/net/packet/header.go index ed7306f3a..86680a5a7 100644 --- a/net/packet/header.go +++ b/net/packet/header.go @@ -36,9 +36,6 @@ type Header interface { // purpose of computing length and checksum fields. Marshal // implementations must not allocate memory. Marshal(buf []byte) error - // ToResponse transforms the header into one for a response packet. - // For instance, this swaps the source and destination IPs. - ToResponse() } // Generate generates a new packet with the given Header and diff --git a/net/packet/ip.go b/net/packet/ip.go index a5703433e..34194f344 100644 --- a/net/packet/ip.go +++ b/net/packet/ip.go @@ -24,6 +24,17 @@ const ( TCP IPProto = 0x06 UDP IPProto = 0x11 + // TSMP is the Tailscale Message Protocol (our ICMP-ish + // thing), an IP protocol used only between Tailscale nodes + // (still encrypted by WireGuard) that communicates why things + // failed, etc. + // + // Proto number 99 is reserved for "any private encryption + // scheme". We never accept these from the host OS stack nor + // send them to the host network stack. It's only used between + // nodes. + TSMP IPProto = 99 + // Fragment represents any non-first IP fragment, for which we // don't have the sub-protocol header (and therefore can't // figure out what the sub-protocol is). @@ -47,6 +58,8 @@ func (p IPProto) String() string { return "UDP" case TCP: return "TCP" + case TSMP: + return "TSMP" default: return "Unknown" } diff --git a/net/packet/packet.go b/net/packet/packet.go index 29dfed514..eab304ae8 100644 --- a/net/packet/packet.go +++ b/net/packet/packet.go @@ -204,6 +204,10 @@ func (q *Parsed) decode4(b []byte) { q.Dst.Port = binary.BigEndian.Uint16(sub[2:4]) q.dataofs = q.subofs + udpHeaderLength return + case TSMP: + // Inter-tailscale messages. + q.dataofs = q.subofs + return default: q.IPProto = Unknown return @@ -291,6 +295,10 @@ func (q *Parsed) decode6(b []byte) { q.Src.Port = binary.BigEndian.Uint16(sub[0:2]) q.Dst.Port = binary.BigEndian.Uint16(sub[2:4]) q.dataofs = q.subofs + udpHeaderLength + case TSMP: + // Inter-tailscale messages. + q.dataofs = q.subofs + return default: q.IPProto = Unknown return diff --git a/net/packet/packet_test.go b/net/packet/packet_test.go index 21615e519..8bac5db4a 100644 --- a/net/packet/packet_test.go +++ b/net/packet/packet_test.go @@ -274,7 +274,38 @@ var igmpPacketDecode = Parsed{ Dst: mustIPPort("224.0.0.251:0"), } -func TestParsed(t *testing.T) { +var ipv4TSMPBuffer = []byte{ + // IPv4 header: + 0x45, 0x00, + 0x00, 0x1b, // 20 + 7 bytes total + 0x00, 0x00, // ID + 0x00, 0x00, // Fragment + 0x40, // TTL + byte(TSMP), + 0x5f, 0xc3, // header checksum (wrong here) + // source IP: + 0x64, 0x5e, 0x0c, 0x0e, + // dest IP: + 0x64, 0x4a, 0x46, 0x03, + byte(TSMPTypeRejectedConn), + byte(TCP), + byte(RejectedDueToACLs), + 0x00, 123, // src port + 0x00, 80, // dst port +} + +var ipv4TSMPDecode = Parsed{ + b: ipv4TSMPBuffer, + subofs: 20, + dataofs: 20, + length: 27, + IPVersion: 4, + IPProto: TSMP, + Src: mustIPPort("100.94.12.14:0"), + Dst: mustIPPort("100.74.70.3:0"), +} + +func TestParsedString(t *testing.T) { tests := []struct { name string qdecode Parsed @@ -288,6 +319,7 @@ func TestParsed(t *testing.T) { {"icmp6", icmp6PacketDecode, "ICMPv6{[fe80::fb57:1dea:9c39:8fb7]:0 > [ff02::2]:0}"}, {"igmp", igmpPacketDecode, "IGMP{192.168.1.82:0 > 224.0.0.251:0}"}, {"unknown", unknownPacketDecode, "Unknown{???}"}, + {"ipv4_tsmp", ipv4TSMPDecode, "TSMP{100.94.12.14:0 > 100.74.70.3:0}"}, } for _, tt := range tests { @@ -324,6 +356,7 @@ func TestDecode(t *testing.T) { {"igmp", igmpPacketBuffer, igmpPacketDecode}, {"unknown", unknownPacketBuffer, unknownPacketDecode}, {"invalid4", invalid4RequestBuffer, invalid4RequestDecode}, + {"ipv4_tsmp", ipv4TSMPBuffer, ipv4TSMPDecode}, } for _, tt := range tests { @@ -331,7 +364,7 @@ func TestDecode(t *testing.T) { var got Parsed got.Decode(tt.buf) if !reflect.DeepEqual(got, tt.want) { - t.Errorf("mismatch\n got: %#v\nwant: %#v", got, tt.want) + t.Errorf("mismatch\n got: %s %#v\nwant: %s %#v", got.String(), got, tt.want.String(), tt.want) } }) } @@ -416,9 +449,16 @@ func TestMarshalResponse(t *testing.T) { icmpHeader := icmp4RequestDecode.ICMP4Header() udpHeader := udp4RequestDecode.UDP4Header() + type HeaderToResponser interface { + Header + // ToResponse transforms the header into one for a response packet. + // For instance, this swaps the source and destination IPs. + ToResponse() + } + tests := []struct { name string - header Header + header HeaderToResponser want []byte }{ {"icmp", &icmpHeader, icmp4ReplyBuffer}, diff --git a/net/packet/tsmp.go b/net/packet/tsmp.go new file mode 100644 index 000000000..8b46a6c98 --- /dev/null +++ b/net/packet/tsmp.go @@ -0,0 +1,140 @@ +// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TSMP is our ICMP-like "Tailscale Message Protocol" for signaling +// Tailscale-specific messages between nodes. It uses IP protocol 99 +// (reserved for "any private encryption scheme") within +// Wireguard's normal encryption between peers and never hits the host +// network stack. + +package packet + +import ( + "encoding/binary" + "errors" + "fmt" + + "inet.af/netaddr" + "tailscale.com/net/flowtrack" +) + +// TailscaleRejectedHeader is a TSMP message that says that one +// Tailscale node has rejected the connection from another. Unlike a +// TCP RST, this includes a reason. +// +// On the wire, after the IP header, it's currently 7 bytes: +// * '!' +// * IPProto byte (IANA protocol number: TCP or UDP) +// * 'A' or 'S' (RejectedDueToACLs, RejectedDueToShieldsUp) +// * srcPort big endian uint16 +// * dstPort big endian uint16 +// +// In the future it might also accept 16 byte IP flow src/dst IPs +// after the header, if they're different than the IP-level ones. +type TailscaleRejectedHeader struct { + IPSrc netaddr.IP // IPv4 or IPv6 header's src IP + IPDst netaddr.IP // IPv4 or IPv6 header's dst IP + Src netaddr.IPPort // rejected flow's src + Dst netaddr.IPPort // rejected flow's dst + Proto IPProto // proto that was rejected (TCP or UDP) + Reason TailscaleRejectReason // why the connection was rejected +} + +func (rh TailscaleRejectedHeader) Flow() flowtrack.Tuple { + return flowtrack.Tuple{Src: rh.Src, Dst: rh.Dst} +} + +func (rh TailscaleRejectedHeader) String() string { + return fmt.Sprintf("TSMP-reject-flow{%s %s > %s}: %s", rh.Proto, rh.Src, rh.Dst, rh.Reason) +} + +type TSMPType uint8 + +const ( + TSMPTypeRejectedConn TSMPType = '!' +) + +type TailscaleRejectReason byte + +const ( + RejectedDueToACLs TailscaleRejectReason = 'A' + RejectedDueToShieldsUp TailscaleRejectReason = 'S' +) + +func (r TailscaleRejectReason) String() string { + switch r { + case RejectedDueToACLs: + return "acl" + case RejectedDueToShieldsUp: + return "shields" + } + return fmt.Sprintf("0x%02x", byte(r)) +} + +func (h TailscaleRejectedHeader) Len() int { + var ipHeaderLen int + if h.IPSrc.Is4() { + ipHeaderLen = ip4HeaderLength + } else if h.IPSrc.Is6() { + ipHeaderLen = ip6HeaderLength + } + return ipHeaderLen + + 1 + // TSMPType byte + 1 + // IPProto byte + 1 + // TailscaleRejectReason byte + 2*2 // 2 uint16 ports +} + +func (h TailscaleRejectedHeader) Marshal(buf []byte) error { + if len(buf) < h.Len() { + return errSmallBuffer + } + if len(buf) > maxPacketLength { + return errLargePacket + } + if h.Src.IP.Is4() { + iph := IP4Header{ + IPProto: TSMP, + Src: h.IPSrc, + Dst: h.IPDst, + } + iph.Marshal(buf) + buf = buf[ip4HeaderLength:] + } else if h.Src.IP.Is6() { + iph := IP6Header{ + IPProto: TSMP, + Src: h.IPSrc, + Dst: h.IPDst, + } + iph.Marshal(buf) + buf = buf[ip6HeaderLength:] + } else { + return errors.New("bogus src IP") + } + buf[0] = byte(TSMPTypeRejectedConn) + buf[1] = byte(h.Proto) + buf[2] = byte(h.Reason) + binary.BigEndian.PutUint16(buf[3:5], h.Src.Port) + binary.BigEndian.PutUint16(buf[5:7], h.Dst.Port) + return nil +} + +// AsTailscaleRejectedHeader parses pp as an incoming rejection +// connection TSMP message. +// +// ok reports whether pp was a valid TSMP rejection packet. +func (pp *Parsed) AsTailscaleRejectedHeader() (h TailscaleRejectedHeader, ok bool) { + p := pp.Payload() + if len(p) < 7 || p[0] != byte(TSMPTypeRejectedConn) { + return + } + return TailscaleRejectedHeader{ + Proto: IPProto(p[1]), + Reason: TailscaleRejectReason(p[2]), + IPSrc: pp.Src.IP, + IPDst: pp.Dst.IP, + Src: netaddr.IPPort{IP: pp.Dst.IP, Port: binary.BigEndian.Uint16(p[3:5])}, + Dst: netaddr.IPPort{IP: pp.Src.IP, Port: binary.BigEndian.Uint16(p[5:7])}, + }, true +} diff --git a/net/packet/tsmp_test.go b/net/packet/tsmp_test.go new file mode 100644 index 000000000..71e4f9439 --- /dev/null +++ b/net/packet/tsmp_test.go @@ -0,0 +1,63 @@ +// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package packet + +import ( + "testing" + + "inet.af/netaddr" +) + +func TestTailscaleRejectedHeader(t *testing.T) { + tests := []struct { + h TailscaleRejectedHeader + wantStr string + }{ + { + h: TailscaleRejectedHeader{ + IPSrc: netaddr.MustParseIP("5.5.5.5"), + IPDst: netaddr.MustParseIP("1.2.3.4"), + Src: netaddr.MustParseIPPort("1.2.3.4:567"), + Dst: netaddr.MustParseIPPort("5.5.5.5:443"), + Proto: TCP, + Reason: RejectedDueToACLs, + }, + wantStr: "TSMP-reject-flow{TCP 1.2.3.4:567 > 5.5.5.5:443}: acl", + }, + { + h: TailscaleRejectedHeader{ + IPSrc: netaddr.MustParseIP("2::2"), + IPDst: netaddr.MustParseIP("1::1"), + Src: netaddr.MustParseIPPort("[1::1]:567"), + Dst: netaddr.MustParseIPPort("[2::2]:443"), + Proto: UDP, + Reason: RejectedDueToShieldsUp, + }, + wantStr: "TSMP-reject-flow{UDP [1::1]:567 > [2::2]:443}: shields", + }, + } + for i, tt := range tests { + gotStr := tt.h.String() + if gotStr != tt.wantStr { + t.Errorf("%v. String = %q; want %q", i, gotStr, tt.wantStr) + continue + } + pkt := make([]byte, tt.h.Len()) + tt.h.Marshal(pkt) + + var p Parsed + p.Decode(pkt) + t.Logf("Parsed: %+v", p) + t.Logf("Parsed: %s", p.String()) + back, ok := p.AsTailscaleRejectedHeader() + if !ok { + t.Errorf("%v. %q (%02x) didn't parse back", i, gotStr, pkt) + continue + } + if back != tt.h { + t.Errorf("%v. %q parsed back as %q", i, tt.h, back) + } + } +} diff --git a/wgengine/filter/filter.go b/wgengine/filter/filter.go index a0bdbf3af..65c6fd289 100644 --- a/wgengine/filter/filter.go +++ b/wgengine/filter/filter.go @@ -39,6 +39,8 @@ type Filter struct { // to an outbound connection that this node made, even if those // incoming packets don't get accepted by matches above. state *filterState + + shieldsUp bool } // filterState is a state cache of past seen packets. @@ -54,15 +56,18 @@ const lruMax = 512 type Response int const ( - Drop Response = iota // do not continue processing packet. - Accept // continue processing packet. - noVerdict // no verdict yet, continue running filter + Drop Response = iota // do not continue processing packet. + DropSilently // do not continue processing packet, but also don't log + Accept // continue processing packet. + noVerdict // no verdict yet, continue running filter ) func (r Response) String() string { switch r { case Drop: return "Drop" + case DropSilently: + return "DropSilently" case Accept: return "Accept" case noVerdict: @@ -72,6 +77,10 @@ func (r Response) String() string { } } +func (r Response) IsDrop() bool { + return r == Drop || r == DropSilently +} + // RunFlags controls the filter's debug log verbosity at runtime. type RunFlags int @@ -123,6 +132,12 @@ func NewAllowNone(logf logger.Logf) *Filter { return New(nil, nil, nil, logf) } +func NewShieldsUpFilter(logf logger.Logf) *Filter { + f := New(nil, nil, nil, logf) + f.shieldsUp = true + return f +} + // New creates a new packet filter. The filter enforces that incoming // packets must be destined to an IP in localNets, and must be allowed // by matches. If shareStateWith is non-nil, the returned filter @@ -253,6 +268,10 @@ func (f *Filter) CheckTCP(srcIP, dstIP netaddr.IP, dstPort uint16) Response { return f.RunIn(pkt, 0) } +// ShieldsUp reports whether this is a "shields up" (block everything +// incoming) filter. +func (f *Filter) ShieldsUp() bool { return f.shieldsUp } + // RunIn determines whether this node is allowed to receive q from a // Tailscale peer. func (f *Filter) RunIn(q *packet.Parsed, rf RunFlags) Response { @@ -339,6 +358,8 @@ func (f *Filter) runIn4(q *packet.Parsed) (r Response, why string) { if f.matches4.match(q) { return Accept, "udp ok" } + case packet.TSMP: + return Accept, "tsmp ok" default: return Drop, "Unknown proto" } diff --git a/wgengine/pendopen.go b/wgengine/pendopen.go index dc1a21fec..a4d0a6f8b 100644 --- a/wgengine/pendopen.go +++ b/wgengine/pendopen.go @@ -32,35 +32,47 @@ type pendingOpenFlow struct { timer *time.Timer // until giving up on the flow } +func (e *userspaceEngine) removeFlow(f flowtrack.Tuple) (removed bool) { + e.mu.Lock() + defer e.mu.Unlock() + of, ok := e.pendOpen[f] + if !ok { + // Not a tracked flow (likely already removed) + return false + } + of.timer.Stop() + delete(e.pendOpen, f) + return true +} + func (e *userspaceEngine) trackOpenPreFilterIn(pp *packet.Parsed, t *tstun.TUN) (res filter.Response) { res = filter.Accept // always + if pp.IPProto == packet.TSMP { + res = filter.DropSilently + rh, ok := pp.AsTailscaleRejectedHeader() + if !ok { + return + } + if f := rh.Flow(); e.removeFlow(f) { + e.logf("open-conn-track: flow %v %v > %v rejected due to %v", rh.Proto, rh.Src, rh.Dst, rh.Reason) + } + return + } + if pp.IPVersion == 0 || pp.IPProto != packet.TCP || pp.TCPFlags&(packet.TCPSyn|packet.TCPRst) == 0 { return } - flow := flowtrack.Tuple{Dst: pp.Src, Src: pp.Dst} // src/dst reversed + // Either a SYN or a RST came back. Remove it in either case. - e.mu.Lock() - defer e.mu.Unlock() - of, ok := e.pendOpen[flow] - if !ok { - // Not a tracked flow. - return + f := flowtrack.Tuple{Dst: pp.Src, Src: pp.Dst} // src/dst reversed + removed := e.removeFlow(f) + if removed && pp.TCPFlags&packet.TCPRst != 0 { + e.logf("open-conn-track: flow TCP %v got RST by peer", f) } - of.timer.Stop() - delete(e.pendOpen, flow) - - if pp.TCPFlags&packet.TCPRst != 0 { - // TODO(bradfitz): have peer send a IP proto 99 "why" - // packet first with details and log that instead - // (e.g. ACL prohibited, shields up, etc). - e.logf("open-conn-track: flow %v got RST by peer", flow) - return - } - return } diff --git a/wgengine/tstun/tun.go b/wgengine/tstun/tun.go index a1a9a2a3d..8d73c065a 100644 --- a/wgengine/tstun/tun.go +++ b/wgengine/tstun/tun.go @@ -218,8 +218,8 @@ func (t *TUN) poll() { func (t *TUN) filterOut(p *packet.Parsed) filter.Response { if t.PreFilterOut != nil { - if t.PreFilterOut(p, t) == filter.Drop { - return filter.Drop + if res := t.PreFilterOut(p, t); res.IsDrop() { + return res } } @@ -234,8 +234,8 @@ func (t *TUN) filterOut(p *packet.Parsed) filter.Response { } if t.PostFilterOut != nil { - if t.PostFilterOut(p, t) == filter.Drop { - return filter.Drop + if res := t.PostFilterOut(p, t); res.IsDrop() { + return res } } @@ -264,12 +264,12 @@ func (t *TUN) Read(buf []byte, offset int) (int, error) { return 0, io.EOF case err := <-t.errors: return 0, err - case packet := <-t.outbound: - n = copy(buf[offset:], packet) + case pkt := <-t.outbound: + n = copy(buf[offset:], pkt) // t.buffer has a fixed location in memory, // so this is the easiest way to tell when it has been consumed. - // &packet[0] can be used because empty packets do not reach t.outbound. - if &packet[0] == &t.buffer[PacketStartOffset] { + // &pkt[0] can be used because empty packets do not reach t.outbound. + if &pkt[0] == &t.buffer[PacketStartOffset] { t.bufferConsumed <- struct{}{} } else { // If the packet is not from t.buffer, then it is an injected packet. @@ -307,8 +307,8 @@ func (t *TUN) filterIn(buf []byte) filter.Response { p.Decode(buf) if t.PreFilterIn != nil { - if t.PreFilterIn(p, t) == filter.Drop { - return filter.Drop + if res := t.PreFilterIn(p, t); res.IsDrop() { + return res } } @@ -319,6 +319,29 @@ func (t *TUN) filterIn(buf []byte) filter.Response { } if filt.RunIn(p, t.filterFlags) != filter.Accept { + + // Tell them, via TSMP, we're dropping them due to the ACL. + // Their host networking stack can translate this into ICMP + // or whatnot as required. But notably, their GUI or tailscale CLI + // can show them a rejection history with reasons. + if p.IPVersion == 4 && p.IPProto == packet.TCP && p.TCPFlags&packet.TCPSyn != 0 { + rj := packet.TailscaleRejectedHeader{ + IPSrc: p.Dst.IP, + IPDst: p.Src.IP, + Src: p.Src, + Dst: p.Dst, + Proto: p.IPProto, + Reason: packet.RejectedDueToACLs, + } + if filt.ShieldsUp() { + rj.Reason = packet.RejectedDueToShieldsUp + } + pkt := packet.Generate(rj, nil) + t.InjectOutbound(pkt) + + // TODO(bradfitz): also send a TCP RST, after the TSMP message. + } + return filter.Drop } @@ -331,10 +354,15 @@ func (t *TUN) filterIn(buf []byte) filter.Response { return filter.Accept } +// Write accepts an incoming packet. The packet begins at buf[offset:], +// like wireguard-go/tun.Device.Write. func (t *TUN) Write(buf []byte, offset int) (int, error) { if !t.disableFilter { - response := t.filterIn(buf[offset:]) - if response != filter.Accept { + res := t.filterIn(buf[offset:]) + if res == filter.DropSilently { + return len(buf), nil + } + if res != filter.Accept { return 0, ErrFiltered } }