cmd/tta, vnet: add host firewall, env var support, more tests

In particular, tests showing that #3824 works. But that test doesn't
actually work yet; it only gets a DERP connection. (why?)

Updates #13038

Change-Id: Ie1fd1b6a38d4e90fae7e72a0b9a142a95f0b2e8f
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>
pull/13114/head
Brad Fitzpatrick 3 months ago committed by Brad Fitzpatrick
parent b692985aef
commit a61825c7b8

@ -0,0 +1,128 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package main
import (
"encoding/binary"
"github.com/google/nftables"
"github.com/google/nftables/expr"
"tailscale.com/types/ptr"
)
func init() {
addFirewall = addFirewallLinux
}
func addFirewallLinux() error {
c, err := nftables.New()
if err != nil {
return err
}
// Create a new table
table := &nftables.Table{
Family: nftables.TableFamilyIPv4, // TableFamilyINet doesn't work (why?. oh well.)
Name: "filter",
}
c.AddTable(table)
// Create a new chain for incoming traffic
inputChain := &nftables.Chain{
Name: "input",
Table: table,
Type: nftables.ChainTypeFilter,
Hooknum: nftables.ChainHookInput,
Priority: nftables.ChainPriorityFilter,
Policy: ptr.To(nftables.ChainPolicyDrop),
}
c.AddChain(inputChain)
// Allow traffic from the loopback interface
c.AddRule(&nftables.Rule{
Table: table,
Chain: inputChain,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte("lo"),
},
&expr.Verdict{
Kind: expr.VerdictAccept,
},
},
})
// Accept established and related connections
c.AddRule(&nftables.Rule{
Table: table,
Chain: inputChain,
Exprs: []expr.Any{
&expr.Ct{
Register: 1,
Key: expr.CtKeySTATE,
},
&expr.Bitwise{
SourceRegister: 1,
DestRegister: 1,
Len: 4,
Mask: binary.NativeEndian.AppendUint32(nil, 0x06), // CT_STATE_BIT_ESTABLISHED | CT_STATE_BIT_RELATED
Xor: binary.NativeEndian.AppendUint32(nil, 0),
},
&expr.Cmp{
Op: expr.CmpOpNeq,
Register: 1,
Data: binary.NativeEndian.AppendUint32(nil, 0x00),
},
&expr.Verdict{
Kind: expr.VerdictAccept,
},
},
})
// Allow TCP packets in that don't have the SYN bit set, even if they're not
// ESTABLISHED or RELATED. This is because the test suite gets TCP
// connections up & idle (for HTTP) before it conditionally installs these
// firewall rules. But because conntrack wasn't previously active, existing
// TCP flows aren't ESTABLISHED and get dropped. So this rule allows
// previously established TCP connections that predates the firewall rules
// to continue working, as they don't have conntrack state.
c.AddRule(&nftables.Rule{
Table: table,
Chain: inputChain,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte{0x06}, // TCP
},
&expr.Payload{ // get TCP flags
DestRegister: 1,
Base: 2,
Offset: 13, // flags
Len: 1,
},
&expr.Bitwise{
SourceRegister: 1,
DestRegister: 1,
Len: 1,
Mask: []byte{2}, // TCP_SYN
Xor: []byte{0},
},
&expr.Cmp{
Op: expr.CmpOpNeq,
Register: 1,
Data: []byte{2}, // TCP_SYN
},
&expr.Verdict{
Kind: expr.VerdictAccept,
},
},
})
return c.Flush()
}

@ -130,13 +130,14 @@ func main() {
hs.ConnState = func(c net.Conn, s http.ConnState) {
stMu.Lock()
defer stMu.Unlock()
oldLen := len(newSet)
switch s {
case http.StateNew:
newSet.Add(c)
default:
newSet.Delete(c)
}
if len(newSet) == 0 {
if oldLen != 0 && len(newSet) == 0 {
select {
case needConnCh <- true:
default:
@ -147,7 +148,12 @@ func main() {
lcRP := httputil.NewSingleHostReverseProxy(must.Get(url.Parse("http://local-tailscaled.sock")))
lcRP.Transport = new(localClientRoundTripper)
ttaMux.Handle("/localapi/", lcRP)
ttaMux.HandleFunc("/localapi/", func(w http.ResponseWriter, r *http.Request) {
log.Printf("Got localapi request: %v", r.URL)
t0 := time.Now()
lcRP.ServeHTTP(w, r)
log.Printf("Did localapi request in %v: %v", time.Since(t0).Round(time.Millisecond), r.URL)
})
ttaMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, "TTA\n")
@ -156,8 +162,19 @@ func main() {
ttaMux.HandleFunc("/up", func(w http.ResponseWriter, r *http.Request) {
serveCmd(w, "tailscale", "up", "--login-server=http://control.tailscale")
})
ttaMux.HandleFunc("/fw", addFirewallHandler)
go hs.Serve(chanListener(conns))
// For doing agent operations locally from gokrazy:
// (e.g. with "wget -O - localhost:8123/fw")
go func() {
err := http.ListenAndServe("127.0.0.1:8123", &ttaMux)
if err != nil {
log.Fatalf("ListenAndServe: %v", err)
}
}()
var lastErr string
needConnCh <- true
for {
@ -204,3 +221,18 @@ func (cl chanListener) Addr() net.Addr {
Port: 123,
}
}
func addFirewallHandler(w http.ResponseWriter, r *http.Request) {
if addFirewall == nil {
http.Error(w, "firewall not supported", 500)
return
}
err := addFirewall()
if err != nil {
http.Error(w, err.Error(), 500)
return
}
io.WriteString(w, "OK\n")
}
var addFirewall func() error // set by fw_linux.go

@ -7,7 +7,7 @@ qemu-system-x86_64 -M microvm,isa-serial=off \
-m 1G \
-nodefaults -no-user-config -nographic \
-kernel $HOME/src/github.com/tailscale/gokrazy-kernel/vmlinuz \
-append "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet tailscale-tta=1" \
-append "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet tailscale-tta=1 tailscaled.env=TS_DEBUG_RAW_DISCO=1" \
-drive id=blk0,file=$HOME/src/tailscale.com/gokrazy/tsapp.img,format=raw \
-device virtio-blk-device,drive=blk0 \
-netdev stream,id=net0,addr.type=unix,addr.path=/tmp/qemu.sock \

@ -464,6 +464,11 @@ func New(collection string, netMon *netmon.Monitor, health *health.Tracker, logf
// The netMon parameter is optional. It should be specified in environments where
// Tailscaled is manipulating the routing table.
func NewWithConfigPath(collection, dir, cmdName string, netMon *netmon.Monitor, health *health.Tracker, logf logger.Logf) *Policy {
if hostinfo.IsNATLabGuestVM() {
// In NATLab Gokrazy instances, tailscaled comes up concurently with
// DHCP and the doesn't have DNS for a while. Wait for DHCP first.
awaitGokrazyNetwork()
}
var lflags int
if term.IsTerminal(2) || runtime.GOOS == "windows" {
lflags = 0
@ -567,7 +572,7 @@ func NewWithConfigPath(collection, dir, cmdName string, netMon *netmon.Monitor,
conf.IncludeProcSequence = true
}
if envknob.NoLogsNoSupport() || testenv.InTest() || hostinfo.IsNATLabGuestVM() {
if envknob.NoLogsNoSupport() || testenv.InTest() {
logf("You have disabled logging. Tailscale will not be able to provide support.")
conf.HTTPC = &http.Client{Transport: noopPretendSuccessTransport{}}
} else if val := getLogTarget(); val != "" {
@ -817,3 +822,25 @@ func (noopPretendSuccessTransport) RoundTrip(req *http.Request) (*http.Response,
Status: "200 OK",
}, nil
}
func awaitGokrazyNetwork() {
if runtime.GOOS != "linux" || distro.Get() != distro.Gokrazy {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
for {
// Before DHCP finishes, the /etc/resolv.conf file has just "#MANUAL".
all, _ := os.ReadFile("/etc/resolv.conf")
if bytes.Contains(all, []byte("nameserver ")) {
return
}
select {
case <-ctx.Done():
return
case <-time.After(500 * time.Millisecond):
}
}
}

@ -26,6 +26,7 @@ import (
"tailscale.com/envknob"
"tailscale.com/health"
"tailscale.com/hostinfo"
)
var counterFallbackOK int32 // atomic
@ -77,6 +78,12 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config {
// (with the baked-in fallback root) in the VerifyConnection hook.
conf.InsecureSkipVerify = true
conf.VerifyConnection = func(cs tls.ConnectionState) (retErr error) {
if host == "log.tailscale.io" && hostinfo.IsNATLabGuestVM() {
// Allow log.tailscale.io TLS MITM for integration tests when
// the client's running within a NATLab VM.
return nil
}
// Perform some health checks on this certificate before we do
// any verification.
var selfSignedIssuer string

@ -102,6 +102,14 @@ func easy(c *vnet.Config) *vnet.Node {
fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT))
}
// easy + host firewall
func easyFW(c *vnet.Config) *vnet.Node {
n := c.NumNodes() + 1
return c.AddNode(vnet.HostFirewall, c.AddNetwork(
fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT))
}
func easyAF(c *vnet.Config) *vnet.Node {
n := c.NumNodes() + 1
return c.AddNode(c.AddNetwork(
@ -134,6 +142,29 @@ func easyPMP(c *vnet.Config) *vnet.Node {
fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT, vnet.NATPMP))
}
// easy + port mapping + host firewall
func easyPMPFW(c *vnet.Config) *vnet.Node {
n := c.NumNodes() + 1
return c.AddNode(vnet.HostFirewall,
c.AddNetwork(
fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT, vnet.NATPMP))
}
// easy + port mapping + host firewall - BPF
func easyPMPFWNoBPF(c *vnet.Config) *vnet.Node {
n := c.NumNodes() + 1
return c.AddNode(
vnet.HostFirewall,
vnet.TailscaledEnv{
Key: "TS_DEBUG_DISABLE_RAW_DISCO",
Value: "1",
},
c.AddNetwork(
fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT, vnet.NATPMP))
}
func hard(c *vnet.Config) *vnet.Node {
n := c.NumNodes() + 1
return c.AddNode(c.AddNetwork(
@ -203,12 +234,18 @@ func (nt *natTest) runTest(node1, node2 addNodeFunc) pingRoute {
t.Fatalf("qemu-img create: %v, %s", err, out)
}
var envBuf bytes.Buffer
for _, e := range node.Env() {
fmt.Fprintf(&envBuf, " tailscaled.env=%s=%s", e.Key, e.Value)
}
envStr := envBuf.String()
cmd := exec.Command("qemu-system-x86_64",
"-M", "microvm,isa-serial=off",
"-m", "384M",
"-nodefaults", "-no-user-config", "-nographic",
"-kernel", nt.kernel,
"-append", "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet tailscale-tta=1",
"-append", "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet tailscale-tta=1"+envStr,
"-drive", "id=blk0,file="+disk+",format=qcow2",
"-device", "virtio-blk-device,drive=blk0",
"-netdev", "stream,id=net0,addr.type=unix,addr.path="+sockAddr,
@ -254,10 +291,20 @@ func (nt *natTest) runTest(node1, node2 addNodeFunc) pingRoute {
return fmt.Errorf("node%d status: %w", i, err)
}
t.Logf("node%d status: %v", i, st)
node := nodes[i]
if node.HostFirewall() {
if err := c.EnableHostFirewall(ctx); err != nil {
return fmt.Errorf("node%d firewall: %w", i, err)
}
t.Logf("node%d firewalled", i)
}
if err := up(ctx, c); err != nil {
return fmt.Errorf("node%d up: %w", i, err)
}
t.Logf("node%d up!", i)
st, err = c.Status(ctx)
if err != nil {
return fmt.Errorf("node%d status: %w", i, err)
@ -408,6 +455,31 @@ func TestEasyEasy(t *testing.T) {
nt.runTest(easy, easy)
}
// Tests https://github.com/tailscale/tailscale/issues/3824 ...
// * server behind a Hard NAT
// * client behind a NAT with UPnP support
// * client machine has a stateful host firewall (e.g. ufw)
func TestBPFDisco(t *testing.T) {
nt := newNatTest(t)
nt.runTest(easyPMPFW, hard)
}
func TestHostFWNoBPF(t *testing.T) {
nt := newNatTest(t)
nt.runTest(easyPMPFWNoBPF, hard)
}
func TestHostFWPair(t *testing.T) {
nt := newNatTest(t)
nt.runTest(easyFW, easyFW)
}
func TestOneHostFW(t *testing.T) {
nt := newNatTest(t)
nt.runTest(easy, easyFW)
}
var pair = flag.String("pair", "", "comma-separated pair of types to test (easy, easyAF, hard, easyPMP, hardPMP, one2one, sameLAN)")
func TestPair(t *testing.T) {

@ -70,6 +70,16 @@ func (c *Config) AddNode(opts ...any) *Node {
o.nodes = append(o.nodes, n)
}
n.nets = append(n.nets, o)
case TailscaledEnv:
n.env = append(n.env, o)
case NodeOption:
if o == HostFirewall {
n.hostFW = true
} else {
if n.err == nil {
n.err = fmt.Errorf("unknown NodeOption %q", o)
}
}
default:
if n.err == nil {
n.err = fmt.Errorf("unknown AddNode option type %T", o)
@ -79,6 +89,19 @@ func (c *Config) AddNode(opts ...any) *Node {
return n
}
// NodeOption is an option that can be passed to Config.AddNode.
type NodeOption string
const (
HostFirewall NodeOption = "HostFirewall"
)
// TailscaledEnv is а option that can be passed to Config.AddNode
// to set an environment variable for tailscaled.
type TailscaledEnv struct {
Key, Value string
}
// AddNetwork add a new network.
//
// The opts may be of the following types:
@ -125,6 +148,9 @@ type Node struct {
err error
n *node // nil until NewServer called
env []TailscaledEnv
hostFW bool
// TODO(bradfitz): this is halfway converted to supporting multiple NICs
// but not done. We need a MAC-per-Network.
@ -137,6 +163,14 @@ func (n *Node) MAC() MAC {
return n.mac
}
func (n *Node) Env() []TailscaledEnv {
return n.env
}
func (n *Node) HostFirewall() bool {
return n.hostFW
}
// Network returns the first network this node is connected to,
// or nil if none.
func (n *Node) Network() *Network {

@ -15,6 +15,7 @@ package vnet
import (
"bufio"
"bytes"
"context"
"crypto/tls"
"encoding/binary"
@ -61,6 +62,7 @@ import (
"tailscale.com/util/mak"
"tailscale.com/util/must"
"tailscale.com/util/set"
"tailscale.com/util/zstdframe"
)
const nicID = 1
@ -325,6 +327,13 @@ func (n *network) acceptTCP(r *tcp.ForwarderRequest) {
go hs.Serve(netutil.NewOneConnListener(tc, nil))
return
}
if destPort == 443 && destIP == fakeLogCatcherIP {
r.Complete(false)
tc := gonet.NewTCPConn(&wq, ep)
go n.serveLogCatcherConn(clientRemoteIP, tc)
return
}
log.Printf("vnet-AcceptTCP: %v", stringifyTEI(reqDetails))
@ -354,6 +363,51 @@ func (n *network) acceptTCP(r *tcp.ForwarderRequest) {
}
}
// serveLogCatchConn serves a TCP connection to "log.tailscale.io", speaking the
// logtail/logcatcher protocol.
//
// We terminate TLS with an arbitrary cert; the client is configured to not
// validate TLS certs for this hostname when running under these integration
// tests.
func (n *network) serveLogCatcherConn(clientRemoteIP netip.Addr, c net.Conn) {
tlsConfig := n.s.derps[0].tlsConfig // self-signed (stealing DERP's); test client configure to not check
tlsConn := tls.Server(c, tlsConfig)
var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
all, _ := io.ReadAll(r.Body)
if r.Header.Get("Content-Encoding") == "zstd" {
var err error
all, err = zstdframe.AppendDecode(nil, all)
if err != nil {
log.Printf("LOGS DECODE ERROR zstd decode: %v", err)
http.Error(w, "zstd decode error", http.StatusBadRequest)
return
}
}
var logs []struct {
Logtail struct {
Client_Time time.Time
}
Text string
}
if err := json.Unmarshal(all, &logs); err != nil {
log.Printf("Logs decode error: %v", err)
return
}
node := n.nodesByIP[clientRemoteIP]
if node != nil {
node.logMu.Lock()
defer node.logMu.Unlock()
for _, lg := range logs {
tStr := lg.Logtail.Client_Time.Round(time.Millisecond).Format(time.RFC3339Nano)
fmt.Fprintf(&node.logBuf, "[%v] %s\n", tStr, lg.Text)
log.Printf("LOG %v [%v] %s\n", clientRemoteIP, tStr, lg.Text)
}
}
})
hs := &http.Server{Handler: handler}
hs.Serve(netutil.NewOneConnListener(tlsConn, nil))
}
var (
fakeDNSIP = netip.AddrFrom4([4]byte{4, 11, 4, 11})
fakeProxyControlplaneIP = netip.AddrFrom4([4]byte{52, 52, 0, 1}) // real controlplane.tailscale.com proxy
@ -451,6 +505,12 @@ type node struct {
interfaceID int
net *network
lanIP netip.Addr // must be in net.lanIP prefix + unique in net
// logMu guards logBuf.
// TODO(bradfitz): conditionally write these out to separate files at the end?
// Currently they only hold logcatcher logs.
logMu sync.Mutex
logBuf bytes.Buffer
}
type derpServer struct {
@ -1153,7 +1213,7 @@ func (s *Server) shouldInterceptTCP(pkt gopacket.Packet) bool {
dstIP, _ := netip.AddrFromSlice(ipv4.DstIP.To4())
if tcp.DstPort == 80 || tcp.DstPort == 443 {
switch dstIP {
case fakeControlIP, fakeDERP1IP, fakeDERP2IP:
case fakeControlIP, fakeDERP1IP, fakeDERP2IP, fakeLogCatcherIP:
return true
}
if dstIP == fakeProxyControlplaneIP {
@ -1613,7 +1673,9 @@ func (s *Server) NodeAgentClient(n *Node) *NodeAgentClient {
d := s.NodeAgentDialer(n)
return &NodeAgentClient{
LocalClient: &tailscale.LocalClient{
Dial: d,
UseSocketOnly: true,
OmitAuth: true,
Dial: d,
},
HTTPClient: &http.Client{
Transport: &http.Transport{
@ -1622,3 +1684,21 @@ func (s *Server) NodeAgentClient(n *Node) *NodeAgentClient {
},
}
}
// EnableHostFirewall enables the host's stateful firewall.
func (c *NodeAgentClient) EnableHostFirewall(ctx context.Context) error {
req, err := http.NewRequestWithContext(ctx, "GET", "http://unused/fw", nil)
if err != nil {
return err
}
res, err := c.HTTPClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
all, _ := io.ReadAll(res.Body)
if res.StatusCode != 200 {
return fmt.Errorf("unexpected status code %v: %s", res.Status, all)
}
return nil
}

@ -32,6 +32,9 @@ const (
// Enable/disable using raw sockets to receive disco traffic.
var debugDisableRawDisco = envknob.RegisterBool("TS_DEBUG_DISABLE_RAW_DISCO")
// debugRawDiscoReads enables logging of raw disco reads.
var debugRawDiscoReads = envknob.RegisterBool("TS_DEBUG_RAW_DISCO")
// These are our BPF filters that we use for testing packets.
var (
magicsockFilterV4 = []bpf.Instruction{
@ -211,6 +214,9 @@ func (c *Conn) receiveDisco(pc net.PacketConn, isIPV6 bool) {
var buf [1500]byte
for {
n, src, err := pc.ReadFrom(buf[:])
if debugRawDiscoReads() {
c.logf("raw disco read from %v = (%v, %v)", src, n, err)
}
if errors.Is(err, net.ErrClosed) {
return
} else if err != nil {

Loading…
Cancel
Save