tstest/natlab/vnet: capture network wan/lan interfaces

Updates #13038

Signed-off-by: Maisem Ali <maisem@tailscale.com>

tstest/natlab/vnet/conf.go

@@ -14,6 +14,7 @@ import (
 	"github.com/google/gopacket/layers"
 	"github.com/google/gopacket/pcapgo"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/must"
 	"tailscale.com/util/set"
 )
 
@@ -206,7 +207,7 @@ func (s *Server) initFromConfig(c *Config) error {
 		}
 		s.pcapWriter = pw
 	}
-	for _, conf := range c.networks {
+	for i, conf := range c.networks {
 		if conf.err != nil {
 			return conf.err
 		}
@@ -228,6 +229,14 @@ func (s *Server) initFromConfig(c *Config) error {
 			return fmt.Errorf("two networks have the same WAN IP %v; Anycast not (yet?) supported", conf.wanIP)
 		}
 		s.networkByWAN[conf.wanIP] = n
+		n.lanInterfaceID = must.Get(s.pcapWriter.AddInterface(pcapgo.NgInterface{
+			Name:     fmt.Sprintf("network%d-lan", i+1),
+			LinkType: layers.LinkTypeIPv4,
+		}))
+		n.wanInterfaceID = must.Get(s.pcapWriter.AddInterface(pcapgo.NgInterface{
+			Name:     fmt.Sprintf("network%d-wan", i+1),
+			LinkType: layers.LinkTypeIPv4,
+		}))
 	}
 	for i, conf := range c.nodes {
 		if conf.err != nil {
@@ -235,15 +244,12 @@ func (s *Server) initFromConfig(c *Config) error {
 		}
 		n := &node{
 			mac: conf.mac,
-			id:  i + 1,
 			net: netOfConf[conf.Network()],
 		}
-		if s.pcapWriter != nil {
+		n.interfaceID = must.Get(s.pcapWriter.AddInterface(pcapgo.NgInterface{
-			s.pcapWriter.w.AddInterface(pcapgo.NgInterface{
+			Name:     fmt.Sprintf("node%d", i+1),
-				Name:     fmt.Sprintf("node%d", n.id),
 			LinkType: layers.LinkTypeEthernet,
-			})
+		}))
-		}
 		conf.n = n
 		if _, ok := s.nodeByMAC[n.mac]; ok {
 			return fmt.Errorf("two nodes have the same MAC %v", n.mac)

tstest/natlab/vnet/pcap.go

@@ -12,6 +12,8 @@ import (
 	"github.com/google/gopacket/pcapgo"
 )
 
+// pcapWriter is a pcapgo.NgWriter that writes to a file.
+// It is safe for concurrent use. The nil value is a no-op.
 type pcapWriter struct {
 	f *os.File
 
@@ -20,6 +22,9 @@ type pcapWriter struct {
 }
 
 func (p *pcapWriter) WritePacket(ci gopacket.CaptureInfo, data []byte) error {
+	if p == nil {
+		return nil
+	}
 	p.mu.Lock()
 	defer p.mu.Unlock()
 	if p.w == nil {
@@ -28,7 +33,19 @@ func (p *pcapWriter) WritePacket(ci gopacket.CaptureInfo, data []byte) error {
 	return p.w.WritePacket(ci, data)
 }
 
+func (p *pcapWriter) AddInterface(i pcapgo.NgInterface) (int, error) {
+	if p == nil {
+		return 0, nil
+	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.w.AddInterface(i)
+}
+
 func (p *pcapWriter) Close() error {
+	if p == nil {
+		return nil
+	}
 	p.mu.Lock()
 	defer p.mu.Unlock()
 	if p.w != nil {

tstest/natlab/vnet/vnet.go

@@ -407,6 +407,8 @@ type network struct {
 	s              *Server
 	mac            MAC
 	portmap        bool
+	lanInterfaceID int
+	wanInterfaceID int
 	wanIP          netip.Addr
 	lanIP          netip.Prefix // with host bits set (e.g. 192.168.2.1/24)
 	nodesByIP      map[netip.Addr]*node
@@ -446,7 +448,7 @@ func (n *network) MACOfIP(ip netip.Addr) (_ MAC, ok bool) {
 
 type node struct {
 	mac         MAC
-	id    int
+	interfaceID int
 	net         *network
 	lanIP       netip.Addr // must be in net.lanIP prefix + unique in net
 }
@@ -570,10 +572,8 @@ func New(c *Config) (*Server, error) {
 func (s *Server) Close() {
 	if shutdown := s.shuttingDown.Swap(true); !shutdown {
 		s.shutdownCancel()
-		if s.pcapWriter != nil {
 		s.pcapWriter.Close()
 	}
-	}
 	s.wg.Wait()
 }
 
@@ -647,17 +647,12 @@ func (s *Server) ServeUnixConn(uc *net.UnixConn, proto Protocol) {
 		if err := bw.Flush(); err != nil {
 			log.Printf("Flush: %v", err)
 		}
-		if s.pcapWriter != nil {
+		must.Do(s.pcapWriter.WritePacket(gopacket.CaptureInfo{
-			ci := gopacket.CaptureInfo{
 			Timestamp:      time.Now(),
 			CaptureLength:  len(pkt),
 			Length:         len(pkt),
-			}
+			InterfaceIndex: srcNode.interfaceID,
-			if srcNode != nil {
+		}, pkt))
-				ci.InterfaceIndex = srcNode.id
-			}
-			must.Do(s.pcapWriter.WritePacket(ci, pkt))
-		}
 	}
 
 	buf := make([]byte, 16<<10)
@@ -717,17 +712,12 @@ func (s *Server) ServeUnixConn(uc *net.UnixConn, proto Protocol) {
 				continue
 			}
 		}
-		if s.pcapWriter != nil {
+		must.Do(s.pcapWriter.WritePacket(gopacket.CaptureInfo{
-			ci := gopacket.CaptureInfo{
 			Timestamp:      time.Now(),
 			CaptureLength:  len(packetRaw),
 			Length:         len(packetRaw),
-			}
+			InterfaceIndex: srcNode.interfaceID,
-			if srcNode != nil {
+		}, packetRaw))
-				ci.InterfaceIndex = srcNode.id
-			}
-			must.Do(s.pcapWriter.WritePacket(ci, packetRaw))
-		}
 		netw.HandleEthernetPacket(ep)
 	}
 }
@@ -825,12 +815,34 @@ func (n *network) HandleEthernetPacket(ep EthernetPacket) {
 // LAN IP here and wrapped in an ethernet layer and delivered
 // to the network.
 func (n *network) HandleUDPPacket(p UDPPacket) {
+	buf, err := n.serializedUDPPacket(p.Src, p.Dst, p.Payload, nil)
+	if err != nil {
+		n.logf("serializing UDP packet: %v", err)
+		return
+	}
+	n.s.pcapWriter.WritePacket(gopacket.CaptureInfo{
+		Timestamp:      time.Now(),
+		CaptureLength:  len(buf),
+		Length:         len(buf),
+		InterfaceIndex: n.wanInterfaceID,
+	}, buf)
 	dst := n.doNATIn(p.Src, p.Dst)
 	if !dst.IsValid() {
 		n.logf("Warning: NAT dropped packet; no mapping for %v=>%v", p.Src, p.Dst)
 		return
 	}
 	p.Dst = dst
+	buf, err = n.serializedUDPPacket(p.Src, p.Dst, p.Payload, nil)
+	if err != nil {
+		n.logf("serializing UDP packet: %v", err)
+		return
+	}
+	n.s.pcapWriter.WritePacket(gopacket.CaptureInfo{
+		Timestamp:      time.Now(),
+		CaptureLength:  len(buf),
+		Length:         len(buf),
+		InterfaceIndex: n.lanInterfaceID,
+	}, buf)
 	n.WriteUDPPacketNoNAT(p)
 }
 
@@ -853,6 +865,20 @@ func (n *network) WriteUDPPacketNoNAT(p UDPPacket) {
 		DstMAC:       node.mac.HWAddr(),
 		EthernetType: layers.EthernetTypeIPv4,
 	}
+	ethRaw, err := n.serializedUDPPacket(src, dst, p.Payload, eth)
+	if err != nil {
+		n.logf("serializing UDP packet: %v", err)
+		return
+	}
+	n.writeEth(ethRaw)
+}
+
+// serializedUDPPacket serializes a UDP packet with the given source and
+// destination IP:port pairs, and payload.
+//
+// If eth is non-nil, it will be used as the Ethernet layer, otherwise the
+// Ethernet layer will be omitted from the serialization.
+func (n *network) serializedUDPPacket(src, dst netip.AddrPort, payload []byte, eth *layers.Ethernet) ([]byte, error) {
 	ip := &layers.IPv4{
 		Version:  4,
 		TTL:      64,
@@ -868,12 +894,14 @@ func (n *network) WriteUDPPacketNoNAT(p UDPPacket) {
 
 	buffer := gopacket.NewSerializeBuffer()
 	options := gopacket.SerializeOptions{FixLengths: true, ComputeChecksums: true}
-	if err := gopacket.SerializeLayers(buffer, options, eth, ip, udp, gopacket.Payload(p.Payload)); err != nil {
+	layers := []gopacket.SerializableLayer{eth, ip, udp, gopacket.Payload(payload)}
-		n.logf("serializing UDP: %v", err)
+	if eth == nil {
-		return
+		layers = layers[1:]
 	}
-	ethRaw := buffer.Bytes()
+	if err := gopacket.SerializeLayers(buffer, options, layers...); err != nil {
-	n.writeEth(ethRaw)
+		return nil, fmt.Errorf("serializing UDP: %v", err)
+	}
+	return buffer.Bytes(), nil
 }
 
 // HandleEthernetIPv4PacketForRouter handles an IPv4 packet that is
@@ -931,10 +959,30 @@ func (n *network) HandleEthernetIPv4PacketForRouter(ep EthernetPacket) {
 	if toForward && isUDP {
 		src := netip.AddrPortFrom(srcIP, uint16(udp.SrcPort))
 		dst := netip.AddrPortFrom(dstIP, uint16(udp.DstPort))
-		src0 := src
+		buf, err := n.serializedUDPPacket(src, dst, udp.Payload, nil)
+		if err != nil {
+			n.logf("serializing UDP packet: %v", err)
+			return
+		}
+		n.s.pcapWriter.WritePacket(gopacket.CaptureInfo{
+			Timestamp:      time.Now(),
+			CaptureLength:  len(buf),
+			Length:         len(buf),
+			InterfaceIndex: n.lanInterfaceID,
+		}, buf)
+
 		src = n.doNATOut(src, dst)
-		_ = src0
+		buf, err = n.serializedUDPPacket(src, dst, udp.Payload, nil)
-		//log.Printf("XXX UDP out %v=>%v to %v", src0, src, dst)
+		if err != nil {
+			n.logf("serializing UDP packet: %v", err)
+			return
+		}
+		n.s.pcapWriter.WritePacket(gopacket.CaptureInfo{
+			Timestamp:      time.Now(),
+			CaptureLength:  len(buf),
+			Length:         len(buf),
+			InterfaceIndex: n.wanInterfaceID,
+		}, buf)
 
 		n.s.routeUDPPacket(UDPPacket{
 			Src:     src,