tstest/natlab: add unix address to writer for dgram mode

updates tailcale/corp#22371

For dgram mode, we need to store the write addresses of
the client socket(s) alongside the writer functions and
the write operation needs to use WriteToUnix.

Unix also has multiple clients writing to the same socket,
so the serve method is modified to handle packets from
multiple mac addresses.

Cleans up a bit of cruft from the initial tailmac tooling
commit.

Now all the macOS packets are belong to us.

Signed-off-by: Jonathan Nobels <jonathan@tailscale.com>

tstest/natlab/machost/README.md

@@ -1,84 +0,0 @@
-# macOS VM's for tstest and natlab
-
-## Building
-
-```
-%make all
-```
-
-Will build both the TailMac and the VMHost app.  You will need a developer account.  The default bundle identifiers
-default to tailscale owned ids, so if you don't have (or aren't using) a tailscale dev account, you will need to change this.
-This should build automatically as long as you have a valid developer cert.  Signing is automatic.  The binaries both
-require proper entitlements, so they do need to be signed.
-
-There are separate recipes in the makefile to rebuild the individual components if needed.
-
-All binaries are copied to the bin directory.
-
-You can generally do all interactions via the TailMac command line util.
-
-## Locations
-
-Everything is persisted at ~/VM.bundle
-
-Each vm gets it's own directory under there.  
-
-RestoreImage.ipsw is used to build new VMs.  You may replace this manually if you wish.
-
-Individual parameters for each instance are saved in a json config file (config.json)
-
-## Installing
-
-### Default a parameters
-
-The default virtio socket device port is 51009
-The default server socket for the virtual network device is /tmp/qemu.sock
-The default memory size is 4Gb
-The default mac address for the socket based network is 5a:94:ef:e4:0c:ee
-The defualt mac address for normal ethernet is 5a:94:ef:e4:0c:ef
-
-All of these parameters are configurable.
-
-### Creating and managing VMs
-
-To create a new VM (this will grab a restore image if needed).  Restore images are large.  Installation takes a minute
-```
-TailMac create --id my_vm_id
-```
-
-To delete a new VM 
-```
-TailMac delete --id my_vm_id
-```
-
-To refresh an existing restore image:
-```
-TailMac refresh
-```
-
-To clone an existing vm (this will clone the mac and port as well)
-```
-TailMac clone --id old_vm_id --target-id new_vm_id
-```
-
-To reconfigure a vm with a specific mac and a virtio socket device port:
-```
-TailMac configure --id vm_id --mac 11:22:33:44:55:66 --port 12345 --ethermac 22:33:44:55:66:77 --mem 4000000000 --sock "/var/netdevice.sock"
-```
-
-## Running a VM
-
-MacHost is an app bundle, but the main binary behaves as a command line util.  You can invoke it
-thusly:
-
-```
-TailMac --id machine_1
- ```
-
- You may invoke multiple vms, but the limit on the number of concurrent instances is on the order of 2.
-
- To stop a running VM (this is a fire and forget thing):
-
- ```
- TailMac stop --id machine_1
- ```

tstest/natlab/vnet/vnet.go

@@ -222,8 +222,8 @@ func (n *network) initStack() error {
 				log.Printf("Serialize error: %v", err)
 				continue
 			}
-			if writeFunc, ok := n.writeFunc.Load(node.mac); ok {
+			if nw, ok := n.writers.Load(node.mac); ok {
-				writeFunc(buffer.Bytes())
+				nw.write(buffer.Bytes())
 			} else {
 				log.Printf("No writeFunc for %v", node.mac)
 			}
@@ -465,6 +465,20 @@ type portMapping struct {
 	expiry time.Time
 }
 
+type writerFunc func([]byte, *net.UnixAddr, int)
+
+// Encapsulates both a write function, an optional outbound socket address
+// for dgram mode and an interfaceID for packet captures.
+type networkWriter struct {
+	writer      writerFunc    // Function to write packets to the network
+	addr        *net.UnixAddr // Outbound socket address for dgram mode
+	interfaceID int           // The interface ID of the src node (for writing pcaps)
+}
+
+func (nw *networkWriter) write(b []byte) {
+	nw.writer(b, nw.addr, nw.interfaceID)
+}
+
 type network struct {
 	s              *Server
 	mac            MAC
@@ -485,16 +499,23 @@ type network struct {
 	portMap     map[netip.AddrPort]portMapping    // WAN ip:port -> LAN ip:port
 	portMapFlow map[portmapFlowKey]netip.AddrPort // (lanAP, peerWANAP) -> portmapped wanAP
 
-	// writeFunc is a map of MAC -> func to write to that MAC.
+	// writers is a map of MAC -> networkWriters to write packets to that MAC.
 	// It contains entries for connected nodes only.
-	writeFunc syncs.Map[MAC, func([]byte)] // MAC -> func to write to that MAC
+	writers syncs.Map[MAC, networkWriter] // MAC -> to networkWriter for that MAC
-}
+}
-
+
-func (n *network) registerWriter(mac MAC, f func([]byte)) {
+// Regsiters a writerFunc for a MAC address.
-	if f != nil {
+// raddr is and optional outbound socket address of the client interface for dgram mode.
-		n.writeFunc.Store(mac, f)
+// Pass nil for the writerFunc to deregister the writer.
+func (n *network) registerWriter(mac MAC, raddr *net.UnixAddr, interfaceID int, wf writerFunc) {
+	if wf != nil {
+		n.writers.Store(mac, networkWriter{
+			writer:      wf,
+			addr:        raddr,
+			interfaceID: interfaceID,
+		})
 	} else {
-		n.writeFunc.Delete(mac)
+		n.writers.Delete(mac)
 	}
 }
 
@@ -684,10 +705,10 @@ type Protocol int
 
 const (
 	ProtocolQEMU      = Protocol(iota + 1)
-	ProtocolUnixDGRAM // for macOS Hypervisor.Framework and VZFileHandleNetworkDeviceAttachment
+	ProtocolUnixDGRAM // for macOS Virtualization.Framework and VZFileHandleNetworkDeviceAttachment
 )
 
-// serveConn serves a single connection from a client.
+// Handles a single connection from a QEMU-style client or muxd connections for dgram mode
 func (s *Server) ServeUnixConn(uc *net.UnixConn, proto Protocol) {
 	if s.shuttingDown.Load() {
 		return
@@ -702,24 +723,37 @@ func (s *Server) ServeUnixConn(uc *net.UnixConn, proto Protocol) {
 
 	bw := bufio.NewWriterSize(uc, 2<<10)
 	var writeMu sync.Mutex
-	var srcNode *node
+	writePkt := func(pkt []byte, raddr *net.UnixAddr, interfaceID int) {
-	writePkt := func(pkt []byte) {
 		if pkt == nil {
 			return
 		}
 		writeMu.Lock()
 		defer writeMu.Unlock()
-		if proto == ProtocolQEMU {
+		switch proto {
+		case ProtocolQEMU:
 			hdr := binary.BigEndian.AppendUint32(bw.AvailableBuffer()[:0], uint32(len(pkt)))
 			if _, err := bw.Write(hdr); err != nil {
 				log.Printf("Write hdr: %v", err)
 				return
 			}
+
+			if _, err := bw.Write(pkt); err != nil {
+				log.Printf("Write pkt: %v", err)
+				return
+
+			}
+		case ProtocolUnixDGRAM:
+			if raddr == nil {
+				log.Printf("Write pkt: dgram mode write failure, no outbound socket address")
+				return
+			}
+
+			if _, err := uc.WriteToUnix(pkt, raddr); err != nil {
+				log.Printf("Write pkt : %v", err)
+				return
+			}
 		}
-		if _, err := bw.Write(pkt); err != nil {
+
-			log.Printf("Write pkt: %v", err)
-			return
-		}
 		if err := bw.Flush(); err != nil {
 			log.Printf("Flush: %v", err)
 		}
@@ -727,22 +761,25 @@ func (s *Server) ServeUnixConn(uc *net.UnixConn, proto Protocol) {
 			Timestamp:      time.Now(),
 			CaptureLength:  len(pkt),
 			Length:         len(pkt),
-			InterfaceIndex: srcNode.interfaceID,
+			InterfaceIndex: interfaceID,
 		}, pkt))
 	}
 
 	buf := make([]byte, 16<<10)
-	var netw *network // non-nil after first packet
 	for {
 		var packetRaw []byte
-		if proto == ProtocolUnixDGRAM {
+		var raddr *net.UnixAddr
-			n, _, err := uc.ReadFromUnix(buf)
+
+		switch proto {
+		case ProtocolUnixDGRAM:
+			n, addr, err := uc.ReadFromUnix(buf)
+			raddr = addr
 			if err != nil {
 				log.Printf("ReadFromUnix: %v", err)
 				continue
 			}
 			packetRaw = buf[:n]
-		} else if proto == ProtocolQEMU {
+		case ProtocolQEMU:
 			if _, err := io.ReadFull(uc, buf[:4]); err != nil {
 				if s.shutdownCtx.Err() != nil {
 					// Return without logging.
@@ -772,29 +809,29 @@ func (s *Server) ServeUnixConn(uc *net.UnixConn, proto Protocol) {
 		ep := EthernetPacket{le, packet}
 
 		srcMAC := ep.SrcMAC()
-		if srcNode == nil {
+		srcNode, ok := s.nodeByMAC[srcMAC]
-			srcNode, ok = s.nodeByMAC[srcMAC]
+		if !ok {
-			if !ok {
+			log.Printf("[conn %p] got frame from unknown MAC %v", uc, srcMAC)
-				log.Printf("[conn %p] ignoring frame from unknown MAC %v", uc, srcMAC)
+			continue
-				continue
-			}
-			log.Printf("[conn %p] MAC %v is node %v", uc, srcMAC, srcNode.lanIP)
-			netw = srcNode.net
-			netw.registerWriter(srcMAC, writePkt)
-			defer netw.registerWriter(srcMAC, nil)
-		} else {
-			if srcMAC != srcNode.mac {
-				log.Printf("[conn %p] ignoring frame from MAC %v, expected %v", uc, srcMAC, srcNode.mac)
-				continue
-			}
 		}
+
+		// Register a writer for the source MAC address if one doesn't exist.
+		if _, ok := srcNode.net.writers.Load(srcMAC); !ok {
+			log.Printf("[conn %p] Registering writer for MAC %v is node %v", uc, srcMAC, srcNode.lanIP)
+			srcNode.net.registerWriter(srcMAC, raddr, srcNode.interfaceID, writePkt)
+			defer func() {
+				srcNode.net.registerWriter(srcMAC, nil, 0, nil)
+			}()
+			continue
+		}
+
 		must.Do(s.pcapWriter.WritePacket(gopacket.CaptureInfo{
 			Timestamp:      time.Now(),
 			CaptureLength:  len(packetRaw),
 			Length:         len(packetRaw),
 			InterfaceIndex: srcNode.interfaceID,
 		}, packetRaw))
-		netw.HandleEthernetPacket(ep)
+		srcNode.net.HandleEthernetPacket(ep)
 	}
 }
 
@@ -839,8 +876,8 @@ func (n *network) writeEth(res []byte) {
 	dstMAC := MAC(res[0:6])
 	srcMAC := MAC(res[6:12])
 	if dstMAC.IsBroadcast() {
-		n.writeFunc.Range(func(mac MAC, writeFunc func([]byte)) bool {
+		n.writers.Range(func(mac MAC, nw networkWriter) bool {
-			writeFunc(res)
+			nw.write(res)
 			return true
 		})
 		return
@@ -849,8 +886,8 @@ func (n *network) writeEth(res []byte) {
 		n.logf("dropping write of packet from %v to itself", srcMAC)
 		return
 	}
-	if writeFunc, ok := n.writeFunc.Load(dstMAC); ok {
+	if nw, ok := n.writers.Load(dstMAC); ok {
-		writeFunc(res)
+		nw.write(res)
 		return
 	}
 }

tstest/tailmac/TailMac.xcodeproj/xcshareddata/xcschemes/host.xcscheme

@@ -15,7 +15,7 @@
             <BuildableReference
                BuildableIdentifier = "primary"
                BlueprintIdentifier = "8F87D51C26C34111000EADA4"
-               BuildableName = "TailMac.app"
+               BuildableName = "Host.app"
                BlueprintName = "host"
                ReferencedContainer = "container:TailMac.xcodeproj">
             </BuildableReference>
@@ -45,7 +45,7 @@
          <BuildableReference
             BuildableIdentifier = "primary"
             BlueprintIdentifier = "8F87D51C26C34111000EADA4"
-            BuildableName = "TailMac.app"
+            BuildableName = "Host.app"
             BlueprintName = "host"
             ReferencedContainer = "container:TailMac.xcodeproj">
          </BuildableReference>
@@ -62,7 +62,7 @@
          <BuildableReference
             BuildableIdentifier = "primary"
             BlueprintIdentifier = "8F87D51C26C34111000EADA4"
-            BuildableName = "TailMac.app"
+            BuildableName = "Host.app"
             BlueprintName = "host"
             ReferencedContainer = "container:TailMac.xcodeproj">
          </BuildableReference>