diff --git a/tstest/integration/vms/harness_test.go b/tstest/integration/vms/harness_test.go index d7ed97cf8..ea5a7bba0 100644 --- a/tstest/integration/vms/harness_test.go +++ b/tstest/integration/vms/harness_test.go @@ -10,14 +10,19 @@ import ( "bytes" "context" "fmt" + "log" "net" + "net/http" "os" "os/exec" + "path" "path/filepath" + "strconv" + "sync" "testing" "time" - "github.com/gliderlabs/ssh" + "golang.org/x/crypto/ssh" "golang.org/x/net/proxy" "inet.af/netaddr" "tailscale.com/tstest/integration" @@ -28,10 +33,102 @@ type Harness struct { testerDialer proxy.Dialer testerDir string bins *integration.Binaries + pubKey string signer ssh.Signer cs *testcontrol.Server loginServerURL string testerV4 netaddr.IP + ipMu *sync.Mutex + ipMap map[string]ipMapping +} + +func newHarness(t *testing.T) *Harness { + dir := t.TempDir() + bindHost := deriveBindhost(t) + ln, err := net.Listen("tcp", net.JoinHostPort(bindHost, "0")) + if err != nil { + t.Fatalf("can't make TCP listener: %v", err) + } + t.Cleanup(func() { + ln.Close() + }) + t.Logf("host:port: %s", ln.Addr()) + + cs := &testcontrol.Server{} + + derpMap := integration.RunDERPAndSTUN(t, t.Logf, bindHost) + cs.DERPMap = derpMap + + var ( + ipMu sync.Mutex + ipMap = map[string]ipMapping{} + ) + + mux := http.NewServeMux() + mux.Handle("/", cs) + + lc := &integration.LogCatcher{} + if *verboseLogcatcher { + lc.UseLogf(t.Logf) + } + mux.Handle("/c/", lc) + + // This handler will let the virtual machines tell the host information about that VM. + // This is used to maintain a list of port->IP address mappings that are known to be + // working. This allows later steps to connect over SSH. This returns no response to + // clients because no response is needed. + mux.HandleFunc("/myip/", func(w http.ResponseWriter, r *http.Request) { + ipMu.Lock() + defer ipMu.Unlock() + + name := path.Base(r.URL.Path) + host, _, _ := net.SplitHostPort(r.RemoteAddr) + port, err := strconv.Atoi(name) + if err != nil { + log.Panicf("bad port: %v", port) + } + distro := r.UserAgent() + ipMap[distro] = ipMapping{distro, port, host} + t.Logf("%s: %v", name, host) + }) + + hs := &http.Server{Handler: mux} + go hs.Serve(ln) + + run(t, dir, "ssh-keygen", "-t", "ed25519", "-f", "machinekey", "-N", ``) + pubkey, err := os.ReadFile(filepath.Join(dir, "machinekey.pub")) + if err != nil { + t.Fatalf("can't read ssh key: %v", err) + } + + privateKey, err := os.ReadFile(filepath.Join(dir, "machinekey")) + if err != nil { + t.Fatalf("can't read ssh private key: %v", err) + } + + signer, err := ssh.ParsePrivateKey(privateKey) + if err != nil { + t.Fatalf("can't parse private key: %v", err) + } + + loginServer := fmt.Sprintf("http://%s", ln.Addr()) + t.Logf("loginServer: %s", loginServer) + + bins := integration.BuildTestBinaries(t) + + h := &Harness{ + pubKey: string(pubkey), + bins: bins, + signer: signer, + loginServerURL: loginServer, + cs: cs, + ipMu: &ipMu, + ipMap: ipMap, + } + + h.makeTestNode(t, bins, loginServer) + + return h } func (h *Harness) Tailscale(t *testing.T, args ...string) []byte { diff --git a/tstest/integration/vms/nixos_test.go b/tstest/integration/vms/nixos_test.go index e70989bf9..cc13498f3 100644 --- a/tstest/integration/vms/nixos_test.go +++ b/tstest/integration/vms/nixos_test.go @@ -166,7 +166,7 @@ func copyUnit(t *testing.T, bins *integration.Binaries) { } } -func (h Harness) makeNixOSImage(t *testing.T, d Distro, cdir string) string { +func (h *Harness) makeNixOSImage(t *testing.T, d Distro, cdir string) string { copyUnit(t, h.bins) dir := t.TempDir() fname := filepath.Join(dir, d.name+".nix") diff --git a/tstest/integration/vms/vm_setup_test.go b/tstest/integration/vms/vm_setup_test.go index 5cb9e0a32..121bfad88 100644 --- a/tstest/integration/vms/vm_setup_test.go +++ b/tstest/integration/vms/vm_setup_test.go @@ -35,7 +35,7 @@ import ( // mkVM makes a KVM-accelerated virtual machine and prepares it for introduction // to the testcontrol server. The function it returns is for killing the virtual // machine when it is time for it to die. -func (h Harness) mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) { +func (h *Harness) mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) { t.Helper() cdir, err := os.UserCacheDir() @@ -160,7 +160,7 @@ func fetchFromS3(t *testing.T, fout *os.File, d Distro) bool { // fetchDistro fetches a distribution from the internet if it doesn't already exist locally. It // also validates the sha256 sum from a known good hash. -func (h Harness) fetchDistro(t *testing.T, resultDistro Distro) string { +func (h *Harness) fetchDistro(t *testing.T, resultDistro Distro) string { t.Helper() cdir, err := os.UserCacheDir() @@ -253,7 +253,7 @@ func checkCachedImageHash(t *testing.T, d Distro, cacheDir string) (gotHash stri return } -func (h Harness) copyBinaries(t *testing.T, d Distro, conn *ssh.Client) { +func (h *Harness) copyBinaries(t *testing.T, d Distro, conn *ssh.Client) { bins := h.bins if strings.HasPrefix(d.name, "nixos") { return diff --git a/tstest/integration/vms/vms_steps_test.go b/tstest/integration/vms/vms_steps_test.go index 45ba95e12..23c755aeb 100644 --- a/tstest/integration/vms/vms_steps_test.go +++ b/tstest/integration/vms/vms_steps_test.go @@ -39,7 +39,7 @@ func retry(t *testing.T, fn func() error) { t.Fatalf("tried %d times, got: %v", tries, err) } -func (h Harness) testPing(t *testing.T, ipAddr netaddr.IP, cli *ssh.Client) { +func (h *Harness) testPing(t *testing.T, ipAddr netaddr.IP, cli *ssh.Client) { var outp []byte var err error retry(t, func() error { @@ -83,7 +83,7 @@ func getSession(t *testing.T, cli *ssh.Client) *ssh.Session { return sess } -func (h Harness) testOutgoingTCP(t *testing.T, ipAddr netaddr.IP, cli *ssh.Client) { +func (h *Harness) testOutgoingTCP(t *testing.T, ipAddr netaddr.IP, cli *ssh.Client) { const sendmsg = "this is a message that curl won't print" ctx, cancel := context.WithCancel(context.Background()) s := &http.Server{ diff --git a/tstest/integration/vms/vms_test.go b/tstest/integration/vms/vms_test.go index 3fb83bd55..e1d6c8904 100644 --- a/tstest/integration/vms/vms_test.go +++ b/tstest/integration/vms/vms_test.go @@ -11,17 +11,13 @@ import ( "context" "flag" "fmt" - "log" "net" - "net/http" "os" "os/exec" - "path" "path/filepath" "regexp" "strconv" "strings" - "sync" "testing" "text/template" "time" @@ -33,7 +29,6 @@ import ( "inet.af/netaddr" "tailscale.com/tstest" "tailscale.com/tstest/integration" - "tailscale.com/tstest/integration/testcontrol" "tailscale.com/types/logger" ) @@ -75,7 +70,7 @@ func TestDownloadImages(t *testing.T) { t.Parallel() - (Harness{bins: bins}).fetchDistro(t, distro) + (&Harness{bins: bins}).fetchDistro(t, distro) }) } } @@ -245,89 +240,8 @@ func TestVMIntegrationEndToEnd(t *testing.T) { t.Fatalf("missing dependency: %v", err) } - dir := t.TempDir() - - rex := distroRex.Unwrap() - - bindHost := deriveBindhost(t) - ln, err := net.Listen("tcp", net.JoinHostPort(bindHost, "0")) - if err != nil { - t.Fatalf("can't make TCP listener: %v", err) - } - defer ln.Close() - t.Logf("host:port: %s", ln.Addr()) - - cs := &testcontrol.Server{} - - derpMap := integration.RunDERPAndSTUN(t, t.Logf, bindHost) - cs.DERPMap = derpMap - - var ( - ipMu sync.Mutex - ipMap = map[string]ipMapping{} - ) - - mux := http.NewServeMux() - mux.Handle("/", cs) - - lc := &integration.LogCatcher{} - if *verboseLogcatcher { - lc.UseLogf(t.Logf) - } - mux.Handle("/c/", lc) - - // This handler will let the virtual machines tell the host information about that VM. - // This is used to maintain a list of port->IP address mappings that are known to be - // working. This allows later steps to connect over SSH. This returns no response to - // clients because no response is needed. - mux.HandleFunc("/myip/", func(w http.ResponseWriter, r *http.Request) { - ipMu.Lock() - defer ipMu.Unlock() - - name := path.Base(r.URL.Path) - host, _, _ := net.SplitHostPort(r.RemoteAddr) - port, err := strconv.Atoi(name) - if err != nil { - log.Panicf("bad port: %v", port) - } - distro := r.UserAgent() - ipMap[distro] = ipMapping{distro, port, host} - t.Logf("%s: %v", name, host) - }) - - hs := &http.Server{Handler: mux} - go hs.Serve(ln) - - run(t, dir, "ssh-keygen", "-t", "ed25519", "-f", "machinekey", "-N", ``) - pubkey, err := os.ReadFile(filepath.Join(dir, "machinekey.pub")) - if err != nil { - t.Fatalf("can't read ssh key: %v", err) - } - - privateKey, err := os.ReadFile(filepath.Join(dir, "machinekey")) - if err != nil { - t.Fatalf("can't read ssh private key: %v", err) - } - - signer, err := ssh.ParsePrivateKey(privateKey) - if err != nil { - t.Fatalf("can't parse private key: %v", err) - } - - loginServer := fmt.Sprintf("http://%s", ln.Addr()) - t.Logf("loginServer: %s", loginServer) - ramsem := semaphore.NewWeighted(int64(*vmRamLimit)) - bins := integration.BuildTestBinaries(t) - - h := &Harness{ - bins: bins, - signer: signer, - loginServerURL: loginServer, - cs: cs, - } - - h.makeTestNode(t, bins, loginServer) + rex := distroRex.Unwrap() t.Run("do", func(t *testing.T) { for n, distro := range distros { @@ -344,6 +258,7 @@ func TestVMIntegrationEndToEnd(t *testing.T) { t.Parallel() + h := newHarness(t) dir := t.TempDir() err := ramsem.Acquire(ctx, int64(distro.mem)) @@ -352,7 +267,7 @@ func TestVMIntegrationEndToEnd(t *testing.T) { } defer ramsem.Release(int64(distro.mem)) - h.mkVM(t, n, distro, string(pubkey), loginServer, dir) + h.mkVM(t, n, distro, h.pubKey, h.loginServerURL, dir) var ipm ipMapping t.Run("wait-for-start", func(t *testing.T) { @@ -361,12 +276,12 @@ func TestVMIntegrationEndToEnd(t *testing.T) { var ok bool for { <-waiter.C - ipMu.Lock() - if ipm, ok = ipMap[distro.name]; ok { - ipMu.Unlock() + h.ipMu.Lock() + if ipm, ok = h.ipMap[distro.name]; ok { + h.ipMu.Unlock() break } - ipMu.Unlock() + h.ipMu.Unlock() } }) @@ -376,7 +291,7 @@ func TestVMIntegrationEndToEnd(t *testing.T) { }) } -func (h Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) { +func (h *Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) { signer := h.signer loginServer := h.loginServerURL