cmd/containerboot: refactor tests to have more explicit phases.

In preparation for making startup more complex with IPN bus watches.

Signed-off-by: David Anderson <danderson@tailscale.com>

cmd/containerboot/main_test.go

@@ -33,6 +33,7 @@ import (
 	"golang.org/x/sys/unix"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
+	"tailscale.com/tstest"
 )
 
 func TestContainerBoot(t *testing.T) {
@@ -97,74 +98,103 @@ func TestContainerBoot(t *testing.T) {
 	// step. Right now all of containerboot's modes either converge
 	// with no further interaction needed, or with one extra step
 	// only.
+	type phase struct {
+		// Make LocalAPI report this status, then wait for the Wants below to be
+		// satisfied. A zero Status is a valid state for a just-started
+		// tailscaled.
+		Status ipnstate.Status
+
+		// WantCmds is the commands that containerboot should run in this phase.
+		WantCmds []string
+		// WantKubeSecret is the secret keys/values that should exist in the
+		// kube secret.
+		WantKubeSecret map[string]string
+		// WantFiles files that should exist in the container and their
+		// contents.
+		WantFiles map[string]string
+	}
 	tests := []struct {
 		Name       string
 		Env        map[string]string
 		KubeSecret map[string]string
-		WantArgs1      []string        // Wait for containerboot to run these commands...
+		Phases     []phase
-		Status1        ipnstate.Status // ... then report this status in LocalAPI.
-		WantArgs2      []string        // If non-nil, wait for containerboot to run these additional commands...
-		Status2        ipnstate.Status // ... then report this status in LocalAPI.
-		WantKubeSecret map[string]string
-		WantFiles      map[string]string
 	}{
 		{
 			// Out of the box default: runs in userspace mode, ephemeral storage, interactive login.
 			Name: "no_args",
 			Env:  nil,
-			WantArgs1: []string{
+			Phases: []phase{
+				{
+					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
 					},
-			// The tailscale up call blocks until auth is complete, so
+				},
-			// by the time it returns the next converged state is
+				{
-			// Running.
+					Status: ipnstate.Status{
-			Status1: ipnstate.Status{
 						BackendState: "Running",
 						TailscaleIPs: tsIPs,
 					},
 				},
+			},
+		},
 		{
 			// Userspace mode, ephemeral storage, authkey provided on every run.
 			Name: "authkey",
 			Env: map[string]string{
 				"TS_AUTH_KEY": "tskey-key",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
+				{
+					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
 					},
-			Status1: ipnstate.Status{
+				},
+				{
+					Status: ipnstate.Status{
 						BackendState: "Running",
 						TailscaleIPs: tsIPs,
 					},
 				},
+			},
+		},
 		{
 			Name: "authkey_disk_state",
 			Env: map[string]string{
 				"TS_AUTH_KEY":  "tskey-key",
 				"TS_STATE_DIR": filepath.Join(d, "tmp"),
 			},
-			WantArgs1: []string{
+			Phases: []phase{
+				{
+					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --statedir=/tmp --tun=userspace-networking",
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
 					},
-			Status1: ipnstate.Status{
+				},
+				{
+					Status: ipnstate.Status{
 						BackendState: "Running",
 						TailscaleIPs: tsIPs,
 					},
 				},
+			},
+		},
 		{
 			Name: "routes",
 			Env: map[string]string{
 				"TS_AUTH_KEY": "tskey-key",
 				"TS_ROUTES":   "1.2.3.0/24,10.20.30.0/24",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
+				{
+					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=1.2.3.0/24,10.20.30.0/24",
 					},
-			Status1: ipnstate.Status{
+				},
+				{
+					Status: ipnstate.Status{
 						BackendState: "Running",
 						TailscaleIPs: tsIPs,
 					},
@@ -173,6 +203,8 @@ func TestContainerBoot(t *testing.T) {
 						"proc/sys/net/ipv6/conf/all/forwarding": "0",
 					},
 				},
+			},
+		},
 		{
 			Name: "routes_kernel_ipv4",
 			Env: map[string]string{
@@ -180,11 +212,15 @@ func TestContainerBoot(t *testing.T) {
 				"TS_ROUTES":    "1.2.3.0/24,10.20.30.0/24",
 				"TS_USERSPACE": "false",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
+				{
+					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=1.2.3.0/24,10.20.30.0/24",
 					},
-			Status1: ipnstate.Status{
+				},
+				{
+					Status: ipnstate.Status{
 						BackendState: "Running",
 						TailscaleIPs: tsIPs,
 					},
@@ -193,6 +229,8 @@ func TestContainerBoot(t *testing.T) {
 						"proc/sys/net/ipv6/conf/all/forwarding": "0",
 					},
 				},
+			},
+		},
 		{
 			Name: "routes_kernel_ipv6",
 			Env: map[string]string{
@@ -200,11 +238,15 @@ func TestContainerBoot(t *testing.T) {
 				"TS_ROUTES":    "::/64,1::/64",
 				"TS_USERSPACE": "false",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
+				{
+					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=::/64,1::/64",
 					},
-			Status1: ipnstate.Status{
+				},
+				{
+					Status: ipnstate.Status{
 						BackendState: "Running",
 						TailscaleIPs: tsIPs,
 					},
@@ -213,6 +255,8 @@ func TestContainerBoot(t *testing.T) {
 						"proc/sys/net/ipv6/conf/all/forwarding": "1",
 					},
 				},
+			},
+		},
 		{
 			Name: "routes_kernel_all_families",
 			Env: map[string]string{
@@ -220,11 +264,15 @@ func TestContainerBoot(t *testing.T) {
 				"TS_ROUTES":    "::/64,1.2.3.0/24",
 				"TS_USERSPACE": "false",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
+				{
+					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=::/64,1.2.3.0/24",
 					},
-			Status1: ipnstate.Status{
+				},
+				{
+					Status: ipnstate.Status{
 						BackendState: "Running",
 						TailscaleIPs: tsIPs,
 					},
@@ -233,6 +281,8 @@ func TestContainerBoot(t *testing.T) {
 						"proc/sys/net/ipv6/conf/all/forwarding": "1",
 					},
 				},
+			},
+		},
 		{
 			Name: "proxy",
 			Env: map[string]string{
@@ -240,20 +290,22 @@ func TestContainerBoot(t *testing.T) {
 				"TS_DEST_IP":   "1.2.3.4",
 				"TS_USERSPACE": "false",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
+				{
+					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
 					},
-			Status1: ipnstate.Status{
+				},
+				{
+					Status: ipnstate.Status{
 						BackendState: "Running",
 						TailscaleIPs: tsIPs,
 					},
-			WantArgs2: []string{
+					WantCmds: []string{
 						"/usr/bin/iptables -t nat -I PREROUTING 1 -d 100.64.0.1 -j DNAT --to-destination 1.2.3.4",
 					},
-			Status2: ipnstate.Status{
+				},
-				BackendState: "Running",
-				TailscaleIPs: tsIPs,
 			},
 		},
 		{
@@ -262,20 +314,28 @@ func TestContainerBoot(t *testing.T) {
 				"TS_AUTH_KEY":  "tskey-key",
 				"TS_AUTH_ONCE": "true",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
+				{
+					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
 					},
-			Status1: ipnstate.Status{
+				},
+				{
+					Status: ipnstate.Status{
 						BackendState: "NeedsLogin",
 					},
-			WantArgs2: []string{
+					WantCmds: []string{
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
 					},
-			Status2: ipnstate.Status{
+				},
+				{
+					Status: ipnstate.Status{
 						BackendState: "Running",
 						TailscaleIPs: tsIPs,
 					},
 				},
+			},
+		},
 		{
 			Name: "kube_storage",
 			Env: map[string]string{
@@ -285,11 +345,18 @@ func TestContainerBoot(t *testing.T) {
 			KubeSecret: map[string]string{
 				"authkey": "tskey-key",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
+				{
+					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
 					},
-			Status1: ipnstate.Status{
+					WantKubeSecret: map[string]string{
+						"authkey": "tskey-key",
+					},
+				},
+				{
+					Status: ipnstate.Status{
 						BackendState: "Running",
 						TailscaleIPs: tsIPs,
 						Self: &ipnstate.PeerStatus{
@@ -301,6 +368,8 @@ func TestContainerBoot(t *testing.T) {
 						"device_id": "myID",
 					},
 				},
+			},
+		},
 		{
 			// Same as previous, but deletes the authkey from the kube secret.
 			Name: "kube_storage_auth_once",
@@ -312,16 +381,28 @@ func TestContainerBoot(t *testing.T) {
 			KubeSecret: map[string]string{
 				"authkey": "tskey-key",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
+				{
+					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
 					},
-			Status1: ipnstate.Status{
+					WantKubeSecret: map[string]string{
+						"authkey": "tskey-key",
+					},
+				},
+				{
+					Status: ipnstate.Status{
 						BackendState: "NeedsLogin",
 					},
-			WantArgs2: []string{
+					WantCmds: []string{
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
 					},
-			Status2: ipnstate.Status{
+					WantKubeSecret: map[string]string{
+						"authkey": "tskey-key",
+					},
+				},
+				{
+					Status: ipnstate.Status{
 						BackendState: "Running",
 						TailscaleIPs: tsIPs,
 						Self: &ipnstate.PeerStatus{
@@ -332,53 +413,72 @@ func TestContainerBoot(t *testing.T) {
 						"device_id": "myID",
 					},
 				},
+			},
+		},
 		{
 			Name: "proxies",
 			Env: map[string]string{
 				"TS_SOCKS5_SERVER":              "localhost:1080",
 				"TS_OUTBOUND_HTTP_PROXY_LISTEN": "localhost:8080",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
+				{
+					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --socks5-server=localhost:1080 --outbound-http-proxy-listen=localhost:8080",
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
 					},
+				},
+				{
 					// The tailscale up call blocks until auth is complete, so
 					// by the time it returns the next converged state is
 					// Running.
-			Status1: ipnstate.Status{
+					Status: ipnstate.Status{
 						BackendState: "Running",
 						TailscaleIPs: tsIPs,
 					},
 				},
+			},
+		},
 		{
 			Name: "dns",
 			Env: map[string]string{
 				"TS_ACCEPT_DNS": "true",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
+				{
+					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=true",
 					},
-			Status1: ipnstate.Status{
+				},
+				{
+					Status: ipnstate.Status{
 						BackendState: "Running",
 						TailscaleIPs: tsIPs,
 					},
 				},
+			},
+		},
 		{
 			Name: "extra_args",
 			Env: map[string]string{
 				"TS_EXTRA_ARGS":            "--widget=rotated",
 				"TS_TAILSCALED_EXTRA_ARGS": "--experiments=widgets",
 			},
-			WantArgs1: []string{
+			Phases: []phase{
+				{
+					WantCmds: []string{
 						"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --experiments=widgets",
 						"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --widget=rotated",
 					},
-			Status1: ipnstate.Status{
+				}, {
+					Status: ipnstate.Status{
 						BackendState: "Running",
 						TailscaleIPs: tsIPs,
 					},
 				},
+			},
+		},
 	}
 
 	for _, test := range tests {
@@ -419,35 +519,45 @@ func TestContainerBoot(t *testing.T) {
 				cmd.Process.Wait()
 			}()
 
-			waitArgs(t, 2*time.Second, d, argFile, strings.Join(test.WantArgs1, "\n"))
+			var wantCmds []string
-			lapi.SetStatus(test.Status1)
+			for _, p := range test.Phases {
-			if test.WantArgs2 != nil {
+				lapi.SetStatus(p.Status)
-				waitArgs(t, 2*time.Second, d, argFile, strings.Join(append(test.WantArgs1, test.WantArgs2...), "\n"))
+				wantCmds = append(wantCmds, p.WantCmds...)
-				lapi.SetStatus(test.Status2)
+				waitArgs(t, 2*time.Second, d, argFile, strings.Join(wantCmds, "\n"))
-			}
+				err := tstest.WaitFor(2*time.Second, func() error {
-			waitLogLine(t, 2*time.Second, cbOut, "Startup complete, waiting for shutdown signal")
+					if p.WantKubeSecret != nil {
-
-			if test.WantKubeSecret != nil {
 						got := kube.Secret()
-				if diff := cmp.Diff(got, test.WantKubeSecret); diff != "" {
+						if diff := cmp.Diff(got, p.WantKubeSecret); diff != "" {
-					t.Fatalf("unexpected kube secret data (-got+want):\n%s", diff)
+							return fmt.Errorf("unexpected kube secret data (-got+want):\n%s", diff)
 						}
 					} else {
 						got := kube.Secret()
-				if len(got) != 0 {
+						if len(got) > 0 {
-					t.Fatalf("kube secret unexpectedly not empty, got %#v", got)
+							return fmt.Errorf("kube secret unexpectedly not empty, got %#v", got)
 						}
 					}
-
+					return nil
-			for path, want := range test.WantFiles {
+				})
+				if err != nil {
+					t.Fatal(err)
+				}
+				err = tstest.WaitFor(2*time.Second, func() error {
+					for path, want := range p.WantFiles {
 						gotBs, err := os.ReadFile(filepath.Join(d, path))
 						if err != nil {
-					t.Fatalf("reading wanted file %q: %v", path, err)
+							return fmt.Errorf("reading wanted file %q: %v", path, err)
 						}
 						if got := strings.TrimSpace(string(gotBs)); got != want {
-					t.Errorf("wrong file contents for %q, got %q want %q", path, got, want)
+							return fmt.Errorf("wrong file contents for %q, got %q want %q", path, got, want)
 						}
 					}
+					return nil
+				})
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			waitLogLine(t, 2*time.Second, cbOut, "Startup complete, waiting for shutdown signal")
 		})
 	}
 }