archive
/
tailscale
mirror of https://github.com/tailscale/tailscale/
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

cmd/containerboot: speed up tests (#14883)

Browse Source 
The test suite had grown to about 20s on my machine, but it doesn't
do much taxing work so was a good candidate to parallelise. Now runs
in under 2s on my machine.

Updates #cleanup

Change-Id: I2fcc6be9ca226c74c0cb6c906778846e959492e4
Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>
pull/15574/head
Tom Proctor 8 months ago committed by GitHub
parent fd580611bd
commit de949b050e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 812 additions and 762 deletions
Show Stats Download Patch File Download Diff File

366
cmd/containerboot/main_test.go
Unescape Escape View File

@ -41,23 +41,34 @@ import (
"tailscale.com/types/ptr" "tailscale.com/types/ptr"
) )
func TestContainerBoot(t *testing.T) { // testEnv represents the environment needed for a single sub-test so that tests
// can run in parallel.
type testEnv struct {
kube *kubeServer // Fake kube server.
lapi *localAPI // Local TS API server.
d string // Temp dir for the specific test.
argFile string // File with commands test_tailscale{,d}.sh were invoked with.
runningSockPath string // Path to the running tailscaled socket.
localAddrPort int // Port for the containerboot HTTP server.
healthAddrPort int // Port for the (deprecated) containerboot health server.
}
func newTestEnv(t *testing.T) testEnv {
d := t.TempDir() d := t.TempDir()
lapi := localAPI{FSRoot: d} lapi := localAPI{FSRoot: d}
if err := lapi.Start(); err != nil { if err := lapi.Start(); err != nil {
t.Fatal(err) t.Fatal(err)
} }
defer lapi.Close() t.Cleanup(lapi.Close)
kube := kubeServer{FSRoot: d} kube := kubeServer{FSRoot: d}
kube.Start(t) kube.Start(t)
defer kube.Close() t.Cleanup(kube.Close)
tailscaledConf := &ipn.ConfigVAlpha{AuthKey: ptr.To("foo"), Version: "alpha0"} tailscaledConf := &ipn.ConfigVAlpha{AuthKey: ptr.To("foo"), Version: "alpha0"}
serveConf := ipn.ServeConfig{TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}} serveConf := ipn.ServeConfig{TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}}
egressCfg := egressSvcConfig("foo", "foo.tailnetxyz.ts.net") egressCfg := egressSvcConfig("foo", "foo.tailnetxyz.ts.net")
egressStatus := egressSvcStatus("foo", "foo.tailnetxyz.ts.net")
dirs := []string{ dirs := []string{
"var/lib", "var/lib",
@ -86,7 +97,6 @@ func TestContainerBoot(t *testing.T) {
filepath.Join("etc/tailscaled/", egressservices.KeyEgressServices): mustJSON(t, egressCfg), filepath.Join("etc/tailscaled/", egressservices.KeyEgressServices): mustJSON(t, egressCfg),
filepath.Join("etc/tailscaled/", egressservices.KeyHEPPings): []byte("4"), filepath.Join("etc/tailscaled/", egressservices.KeyHEPPings): []byte("4"),
} }
resetFiles := func() {
for path, content := range files { for path, content := range files {
// Making everything executable is a little weird, but the // Making everything executable is a little weird, but the
// stuff that doesn't need to be executable doesn't care if we // stuff that doesn't need to be executable doesn't care if we
@ -95,13 +105,6 @@ func TestContainerBoot(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
} }
}
resetFiles()
boot := filepath.Join(d, "containerboot")
if err := exec.Command("go", "build", "-o", boot, "tailscale.com/cmd/containerboot").Run(); err != nil {
t.Fatalf("Building containerboot: %v", err)
}
argFile := filepath.Join(d, "args") argFile := filepath.Join(d, "args")
runningSockPath := filepath.Join(d, "tmp/tailscaled.sock") runningSockPath := filepath.Join(d, "tmp/tailscaled.sock")
@ -117,6 +120,25 @@ func TestContainerBoot(t *testing.T) {
port := ln.Addr().(*net.TCPAddr).Port port := ln.Addr().(*net.TCPAddr).Port
*p = port *p = port
} }
return testEnv{
kube: &kube,
lapi: &lapi,
d: d,
argFile: argFile,
runningSockPath: runningSockPath,
localAddrPort: localAddrPort,
healthAddrPort: healthAddrPort,
}
}
func TestContainerBoot(t *testing.T) {
boot := filepath.Join(t.TempDir(), "containerboot")
if err := exec.Command("go", "build", "-ldflags", "-X main.testSleepDuration=1ms", "-o", boot, "tailscale.com/cmd/containerboot").Run(); err != nil {
t.Fatalf("Building containerboot: %v", err)
}
egressStatus := egressSvcStatus("foo", "foo.tailnetxyz.ts.net")
metricsURL := func(port int) string { metricsURL := func(port int) string {
return fmt.Sprintf("http://127.0.0.1:%d/metrics", port) return fmt.Sprintf("http://127.0.0.1:%d/metrics", port)
} }
@ -173,16 +195,16 @@ func TestContainerBoot(t *testing.T) {
}).View(), }).View(),
}, },
} }
tests := []struct { type testCase struct {
Name string
Env map[string]string Env map[string]string
KubeSecret map[string]string KubeSecret map[string]string
KubeDenyPatch bool KubeDenyPatch bool
Phases []phase Phases []phase
}{ }
{ tests := map[string]func(env *testEnv) testCase{
"no_args": func(env *testEnv) testCase {
return testCase{
// Out of the box default: runs in userspace mode, ephemeral storage, interactive login. // Out of the box default: runs in userspace mode, ephemeral storage, interactive login.
Name: "no_args",
Env: nil, Env: nil,
Phases: []phase{ Phases: []phase{
{ {
@ -200,10 +222,11 @@ func TestContainerBoot(t *testing.T) {
Notify: runningNotify, Notify: runningNotify,
}, },
}, },
}
}, },
{ "authkey": func(env *testEnv) testCase {
return testCase{
// Userspace mode, ephemeral storage, authkey provided on every run. // Userspace mode, ephemeral storage, authkey provided on every run.
Name: "authkey",
Env: map[string]string{ Env: map[string]string{
"TS_AUTHKEY": "tskey-key", "TS_AUTHKEY": "tskey-key",
}, },
@ -218,10 +241,11 @@ func TestContainerBoot(t *testing.T) {
Notify: runningNotify, Notify: runningNotify,
}, },
}, },
}
}, },
{ "authkey_old_flag": func(env *testEnv) testCase {
return testCase{
// Userspace mode, ephemeral storage, authkey provided on every run. // Userspace mode, ephemeral storage, authkey provided on every run.
Name: "authkey-old-flag",
Env: map[string]string{ Env: map[string]string{
"TS_AUTH_KEY": "tskey-key", "TS_AUTH_KEY": "tskey-key",
}, },
@ -236,12 +260,13 @@ func TestContainerBoot(t *testing.T) {
Notify: runningNotify, Notify: runningNotify,
}, },
}, },
}
}, },
{ "authkey_disk_state": func(env *testEnv) testCase {
Name: "authkey_disk_state", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_AUTHKEY": "tskey-key", "TS_AUTHKEY": "tskey-key",
"TS_STATE_DIR": filepath.Join(d, "tmp"), "TS_STATE_DIR": filepath.Join(env.d, "tmp"),
}, },
Phases: []phase{ Phases: []phase{
{ {
@ -254,9 +279,10 @@ func TestContainerBoot(t *testing.T) {
Notify: runningNotify, Notify: runningNotify,
}, },
}, },
}
}, },
{ "routes": func(env *testEnv) testCase {
Name: "routes", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_AUTHKEY": "tskey-key", "TS_AUTHKEY": "tskey-key",
"TS_ROUTES": "1.2.3.0/24,10.20.30.0/24", "TS_ROUTES": "1.2.3.0/24,10.20.30.0/24",
@ -276,9 +302,10 @@ func TestContainerBoot(t *testing.T) {
}, },
}, },
}, },
}
}, },
{ "empty_routes": func(env *testEnv) testCase {
Name: "empty routes", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_AUTHKEY": "tskey-key", "TS_AUTHKEY": "tskey-key",
"TS_ROUTES": "", "TS_ROUTES": "",
@ -298,9 +325,10 @@ func TestContainerBoot(t *testing.T) {
}, },
}, },
}, },
}
}, },
{ "routes_kernel_ipv4": func(env *testEnv) testCase {
Name: "routes_kernel_ipv4", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_AUTHKEY": "tskey-key", "TS_AUTHKEY": "tskey-key",
"TS_ROUTES": "1.2.3.0/24,10.20.30.0/24", "TS_ROUTES": "1.2.3.0/24,10.20.30.0/24",
@ -321,9 +349,10 @@ func TestContainerBoot(t *testing.T) {
}, },
}, },
}, },
}
}, },
{ "routes_kernel_ipv6": func(env *testEnv) testCase {
Name: "routes_kernel_ipv6", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_AUTHKEY": "tskey-key", "TS_AUTHKEY": "tskey-key",
"TS_ROUTES": "::/64,1::/64", "TS_ROUTES": "::/64,1::/64",
@ -344,9 +373,10 @@ func TestContainerBoot(t *testing.T) {
}, },
}, },
}, },
}
}, },
{ "routes_kernel_all_families": func(env *testEnv) testCase {
Name: "routes_kernel_all_families", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_AUTHKEY": "tskey-key", "TS_AUTHKEY": "tskey-key",
"TS_ROUTES": "::/64,1.2.3.0/24", "TS_ROUTES": "::/64,1.2.3.0/24",
@ -367,9 +397,10 @@ func TestContainerBoot(t *testing.T) {
}, },
}, },
}, },
}
}, },
{ "ingress_proxy": func(env *testEnv) testCase {
Name: "ingress proxy", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_AUTHKEY": "tskey-key", "TS_AUTHKEY": "tskey-key",
"TS_DEST_IP": "1.2.3.4", "TS_DEST_IP": "1.2.3.4",
@ -386,9 +417,10 @@ func TestContainerBoot(t *testing.T) {
Notify: runningNotify, Notify: runningNotify,
}, },
}, },
}
}, },
{ "egress_proxy": func(env *testEnv) testCase {
Name: "egress proxy", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_AUTHKEY": "tskey-key", "TS_AUTHKEY": "tskey-key",
"TS_TAILNET_TARGET_IP": "100.99.99.99", "TS_TAILNET_TARGET_IP": "100.99.99.99",
@ -409,9 +441,10 @@ func TestContainerBoot(t *testing.T) {
Notify: runningNotify, Notify: runningNotify,
}, },
}, },
}
}, },
{ "egress_proxy_fqdn_ipv6_target_on_ipv4_host": func(env *testEnv) testCase {
Name: "egress_proxy_fqdn_ipv6_target_on_ipv4_host", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_AUTHKEY": "tskey-key", "TS_AUTHKEY": "tskey-key",
"TS_TAILNET_TARGET_FQDN": "ipv6-node.test.ts.net", // resolves to IPv6 address "TS_TAILNET_TARGET_FQDN": "ipv6-node.test.ts.net", // resolves to IPv6 address
@ -451,9 +484,10 @@ func TestContainerBoot(t *testing.T) {
WantExitCode: ptr.To(1), WantExitCode: ptr.To(1),
}, },
}, },
}
}, },
{ "authkey_once": func(env *testEnv) testCase {
Name: "authkey_once", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_AUTHKEY": "tskey-key", "TS_AUTHKEY": "tskey-key",
"TS_AUTH_ONCE": "true", "TS_AUTH_ONCE": "true",
@ -479,12 +513,13 @@ func TestContainerBoot(t *testing.T) {
}, },
}, },
}, },
}
}, },
{ "kube_storage": func(env *testEnv) testCase {
Name: "kube_storage", return testCase{
Env: map[string]string{ Env: map[string]string{
"KUBERNETES_SERVICE_HOST": kube.Host, "KUBERNETES_SERVICE_HOST": env.kube.Host,
"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port,
}, },
KubeSecret: map[string]string{ KubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
@ -510,15 +545,16 @@ func TestContainerBoot(t *testing.T) {
}, },
}, },
}, },
}
}, },
{ "kube_disk_storage": func(env *testEnv) testCase {
Name: "kube_disk_storage", return testCase{
Env: map[string]string{ Env: map[string]string{
"KUBERNETES_SERVICE_HOST": kube.Host, "KUBERNETES_SERVICE_HOST": env.kube.Host,
"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port,
// Explicitly set to an empty value, to override the default of "tailscale". // Explicitly set to an empty value, to override the default of "tailscale".
"TS_KUBE_SECRET": "", "TS_KUBE_SECRET": "",
"TS_STATE_DIR": filepath.Join(d, "tmp"), "TS_STATE_DIR": filepath.Join(env.d, "tmp"),
"TS_AUTHKEY": "tskey-key", "TS_AUTHKEY": "tskey-key",
}, },
KubeSecret: map[string]string{}, KubeSecret: map[string]string{},
@ -535,12 +571,13 @@ func TestContainerBoot(t *testing.T) {
WantKubeSecret: map[string]string{}, WantKubeSecret: map[string]string{},
}, },
}, },
}
}, },
{ "kube_storage_no_patch": func(env *testEnv) testCase {
Name: "kube_storage_no_patch", return testCase{
Env: map[string]string{ Env: map[string]string{
"KUBERNETES_SERVICE_HOST": kube.Host, "KUBERNETES_SERVICE_HOST": env.kube.Host,
"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port,
"TS_AUTHKEY": "tskey-key", "TS_AUTHKEY": "tskey-key",
}, },
KubeSecret: map[string]string{}, KubeSecret: map[string]string{},
@ -558,13 +595,14 @@ func TestContainerBoot(t *testing.T) {
WantKubeSecret: map[string]string{}, WantKubeSecret: map[string]string{},
}, },
}, },
}
}, },
{ "kube_storage_auth_once": func(env *testEnv) testCase {
return testCase{
// Same as previous, but deletes the authkey from the kube secret. // Same as previous, but deletes the authkey from the kube secret.
Name: "kube_storage_auth_once",
Env: map[string]string{ Env: map[string]string{
"KUBERNETES_SERVICE_HOST": kube.Host, "KUBERNETES_SERVICE_HOST": env.kube.Host,
"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port,
"TS_AUTH_ONCE": "true", "TS_AUTH_ONCE": "true",
}, },
KubeSecret: map[string]string{ KubeSecret: map[string]string{
@ -603,12 +641,13 @@ func TestContainerBoot(t *testing.T) {
}, },
}, },
}, },
}
}, },
{ "kube_storage_updates": func(env *testEnv) testCase {
Name: "kube_storage_updates", return testCase{
Env: map[string]string{ Env: map[string]string{
"KUBERNETES_SERVICE_HOST": kube.Host, "KUBERNETES_SERVICE_HOST": env.kube.Host,
"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port,
}, },
KubeSecret: map[string]string{ KubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
@ -653,9 +692,10 @@ func TestContainerBoot(t *testing.T) {
}, },
}, },
}, },
}
}, },
{ "proxies": func(env *testEnv) testCase {
Name: "proxies", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_SOCKS5_SERVER": "localhost:1080", "TS_SOCKS5_SERVER": "localhost:1080",
"TS_OUTBOUND_HTTP_PROXY_LISTEN": "localhost:8080", "TS_OUTBOUND_HTTP_PROXY_LISTEN": "localhost:8080",
@ -671,9 +711,10 @@ func TestContainerBoot(t *testing.T) {
Notify: runningNotify, Notify: runningNotify,
}, },
}, },
}
}, },
{ "dns": func(env *testEnv) testCase {
Name: "dns", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_ACCEPT_DNS": "true", "TS_ACCEPT_DNS": "true",
}, },
@ -688,9 +729,10 @@ func TestContainerBoot(t *testing.T) {
Notify: runningNotify, Notify: runningNotify,
}, },
}, },
}
}, },
{ "extra_args": func(env *testEnv) testCase {
Name: "extra_args", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_EXTRA_ARGS": "--widget=rotated", "TS_EXTRA_ARGS": "--widget=rotated",
"TS_TAILSCALED_EXTRA_ARGS": "--experiments=widgets", "TS_TAILSCALED_EXTRA_ARGS": "--experiments=widgets",
@ -705,9 +747,10 @@ func TestContainerBoot(t *testing.T) {
Notify: runningNotify, Notify: runningNotify,
}, },
}, },
}
}, },
{ "extra_args_accept_routes": func(env *testEnv) testCase {
Name: "extra_args_accept_routes", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_EXTRA_ARGS": "--accept-routes", "TS_EXTRA_ARGS": "--accept-routes",
}, },
@ -721,9 +764,10 @@ func TestContainerBoot(t *testing.T) {
Notify: runningNotify, Notify: runningNotify,
}, },
}, },
}
}, },
{ "hostname": func(env *testEnv) testCase {
Name: "hostname", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_HOSTNAME": "my-server", "TS_HOSTNAME": "my-server",
}, },
@ -737,11 +781,12 @@ func TestContainerBoot(t *testing.T) {
Notify: runningNotify, Notify: runningNotify,
}, },
}, },
}
}, },
{ "experimental_tailscaled_config_path": func(env *testEnv) testCase {
Name: "experimental tailscaled config path", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR": filepath.Join(d, "etc/tailscaled/"), "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR": filepath.Join(env.d, "etc/tailscaled/"),
}, },
Phases: []phase{ Phases: []phase{
{ {
@ -752,11 +797,12 @@ func TestContainerBoot(t *testing.T) {
Notify: runningNotify, Notify: runningNotify,
}, },
}, },
}
}, },
{ "metrics_enabled": func(env *testEnv) testCase {
Name: "metrics_enabled", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", localAddrPort), "TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", env.localAddrPort),
"TS_ENABLE_METRICS": "true", "TS_ENABLE_METRICS": "true",
}, },
Phases: []phase{ Phases: []phase{
@ -766,18 +812,19 @@ func TestContainerBoot(t *testing.T) {
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
}, },
EndpointStatuses: map[string]int{ EndpointStatuses: map[string]int{
metricsURL(localAddrPort): 200, metricsURL(env.localAddrPort): 200,
healthURL(localAddrPort): -1, healthURL(env.localAddrPort): -1,
}, },
}, { }, {
Notify: runningNotify, Notify: runningNotify,
}, },
}, },
}
}, },
{ "health_enabled": func(env *testEnv) testCase {
Name: "health_enabled", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", localAddrPort), "TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", env.localAddrPort),
"TS_ENABLE_HEALTH_CHECK": "true", "TS_ENABLE_HEALTH_CHECK": "true",
}, },
Phases: []phase{ Phases: []phase{
@ -787,22 +834,23 @@ func TestContainerBoot(t *testing.T) {
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
}, },
EndpointStatuses: map[string]int{ EndpointStatuses: map[string]int{
metricsURL(localAddrPort): -1, metricsURL(env.localAddrPort): -1,
healthURL(localAddrPort): 503, // Doesn't start passing until the next phase. healthURL(env.localAddrPort): 503, // Doesn't start passing until the next phase.
}, },
}, { }, {
Notify: runningNotify, Notify: runningNotify,
EndpointStatuses: map[string]int{ EndpointStatuses: map[string]int{
metricsURL(localAddrPort): -1, metricsURL(env.localAddrPort): -1,
healthURL(localAddrPort): 200, healthURL(env.localAddrPort): 200,
}, },
}, },
}, },
}
}, },
{ "metrics_and_health_on_same_port": func(env *testEnv) testCase {
Name: "metrics_and_health_on_same_port", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", localAddrPort), "TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", env.localAddrPort),
"TS_ENABLE_METRICS": "true", "TS_ENABLE_METRICS": "true",
"TS_ENABLE_HEALTH_CHECK": "true", "TS_ENABLE_HEALTH_CHECK": "true",
}, },
@ -813,24 +861,25 @@ func TestContainerBoot(t *testing.T) {
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
}, },
EndpointStatuses: map[string]int{ EndpointStatuses: map[string]int{
metricsURL(localAddrPort): 200, metricsURL(env.localAddrPort): 200,
healthURL(localAddrPort): 503, // Doesn't start passing until the next phase. healthURL(env.localAddrPort): 503, // Doesn't start passing until the next phase.
}, },
}, { }, {
Notify: runningNotify, Notify: runningNotify,
EndpointStatuses: map[string]int{ EndpointStatuses: map[string]int{
metricsURL(localAddrPort): 200, metricsURL(env.localAddrPort): 200,
healthURL(localAddrPort): 200, healthURL(env.localAddrPort): 200,
}, },
}, },
}, },
}
}, },
{ "local_metrics_and_deprecated_health": func(env *testEnv) testCase {
Name: "local_metrics_and_deprecated_health", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", localAddrPort), "TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", env.localAddrPort),
"TS_ENABLE_METRICS": "true", "TS_ENABLE_METRICS": "true",
"TS_HEALTHCHECK_ADDR_PORT": fmt.Sprintf("[::]:%d", healthAddrPort), "TS_HEALTHCHECK_ADDR_PORT": fmt.Sprintf("[::]:%d", env.healthAddrPort),
}, },
Phases: []phase{ Phases: []phase{
{ {
@ -839,22 +888,23 @@ func TestContainerBoot(t *testing.T) {
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false", "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
}, },
EndpointStatuses: map[string]int{ EndpointStatuses: map[string]int{
metricsURL(localAddrPort): 200, metricsURL(env.localAddrPort): 200,
healthURL(healthAddrPort): 503, // Doesn't start passing until the next phase. healthURL(env.healthAddrPort): 503, // Doesn't start passing until the next phase.
}, },
}, { }, {
Notify: runningNotify, Notify: runningNotify,
EndpointStatuses: map[string]int{ EndpointStatuses: map[string]int{
metricsURL(localAddrPort): 200, metricsURL(env.localAddrPort): 200,
healthURL(healthAddrPort): 200, healthURL(env.healthAddrPort): 200,
}, },
}, },
}, },
}
}, },
{ "serve_config_no_kube": func(env *testEnv) testCase {
Name: "serve_config_no_kube", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_SERVE_CONFIG": filepath.Join(d, "etc/tailscaled/serve-config.json"), "TS_SERVE_CONFIG": filepath.Join(env.d, "etc/tailscaled/serve-config.json"),
"TS_AUTHKEY": "tskey-key", "TS_AUTHKEY": "tskey-key",
}, },
Phases: []phase{ Phases: []phase{
@ -868,13 +918,14 @@ func TestContainerBoot(t *testing.T) {
Notify: runningNotify, Notify: runningNotify,
}, },
}, },
}
}, },
{ "serve_config_kube": func(env *testEnv) testCase {
Name: "serve_config_kube", return testCase{
Env: map[string]string{ Env: map[string]string{
"KUBERNETES_SERVICE_HOST": kube.Host, "KUBERNETES_SERVICE_HOST": env.kube.Host,
"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port,
"TS_SERVE_CONFIG": filepath.Join(d, "etc/tailscaled/serve-config.json"), "TS_SERVE_CONFIG": filepath.Join(env.d, "etc/tailscaled/serve-config.json"),
}, },
KubeSecret: map[string]string{ KubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
@ -901,14 +952,15 @@ func TestContainerBoot(t *testing.T) {
}, },
}, },
}, },
}
}, },
{ "egress_svcs_config_kube": func(env *testEnv) testCase {
Name: "egress_svcs_config_kube", return testCase{
Env: map[string]string{ Env: map[string]string{
"KUBERNETES_SERVICE_HOST": kube.Host, "KUBERNETES_SERVICE_HOST": env.kube.Host,
"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port,
"TS_EGRESS_PROXIES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled"), "TS_EGRESS_PROXIES_CONFIG_PATH": filepath.Join(env.d, "etc/tailscaled"),
"TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", localAddrPort), "TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", env.localAddrPort),
}, },
KubeSecret: map[string]string{ KubeSecret: map[string]string{
"authkey": "tskey-key", "authkey": "tskey-key",
@ -923,7 +975,7 @@ func TestContainerBoot(t *testing.T) {
"authkey": "tskey-key", "authkey": "tskey-key",
}, },
EndpointStatuses: map[string]int{ EndpointStatuses: map[string]int{
egressSvcTerminateURL(localAddrPort): 200, egressSvcTerminateURL(env.localAddrPort): 200,
}, },
}, },
{ {
@ -937,15 +989,16 @@ func TestContainerBoot(t *testing.T) {
"tailscale_capver": capver, "tailscale_capver": capver,
}, },
EndpointStatuses: map[string]int{ EndpointStatuses: map[string]int{
egressSvcTerminateURL(localAddrPort): 200, egressSvcTerminateURL(env.localAddrPort): 200,
}, },
}, },
}, },
}
}, },
{ "egress_svcs_config_no_kube": func(env *testEnv) testCase {
Name: "egress_svcs_config_no_kube", return testCase{
Env: map[string]string{ Env: map[string]string{
"TS_EGRESS_PROXIES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled"), "TS_EGRESS_PROXIES_CONFIG_PATH": filepath.Join(env.d, "etc/tailscaled"),
"TS_AUTHKEY": "tskey-key", "TS_AUTHKEY": "tskey-key",
}, },
Phases: []phase{ Phases: []phase{
@ -954,12 +1007,13 @@ func TestContainerBoot(t *testing.T) {
WantExitCode: ptr.To(1), WantExitCode: ptr.To(1),
}, },
}, },
}
}, },
{ "kube_shutdown_during_state_write": func(env *testEnv) testCase {
Name: "kube_shutdown_during_state_write", return testCase{
Env: map[string]string{ Env: map[string]string{
"KUBERNETES_SERVICE_HOST": kube.Host, "KUBERNETES_SERVICE_HOST": env.kube.Host,
"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port,
"TS_ENABLE_HEALTH_CHECK": "true", "TS_ENABLE_HEALTH_CHECK": "true",
}, },
KubeSecret: map[string]string{ KubeSecret: map[string]string{
@ -1010,32 +1064,31 @@ func TestContainerBoot(t *testing.T) {
WantExitCode: ptr.To(0), WantExitCode: ptr.To(0),
}, },
}, },
}
}, },
} }
for _, test := range tests { for name, test := range tests {
t.Run(test.Name, func(t *testing.T) { t.Run(name, func(t *testing.T) {
lapi.Reset() t.Parallel()
kube.Reset() env := newTestEnv(t)
os.Remove(argFile) tc := test(&env)
os.Remove(runningSockPath)
resetFiles()
for k, v := range test.KubeSecret { for k, v := range tc.KubeSecret {
kube.SetSecret(k, v) env.kube.SetSecret(k, v)
} }
kube.SetPatching(!test.KubeDenyPatch) env.kube.SetPatching(!tc.KubeDenyPatch)
cmd := exec.Command(boot) cmd := exec.Command(boot)
cmd.Env = []string{ cmd.Env = []string{
fmt.Sprintf("PATH=%s/usr/bin:%s", d, os.Getenv("PATH")), fmt.Sprintf("PATH=%s/usr/bin:%s", env.d, os.Getenv("PATH")),
fmt.Sprintf("TS_TEST_RECORD_ARGS=%s", argFile), fmt.Sprintf("TS_TEST_RECORD_ARGS=%s", env.argFile),
fmt.Sprintf("TS_TEST_SOCKET=%s", lapi.Path), fmt.Sprintf("TS_TEST_SOCKET=%s", env.lapi.Path),
fmt.Sprintf("TS_SOCKET=%s", runningSockPath), fmt.Sprintf("TS_SOCKET=%s", env.runningSockPath),
fmt.Sprintf("TS_TEST_ONLY_ROOT=%s", d), fmt.Sprintf("TS_TEST_ONLY_ROOT=%s", env.d),
fmt.Sprint("TS_TEST_FAKE_NETFILTER=true"), fmt.Sprint("TS_TEST_FAKE_NETFILTER=true"),
} }
for k, v := range test.Env { for k, v := range tc.Env {
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
} }
cbOut := &lockingBuffer{} cbOut := &lockingBuffer{}
@ -1045,6 +1098,7 @@ func TestContainerBoot(t *testing.T) {
} }
}() }()
cmd.Stderr = cbOut cmd.Stderr = cbOut
cmd.Stdout = cbOut
if err := cmd.Start(); err != nil { if err := cmd.Start(); err != nil {
t.Fatalf("starting containerboot: %v", err) t.Fatalf("starting containerboot: %v", err)
} }
@ -1054,11 +1108,11 @@ func TestContainerBoot(t *testing.T) {
}() }()
var wantCmds []string var wantCmds []string
for i, p := range test.Phases { for i, p := range tc.Phases {
for k, v := range p.UpdateKubeSecret { for k, v := range p.UpdateKubeSecret {
kube.SetSecret(k, v) env.kube.SetSecret(k, v)
} }
lapi.Notify(p.Notify) env.lapi.Notify(p.Notify)
if p.Signal != nil { if p.Signal != nil {
cmd.Process.Signal(*p.Signal) cmd.Process.Signal(*p.Signal)
} }
@ -1086,15 +1140,15 @@ func TestContainerBoot(t *testing.T) {
} }
wantCmds = append(wantCmds, p.WantCmds...) wantCmds = append(wantCmds, p.WantCmds...)
waitArgs(t, 2*time.Second, d, argFile, strings.Join(wantCmds, "\n")) waitArgs(t, 2*time.Second, env.d, env.argFile, strings.Join(wantCmds, "\n"))
err := tstest.WaitFor(2*time.Second, func() error { err := tstest.WaitFor(2*time.Second, func() error {
if p.WantKubeSecret != nil { if p.WantKubeSecret != nil {
got := kube.Secret() got := env.kube.Secret()
if diff := cmp.Diff(got, p.WantKubeSecret); diff != "" { if diff := cmp.Diff(got, p.WantKubeSecret); diff != "" {
return fmt.Errorf("unexpected kube secret data (-got+want):\n%s", diff) return fmt.Errorf("unexpected kube secret data (-got+want):\n%s", diff)
} }
} else { } else {
got := kube.Secret() got := env.kube.Secret()
if len(got) > 0 { if len(got) > 0 {
return fmt.Errorf("kube secret unexpectedly not empty, got %#v", got) return fmt.Errorf("kube secret unexpectedly not empty, got %#v", got)
} }
@ -1106,7 +1160,7 @@ func TestContainerBoot(t *testing.T) {
} }
err = tstest.WaitFor(2*time.Second, func() error { err = tstest.WaitFor(2*time.Second, func() error {
for path, want := range p.WantFiles { for path, want := range p.WantFiles {
gotBs, err := os.ReadFile(filepath.Join(d, path)) gotBs, err := os.ReadFile(filepath.Join(env.d, path))
if err != nil { if err != nil {
return fmt.Errorf("reading wanted file %q: %v", path, err) return fmt.Errorf("reading wanted file %q: %v", path, err)
} }
@ -1270,13 +1324,6 @@ func (l *localAPI) Close() {
l.srv.Close() l.srv.Close()
} }
func (l *localAPI) Reset() {
l.Lock()
defer l.Unlock()
l.notify = nil
l.cond.Broadcast()
}
func (l *localAPI) Notify(n *ipn.Notify) { func (l *localAPI) Notify(n *ipn.Notify) {
if n == nil { if n == nil {
return return
@ -1368,13 +1415,8 @@ func (k *kubeServer) SetPatching(canPatch bool) {
k.canPatch = canPatch k.canPatch = canPatch
} }
func (k *kubeServer) Reset() {
k.Lock()
defer k.Unlock()
k.secret = map[string]string{}
}
func (k *kubeServer) Start(t *testing.T) { func (k *kubeServer) Start(t *testing.T) {
k.secret = map[string]string{}
root := filepath.Join(k.FSRoot, "var/run/secrets/kubernetes.io/serviceaccount") root := filepath.Join(k.FSRoot, "var/run/secrets/kubernetes.io/serviceaccount")
if err := os.MkdirAll(root, 0700); err != nil { if err := os.MkdirAll(root, 0700); err != nil {

12
cmd/containerboot/services.go
Unescape Escape View File

@ -35,6 +35,9 @@ import (
const tailscaleTunInterface = "tailscale0" const tailscaleTunInterface = "tailscale0"
// Modified using a build flag to speed up tests.
var testSleepDuration string
// This file contains functionality to run containerboot as a proxy that can // This file contains functionality to run containerboot as a proxy that can
// route cluster traffic to one or more tailnet targets, based on portmapping // route cluster traffic to one or more tailnet targets, based on portmapping
// rules read from a configfile. Currently (9/2024) this is only used for the // rules read from a configfile. Currently (9/2024) this is only used for the
@ -149,8 +152,13 @@ func (ep *egressProxy) configure(opts egressProxyRunOpts) {
ep.podIPv4 = opts.podIPv4 ep.podIPv4 = opts.podIPv4
ep.tailnetAddrs = opts.tailnetAddrs ep.tailnetAddrs = opts.tailnetAddrs
ep.client = &http.Client{} // default HTTP client ep.client = &http.Client{} // default HTTP client
ep.shortSleep = time.Second sleepDuration := time.Second
ep.longSleep = time.Second * 10 if d, err := time.ParseDuration(testSleepDuration); err == nil && d > 0 {
log.Printf("using test sleep duration %v", d)
sleepDuration = d
}
ep.shortSleep = sleepDuration
ep.longSleep = sleepDuration * 10
} }
// sync triggers an egress proxy config resync. The resync calculates the diff between config and status to determine if // sync triggers an egress proxy config resync. The resync calculates the diff between config and status to determine if

4
cmd/containerboot/tailscaled.go
Unescape Escape View File

@ -38,11 +38,11 @@ func startTailscaled(ctx context.Context, cfg *settings) (*local.Client, *os.Pro
} }
log.Printf("Starting tailscaled") log.Printf("Starting tailscaled")
if err := cmd.Start(); err != nil { if err := cmd.Start(); err != nil {
return nil, nil, fmt.Errorf("starting tailscaled failed: %v", err) return nil, nil, fmt.Errorf("starting tailscaled failed: %w", err)
} }
// Wait for the socket file to appear, otherwise API ops will racily fail. // Wait for the socket file to appear, otherwise API ops will racily fail.
log.Printf("Waiting for tailscaled socket") log.Printf("Waiting for tailscaled socket at %s", cfg.Socket)
for { for {
if ctx.Err() != nil { if ctx.Err() != nil {
return nil, nil, errors.New("timed out waiting for tailscaled socket") return nil, nil, errors.New("timed out waiting for tailscaled socket")

Write Preview
Loading…
Cancel
Save