From dec68166e416d661e1482e2ed4d2ac162cd94488 Mon Sep 17 00:00:00 2001 From: Tom DNetto Date: Tue, 12 Apr 2022 13:04:36 -0700 Subject: [PATCH] tstest/integration/vms: smoke test derphttp through mitm proxies Updates #4377 Very smoky/high-level test to ensure that derphttp internals play well with an agressive (stare + bump) meddler-in-the-middle proxy. Signed-off-by: Tom DNetto --- tstest/integration/vms/nixos_test.go | 2 +- tstest/integration/vms/squid.conf | 39 ++++++++ tstest/integration/vms/top_level_test.go | 110 +++++++++++++++++++++-- tstest/integration/vms/vm_setup_test.go | 14 +++ tstest/integration/vms/vms_test.go | 30 ++++--- 5 files changed, 175 insertions(+), 20 deletions(-) create mode 100644 tstest/integration/vms/squid.conf diff --git a/tstest/integration/vms/nixos_test.go b/tstest/integration/vms/nixos_test.go index c7be57744..d4f887de3 100644 --- a/tstest/integration/vms/nixos_test.go +++ b/tstest/integration/vms/nixos_test.go @@ -124,7 +124,7 @@ in { systemd.services.cloud-final.path = with pkgs; [ curl ]; # Curl is needed for one of the integration tests - environment.systemPackages = with pkgs; [ curl ]; + environment.systemPackages = with pkgs; [ curl nix bash squid openssl daemonize ]; # yolo, this vm can sudo freely. security.sudo.wheelNeedsPassword = false; diff --git a/tstest/integration/vms/squid.conf b/tstest/integration/vms/squid.conf new file mode 100644 index 000000000..29d32bd6d --- /dev/null +++ b/tstest/integration/vms/squid.conf @@ -0,0 +1,39 @@ +pid_filename /run/squid.pid +cache_dir ufs /tmp/squid/cache 500 16 256 +maximum_object_size 4096 KB +coredump_dir /tmp/squid/core +visible_hostname localhost +cache_access_log /tmp/squid/access.log +cache_log /tmp/squid/cache.log + +# Access Control lists +acl localhost src 127.0.0.1 ::1 +acl manager proto cache_object +acl SSL_ports port 443 +acl Safe_ports port 80 # http +acl Safe_ports port 21 # ftp +acl Safe_ports port 443 # https +acl Safe_ports port 70 # gopher +acl Safe_ports port 210 # wais +acl Safe_ports port 1025-65535 # unregistered ports +acl Safe_ports port 280 # http-mgmt +acl Safe_ports port 488 # gss-http +acl Safe_ports port 591 # filemaker +acl Safe_ports port 777 # multiling http +acl CONNECT method CONNECT + +http_access allow localhost +http_access deny all +forwarded_for on + +# sslcrtd_program /nix/store/nqlqk1f6qlxdirlrl1aijgb6vbzxs0gs-squid-4.17/libexec/security_file_certgen -s /tmp/squid/ssl_db -M 4MB +sslcrtd_children 5 + +http_port 127.0.0.1:3128 \ + ssl-bump \ + generate-host-certificates=on \ + dynamic_cert_mem_cache_size=4MB \ + cert=/tmp/squid/myca-mitm.pem + +ssl_bump stare all # mimic the Client Hello, drop unsupported extensions +ssl_bump bump all # terminate and establish new TLS connection \ No newline at end of file diff --git a/tstest/integration/vms/top_level_test.go b/tstest/integration/vms/top_level_test.go index a7d6172d0..6e14b493f 100644 --- a/tstest/integration/vms/top_level_test.go +++ b/tstest/integration/vms/top_level_test.go @@ -7,20 +7,120 @@ package vms -import "testing" +import ( + "context" + "testing" + "time" + + "github.com/pkg/sftp" + expect "github.com/tailscale/goexpect" +) func TestRunUbuntu1804(t *testing.T) { - setupTests(t) testOneDistribution(t, 0, Distros[0]) } func TestRunUbuntu2004(t *testing.T) { - setupTests(t) testOneDistribution(t, 1, Distros[1]) } func TestRunNixos2111(t *testing.T) { t.Parallel() - setupTests(t) testOneDistribution(t, 2, Distros[2]) -} \ No newline at end of file +} + +// TestMITMProxy is a smoke test for derphttp through a MITM proxy. +// Encountering such proxies is unfortunately commonplace in more +// traditional enterprise networks. +// +// We invoke tailscale netcheck because the networking check is done +// by tailscale rather than tailscaled, making it easier to configure +// the proxy. +// +// To provide the actual MITM server, we use squid. +func TestMITMProxy(t *testing.T) { + t.Parallel() + setupTests(t) + distro := Distros[2] // nixos-21.11 + + if distroRex.Unwrap().MatchString(distro.Name) { + t.Logf("%s matches %s", distro.Name, distroRex.Unwrap()) + } else { + t.Skip("regex not matched") + } + + ctx, done := context.WithCancel(context.Background()) + t.Cleanup(done) + + h := newHarness(t) + + err := ramsem.sem.Acquire(ctx, int64(distro.MemoryMegs)) + if err != nil { + t.Fatalf("can't acquire ram semaphore: %v", err) + } + t.Cleanup(func() { ramsem.sem.Release(int64(distro.MemoryMegs)) }) + + vm := h.mkVM(t, 2, distro, h.pubKey, h.loginServerURL, t.TempDir()) + vm.waitStartup(t) + + ipm := h.waitForIPMap(t, vm, distro) + _, cli := h.setupSSHShell(t, distro, ipm) + + sftpCli, err := sftp.NewClient(cli) + if err != nil { + t.Fatalf("can't connect over sftp to copy binaries: %v", err) + } + defer sftpCli.Close() + + // Initialize a squid installation. + // + // A few things of note here: + // - The first thing we do is append the nsslcrtd_program stanza to the config. + // This must be an absolute path and is based on the nix path of the squid derivation, + // so we compute and write it out here. + // - Squid expects a pre-initalized directory layout, so we create that in /tmp/squid then + // invoke squid with -z to have it fill in the rest. + // - Doing a meddler-in-the-middle attack requires using some fake keys, so we create + // them using openssl and then use the security_file_certgen tool to setup squids' ssl_db. + // - There were some perms issues, so i yeeted 0777. Its only a test anyway + copyFile(t, sftpCli, "squid.conf", "/tmp/squid.conf") + runTestCommands(t, 30*time.Second, cli, []expect.Batcher{ + &expect.BSnd{S: "echo -e \"\\nsslcrtd_program $(nix eval --raw nixpkgs.squid)/libexec/security_file_certgen -s /tmp/squid/ssl_db -M 4MB\\n\" >> /tmp/squid.conf\n"}, + &expect.BSnd{S: "mkdir -p /tmp/squid/{cache,core}\n"}, + &expect.BSnd{S: "openssl req -batch -new -newkey rsa:4096 -sha256 -days 3650 -nodes -x509 -keyout /tmp/squid/myca-mitm.pem -out /tmp/squid/myca-mitm.pem\n"}, + &expect.BExp{R: `writing new private key to '/tmp/squid/myca-mitm.pem'`}, + &expect.BSnd{S: "$(nix eval --raw nixpkgs.squid)/libexec/security_file_certgen -c -s /tmp/squid/ssl_db -M 4MB\n"}, + &expect.BExp{R: `Done`}, + &expect.BSnd{S: "sudo chmod -R 0777 /tmp/squid\n"}, + &expect.BSnd{S: "squid --foreground -YCs -z -f /tmp/squid.conf\n"}, + &expect.BSnd{S: "echo Success.\n"}, + &expect.BExp{R: `Success.`}, + }) + + // Start the squid server. + runTestCommands(t, 10*time.Second, cli, []expect.Batcher{ + &expect.BSnd{S: "daemonize -v -c /tmp/squid $(nix eval --raw nixpkgs.squid)/bin/squid --foreground -YCs -f /tmp/squid.conf\n"}, // start daemon + // NOTE(tom): Writing to /dev/tcp/* is bash magic, not a file. This + // eldritchian incantation lets us wait till squid is up. + &expect.BSnd{S: "while ! timeout 5 bash -c 'echo > /dev/tcp/localhost/3128'; do sleep 1; done\n"}, + &expect.BSnd{S: "echo Success.\n"}, + &expect.BExp{R: `Success.`}, + }) + + // Uncomment to help debugging this test if it fails. + // + // runTestCommands(t, 30 * time.Second, cli, []expect.Batcher{ + // &expect.BSnd{S: "sudo ifconfig\n"}, + // &expect.BSnd{S: "sudo ip link\n"}, + // &expect.BSnd{S: "sudo ip route\n"}, + // &expect.BSnd{S: "ps -aux\n"}, + // &expect.BSnd{S: "netstat -a\n"}, + // &expect.BSnd{S: "cat /tmp/squid/access.log && cat /tmp/squid/cache.log && cat /tmp/squid.conf && echo Success.\n"}, + // &expect.BExp{R: `Success.`}, + // }) + + runTestCommands(t, 30*time.Second, cli, []expect.Batcher{ + &expect.BSnd{S: "SSL_CERT_FILE=/tmp/squid/myca-mitm.pem HTTPS_PROXY=http://127.0.0.1:3128 tailscale netcheck\n"}, + &expect.BExp{R: `IPv4: yes`}, + }) +} diff --git a/tstest/integration/vms/vm_setup_test.go b/tstest/integration/vms/vm_setup_test.go index f61bdff77..7c3e61d4d 100644 --- a/tstest/integration/vms/vm_setup_test.go +++ b/tstest/integration/vms/vm_setup_test.go @@ -23,6 +23,7 @@ import ( "strconv" "strings" "testing" + "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" @@ -49,6 +50,19 @@ func (vm *vmInstance) running() bool { } } +func (vm *vmInstance) waitStartup(t *testing.T) { + t.Helper() + for i := 0; i < 100; i++ { + if vm.running() { + break + } + time.Sleep(100 * time.Millisecond) + } + if !vm.running() { + t.Fatal("vm not running") + } +} + func (h *Harness) makeImage(t *testing.T, d Distro, cdir string) string { if !strings.HasPrefix(d.Name, "nixos") { t.Fatal("image generation for non-nixos is not implemented") diff --git a/tstest/integration/vms/vms_test.go b/tstest/integration/vms/vms_test.go index 184e9fb47..d51fc6669 100644 --- a/tstest/integration/vms/vms_test.go +++ b/tstest/integration/vms/vms_test.go @@ -276,17 +276,14 @@ func testOneDistribution(t *testing.T, n int, distro Distro) { t.Cleanup(func() { ramsem.sem.Release(int64(distro.MemoryMegs)) }) vm := h.mkVM(t, n, distro, h.pubKey, h.loginServerURL, dir) - var ipm ipMapping + vm.waitStartup(t) - for i := 0; i < 100; i++ { - if vm.running() { - break - } - time.Sleep(100 * time.Millisecond) - } - if !vm.running() { - t.Fatal("vm not running") - } + h.testDistro(t, distro, h.waitForIPMap(t, vm, distro)) +} + +func (h *Harness) waitForIPMap(t *testing.T, vm *vmInstance, distro Distro) ipMapping { + t.Helper() + var ipm ipMapping waiter := time.NewTicker(time.Second) defer waiter.Stop() @@ -305,13 +302,11 @@ func testOneDistribution(t *testing.T, n int, distro Distro) { } <-waiter.C } - - h.testDistro(t, distro, ipm) + return ipm } -func (h *Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) { +func (h *Harness) setupSSHShell(t *testing.T, d Distro, ipm ipMapping) (*ssh.ClientConfig, *ssh.Client) { signer := h.signer - loginServer := h.loginServerURL t.Helper() port := ipm.port @@ -350,6 +345,13 @@ func (h *Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) { } h.copyBinaries(t, d, cli) + return ccfg, cli +} + +func (h *Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) { + loginServer := h.loginServerURL + ccfg, cli := h.setupSSHShell(t, d, ipm) + timeout := 30 * time.Second t.Run("start-tailscale", func(t *testing.T) {