@ -28,12 +28,16 @@ import (
"time"
expect "github.com/google/goexpect"
"github.com/pkg/sftp"
"golang.org/x/crypto/ssh"
"inet.af/netaddr"
"tailscale.com/tstest"
"tailscale.com/tstest/integration"
"tailscale.com/tstest/integration/testcontrol"
)
const securePassword = "hunter2"
var runVMTests = flag . Bool ( "run-vm-tests" , false , "if set, run expensive (10G+ ram) VM based integration tests" )
type Distro struct {
@ -48,11 +52,23 @@ func (d *Distro) InstallPre() string {
switch d . packageManager {
case "yum" :
return ` - [ yum , update , gnupg2 ]
- [ yum , "-y" , install , iptables ]
`
case "zypper" :
return ` - [ zypper , in , "-y" , iptables ]
`
case "dnf" :
return ` - [ dnf , install , "-y" , iptables ]
`
case "apt" :
return ` - [ apt - get , update ]
- [ apt - get , "-y" , install , curl , "apt-transport-https" , gnupg2 ]
`
case "apk" :
return ` - [ apk, "-U", add, curl, "ca-certificates" ] `
}
return ""
@ -135,8 +151,8 @@ func run(t *testing.T, dir, prog string, args ...string) {
}
}
// mkLayeredQcow makes a layered qcow image that allows us to keep the upstream VM images
// pristine and only do our changes on an overlay.
// mkLayeredQcow makes a layered qcow image that allows us to keep the upstream
// VM images pristine and only do our changes on an overlay.
func mkLayeredQcow ( t * testing . T , tdir string , d Distro ) {
t . Helper ( )
@ -153,7 +169,13 @@ func mkLayeredQcow(t *testing.T, tdir string, d Distro) {
)
}
// mkSeed makes the cloud-init seed ISO that is used to configure a VM with tailscale.
var (
metaDataTempl = template . Must ( template . New ( "meta-data.yaml" ) . Parse ( metaDataTemplate ) )
userDataTempl = template . Must ( template . New ( "user-data.yaml" ) . Parse ( userDataTemplate ) )
)
// mkSeed makes the cloud-init seed ISO that is used to configure a VM with
// tailscale.
func mkSeed ( t * testing . T , d Distro , sshKey , hostURL , tdir string , port int ) {
t . Helper ( )
@ -167,7 +189,7 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
t . Fatal ( err )
}
err = template. Must ( template . New ( "meta-data.yaml" ) . Parse ( metaDataTemplate) ) . Execute ( fout , struct {
err = . Execute ( fout , struct {
ID string
Hostname string
} {
@ -191,18 +213,20 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
t . Fatal ( err )
}
err = template. Must ( template . New ( "user-data.yaml" ) . Parse ( userDataTemplate) ) . Execute ( fout , struct {
err = . Execute ( fout , struct {
SSHKey string
HostURL string
Hostname string
Port int
InstallPre string
Password string
} {
SSHKey : strings . TrimSpace ( sshKey ) ,
HostURL : hostURL ,
Hostname : d . name ,
Port : port ,
InstallPre : d . InstallPre ( ) ,
Password : securePassword ,
} )
if err != nil {
t . Fatal ( err )
@ -222,9 +246,9 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
)
}
// mkVM makes a KVM-accelerated virtual machine and prepares it for introduction to the
// t estcontrol server. The function it returns is for killing the virtual machine when it
// is time for it to die.
// mkVM makes a KVM-accelerated virtual machine and prepares it for introduction
// t o the t estcontrol server. The function it returns is for killing the virtual
// machine when it is time for it to die.
func mkVM ( t * testing . T , n int , d Distro , sshKey , hostURL , tdir string ) func ( ) {
t . Helper ( )
@ -277,12 +301,16 @@ func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) func() {
}
}
// TestVMIntegrationEndToEnd creates a virtual machine with mkvm(1X), installs tailscale on it and then ensures that it connects to the network successfully.
// TestVMIntegrationEndToEnd creates a virtual machine with qemu, installs
// tailscale on it and then ensures that it connects to the network
// successfully.
func TestVMIntegrationEndToEnd ( t * testing . T ) {
if ! * runVMTests {
t . Skip ( "not running integration tests (need -run-vm-tests)" )
}
os . Setenv ( "CGO_ENABLED" , "0" )
if _ , err := exec . LookPath ( "qemu-system-x86_64" ) ; err != nil {
t . Logf ( "hint: nix-shell -p go -p qemu -p cdrkit --run 'go test -v -timeout=60m -run-vm-tests'" )
t . Fatalf ( "missing dependency: %v" , err )
@ -294,13 +322,30 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
}
distros := [ ] Distro {
// NOTE(Xe): If you run into issues getting the autoconfig to work, comment
// out all the other distros and uncomment this one. Connect with a VNC
// client with a command like this:
//
// $ vncviewer :0
//
// On NixOS you can get away with something like this:
//
// $ env NIXPKGS_ALLOW_UNFREE=1 nix-shell -p tigervnc --run 'vncviewer :0'
//
// Login as root with the password root. Then look in
// /var/log/cloud-init-output.log for what you messed up.
// {"alpine-edge", "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-edge-2021-05-18-cloud-init-within.qcow2", "b3bb15311c0bd3beffa1b554f022b75d3b7309b5fdf76fb146fe7c72b83b16d0", 256, "apk"},
// TODO(Xe): This is broken, and I don't know why, see #1988
//{"opensuse-leap-15.1", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2", "3203e256dab5981ca3301408574b63bc522a69972fbe9850b65b54ff44a96e0a", 512, "zypper"},
{ "amazon-linux" , "https://cdn.amazonlinux.com/os-images/2.0.20210427.0/kvm/amzn2-kvm-2.0.20210427.0-x86_64.xfs.gpt.qcow2" , "6ef9daef32cec69b2d0088626ec96410cd24afc504d57278bbf2f2ba2b7e529b" , 512 , "yum" } ,
{ "centos-7" , "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2" , "1db30c9c272fb37b00111b93dcebff16c278384755bdbe158559e9c240b73b80" , 512 , "yum" } ,
{ "centos-8" , "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2" , "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0" , 768 , "dnf" } ,
{ "debian-9" , "https://cdimage.debian.org/cdimage/openstack/9.13.21-20210511/debian-9.13.21-20210511-openstack-amd64.qcow2" , "0667a08e2d947b331aee068db4bbf3a703e03edaf5afa52e23d534adff44b62a" , 512 , "apt" } ,
{ "debian-10" , "https://cdimage.debian.org/images/cloud/buster/20210329-591/debian-10-generic-amd64-20210329-591.qcow2" , "70c61956095870c4082103d1a7a1cb5925293f8405fc6cb348588ec97e8611b0" , 768 , "apt" } ,
{ "fedora-34" , "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2" , "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea" , 768 , "dnf" } ,
{ "opensuse-leap-15.1" , "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2" , "3203e256dab5981ca3301408574b63bc522a69972fbe9850b65b54ff44a96e0a" , 512 , "zypper" } ,
{ "opensuse-leap-15.2" , "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2" , "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f" , 512 , "zypper" } ,
{ "opensuse-tumbleweed" , "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2" , "ba3ecd281045b5019f0fb11378329a644a41870b77631ea647b128cd07eb804b" , 512 , "zypper" } ,
{ "ubuntu-16-04" , "https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img" , "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b" , 512 , "apt" } ,
@ -419,20 +464,39 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
port := port
t . Run ( port , func ( t * testing . T ) {
tstest . FixLogs ( t )
config := & ssh . ClientConfig {
User : "ts" ,
Auth : [ ] ssh . AuthMethod { ssh . PublicKeys ( signer ) , ssh . Password ( "hunter2" ) } ,
HostKeyCallback : ssh . InsecureIgnoreHostKey ( ) ,
t . Parallel ( )
hostport := fmt . Sprintf ( "127.0.0.1:%s" , port )
// NOTE(Xe): This retry loop helps to make things a bit faster, centos sometimes is slow at starting its sshd. I don't know why they don't use socket activation.
const maxRetries = 5
var working bool
for i := 0 ; i < maxRetries ; i ++ {
conn , err := net . Dial ( "tcp" , hostport )
if err == nil {
working = true
conn . Close ( )
break
}
time . Sleep ( 5 * time . Second )
}
cli , err := ssh . Dial ( "tcp" , fmt . Sprintf ( "127.0.0.1:%s" , port ) , config )
if err != nil {
t . Fatalf ( "can't dial 127.0.0.1:%s: %v" , port , err )
if ! working {
t . Fatalf ( "can't connect to %s, tried %d times" , hostport , maxRetries )
}
defer cli . Close ( )
t . Parallel ( )
t . Logf ( "about to ssh into 127.0.0.1:%s" , port )
cli , err := ssh . Dial ( "tcp" , hostport , & ssh . ClientConfig {
User : "root" ,
Auth : [ ] ssh . AuthMethod { ssh . PublicKeys ( signer ) , ssh . Password ( securePassword ) } ,
HostKeyCallback : ssh . InsecureIgnoreHostKey ( ) ,
} )
if err != nil {
t . Fatal ( err )
}
copyBinaries ( t , cli )
timeout := 5 * time . Minute
e , _ , err := expect . SpawnSSH ( cli , timeout , expect . Verbose ( true ) , expect . VerboseWriter ( log . Writer ( ) ) )
@ -441,18 +505,28 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
}
defer e . Close ( )
_ , _ , err = e . Expect ( regexp . MustCompile ( ` (\$|\>) ` ) , timeout )
t . Log ( "opened session" )
_ , _ , err = e . Expect ( regexp . MustCompile ( ` (\#) ` ) , timeout )
if err != nil {
t . Fatalf ( "%s: can't get a shell: %v" , port , err )
}
t . Logf ( "got shell for %s" , port )
err = e . Send ( "systemctl start tailscaled.service\n" )
if err != nil {
t . Fatalf ( "can't send command to start tailscaled: %v" , err )
}
_ , _ , err = e . Expect ( regexp . MustCompile ( ` (\#) ` ) , timeout )
if err != nil {
t . Fatalf ( "%s: can't get a shell: %v" , port , err )
}
err = e . Send ( fmt . Sprintf ( "sudo tailscale up --login-server %s\n" , loginServer ) )
if err != nil {
t . Fatalf ( "%s: can't send tailscale up command: %v" , port , err )
}
_ , _ , err = e . Expect ( regexp . MustCompile ( ` Success. ` ) , timeout )
if err != nil {
t . Fatalf ( "can't extract URL: %v" , err )
t . Fatalf ( " not successful : %v", err )
}
} )
}
@ -463,6 +537,80 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
}
}
func copyBinaries ( t * testing . T , conn * ssh . Client ) {
bins := integration . BuildTestBinaries ( t )
cli , err := sftp . NewClient ( conn )
if err != nil {
t . Fatalf ( "can't connect over sftp to copy binaries: %v" , err )
}
mkdir ( t , cli , "/usr/bin" )
mkdir ( t , cli , "/usr/sbin" )
mkdir ( t , cli , "/etc/systemd/system" )
mkdir ( t , cli , "/etc/default" )
copyFile ( t , cli , bins . Daemon , "/usr/sbin/tailscaled" )
copyFile ( t , cli , bins . CLI , "/usr/bin/tailscale" )
// TODO(Xe): revisit this life decision, hopefully before this assumption
// breaks the test.
copyFile ( t , cli , "../../../cmd/tailscaled/tailscaled.defaults" , "/etc/default/tailscaled" )
copyFile ( t , cli , "../../../cmd/tailscaled/tailscaled.service" , "/etc/systemd/system/tailscaled.service" )
t . Log ( "tailscale installed!" )
}
func mkdir ( t * testing . T , cli * sftp . Client , name string ) {
t . Helper ( )
err := cli . MkdirAll ( name )
if err != nil {
t . Fatalf ( "can't make %s: %v" , name , err )
}
}
func copyFile ( t * testing . T , cli * sftp . Client , localSrc , remoteDest string ) {
t . Helper ( )
fin , err := os . Open ( localSrc )
if err != nil {
t . Fatalf ( "can't open: %v" , err )
}
defer fin . Close ( )
fi , err := fin . Stat ( )
if err != nil {
t . Fatalf ( "can't stat: %v" , err )
}
fout , err := cli . Create ( remoteDest )
if err != nil {
t . Fatalf ( "can't create output file: %v" , err )
}
err = fout . Chmod ( fi . Mode ( ) )
if err != nil {
fout . Close ( )
t . Fatalf ( "can't chmod fout: %v" , err )
}
n , err := io . Copy ( fout , fin )
if err != nil {
fout . Close ( )
t . Fatalf ( "copy failed: %v" , err )
}
if fi . Size ( ) != n {
t . Fatalf ( "incorrect number of bytes copied: wanted: %d, got: %d" , fi . Size ( ) , n )
}
err = fout . Close ( )
if err != nil {
t . Fatalf ( "can't close fout on remote host: %v" , err )
}
}
func deriveBindhost ( t * testing . T ) string {
t . Helper ( )
@ -514,12 +662,15 @@ cloud_final_modules:
- [ scripts - user , once - per - instance ]
users :
- name : ts
plain_text_passwd : hunter2
groups : [ wheel ]
sudo : [ "ALL=(ALL) NOPASSWD:ALL" ]
shell : / bin / sh
ssh - authorized - keys :
- name : root
ssh - authorized - keys :
- { { . SSHKey } }
- name : ts
plain_text_passwd : { { . Password } }
groups : [ wheel ]
sudo : [ "ALL=(ALL) NOPASSWD:ALL" ]
shell : / bin / sh
ssh - authorized - keys :
- { { . SSHKey } }
write_files :
@ -531,7 +682,5 @@ write_files:
runcmd :
{ { . InstallPre } }
- [ "sh" , "-c" , "curl https://raw.githubusercontent.com/tailscale/tailscale/Xe/test-install-script-libvirtd/scripts/installer.sh | sh" ]
- [ systemctl , enable , -- now , tailscaled . service ]
- [ curl , "{{.HostURL}}/myip/{{.Port}}" , "-H" , "User-Agent: {{.Hostname}}" ]
`
Christine Dodrill
4 years ago
committed by