diff --git a/go.mod b/go.mod index fc265a03e..6655a0437 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/aws/aws-sdk-go v1.38.52 github.com/coreos/go-iptables v0.6.0 + github.com/dave/jennifer v1.4.1 // indirect github.com/frankban/quicktest v1.13.0 github.com/gliderlabs/ssh v0.3.2 github.com/go-multierror/multierror v1.0.2 @@ -16,6 +17,7 @@ require ( github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f github.com/google/uuid v1.1.2 github.com/goreleaser/nfpm v1.10.3 + github.com/iancoleman/strcase v0.2.0 // indirect github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/klauspost/compress v1.12.2 @@ -30,6 +32,7 @@ require ( github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027 github.com/tailscale/goupnp v1.0.1-0.20210710010003-1cf2d718bbb2 // indirect + github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2 // indirect github.com/tcnksm/go-httpstat v0.2.0 github.com/toqueteos/webbrowser v1.2.0 go4.org/mem v0.0.0-20201119185036-c04c5a6ff174 diff --git a/go.sum b/go.sum index 839b404ca..f1ca9668d 100644 --- a/go.sum +++ b/go.sum @@ -88,6 +88,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/daixiang0/gci v0.2.4/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD/hfR4= github.com/daixiang0/gci v0.2.7 h1:bosLNficubzJZICsVzxuyNc6oAbdz0zcqLG2G/RxtY4= github.com/daixiang0/gci v0.2.7/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc= +github.com/dave/jennifer v1.4.1 h1:XyqG6cn5RQsTj3qlWQTKlRGAyrTcsk1kUmWdZBzRjDw= +github.com/dave/jennifer v1.4.1/go.mod h1:7jEdnm+qBcxl8PC0zyp7vxcpSRnzXSt9r39tpTVGlwA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -296,6 +298,8 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -586,6 +590,8 @@ github.com/tailscale/goupnp v1.0.1-0.20210629175715-39c5a55db683 h1:ZXmZQuVebYll github.com/tailscale/goupnp v1.0.1-0.20210629175715-39c5a55db683/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8= github.com/tailscale/goupnp v1.0.1-0.20210710010003-1cf2d718bbb2 h1:AIJ8AF9O7jBmCwilP0ydwJMIzW5dw48Us8f3hLJhYBY= github.com/tailscale/goupnp v1.0.1-0.20210710010003-1cf2d718bbb2/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8= +github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2 h1:reREUgl2FG+o7YCsrZB8XLjnuKv5hEIWtnOdAbRAXZI= +github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2/go.mod h1:STqf+YV0ADdzk4ejtXFsGqDpATP9JoL0OB+hiFQbkdE= github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0= github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8= github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= diff --git a/tstest/integration/vms/distros.go b/tstest/integration/vms/distros.go index bf5d4cb69..cc9dffb3e 100644 --- a/tstest/integration/vms/distros.go +++ b/tstest/integration/vms/distros.go @@ -2,21 +2,28 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build linux - package vms +import ( + _ "embed" + "log" + + "github.com/tailscale/hujson" +) + +// go:generate go run ./gen + type Distro struct { - name string // amazon-linux - url string // URL to a qcow2 image - sha256sum string // hex-encoded sha256 sum of contents of URL - mem int // VM memory in megabytes - packageManager string // yum/apt/dnf/zypper - initSystem string // systemd/openrc + Name string // amazon-linux + URL string // URL to a qcow2 image + SHA256Sum string // hex-encoded sha256 sum of contents of URL + MemoryMegs int // VM memory in megabytes + PackageManager string // yum/apt/dnf/zypper + InitSystem string // systemd/openrc } func (d *Distro) InstallPre() string { - switch d.packageManager { + switch d.PackageManager { case "yum": return ` - [ yum, update, gnupg2 ] - [ yum, "-y", install, iptables ]` @@ -38,58 +45,15 @@ func (d *Distro) InstallPre() string { return "" } -var distros = []Distro{ - // NOTE(Xe): If you run into issues getting the autoconfig to work, run - // this test with the flag `--distro-regex=alpine-edge`. Connect with a VNC - // client with a command like this: - // - // $ vncviewer :0 - // - // On NixOS you can get away with something like this: - // - // $ env NIXPKGS_ALLOW_UNFREE=1 nix-shell -p tigervnc --run 'vncviewer :0' - // - // Login as root with the password root. Then look in - // /var/log/cloud-init-output.log for what you messed up. +//go:embed distros.hujson +var distroData string - // NOTE(Xe): These images are not official images created by the Alpine Linux - // cloud team because the cloud team hasn't created any official images yet. - // These images were created under the guidance of the cloud team and contain - // few notable differences from what they would end up shipping. The Alpine - // Linux cloud team probably won't have official images up until a year or so - // after this comment is written (2021-06-11), but overall they will be - // compatible with these images. These images were created using the setup in - // this repo: https://github.com/Xe/alpine-image. I hereby promise to not break - // these links. - {"alpine-3-13-5", "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-3.13.5-cloud-init-within.qcow2", "a2665c16724e75899723e81d81126bd0254a876e5de286b0b21553734baec287", 256, "apk", "openrc"}, - {"alpine-edge", "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-edge-2021-05-18-cloud-init-within.qcow2", "b3bb15311c0bd3beffa1b554f022b75d3b7309b5fdf76fb146fe7c72b83b16d0", 256, "apk", "openrc"}, - - // NOTE(Xe): All of the following images are official images straight from each - // distribution's official documentation. - {"amazon-linux", "https://cdn.amazonlinux.com/os-images/2.0.20210427.0/kvm/amzn2-kvm-2.0.20210427.0-x86_64.xfs.gpt.qcow2", "6ef9daef32cec69b2d0088626ec96410cd24afc504d57278bbf2f2ba2b7e529b", 512, "yum", "systemd"}, - {"arch", "https://mirror.pkgbuild.com/images/v20210515.22945/Arch-Linux-x86_64-cloudimg-20210515.22945.qcow2", "e4077f5ba3c5d545478f64834bc4852f9f7a2e05950fce8ecd0df84193162a27", 512, "pacman", "systemd"}, - {"centos-7", "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2003.qcow2c", "b7555ecf90b24111f2efbc03c1e80f7b38f1e1fc7e1b15d8fee277d1a4575e87", 512, "yum", "systemd"}, - {"centos-8", "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2", "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0", 768, "dnf", "systemd"}, - {"debian-9", "http://cloud.debian.org/images/cloud/OpenStack/9.13.22-20210531/debian-9.13.22-20210531-openstack-amd64.qcow2", "c36e25f2ab0b5be722180db42ed9928476812f02d053620e1c287f983e9f6f1d", 512, "apt", "systemd"}, - {"debian-10", "https://cdimage.debian.org/images/cloud/buster/20210329-591/debian-10-generic-amd64-20210329-591.qcow2", "70c61956095870c4082103d1a7a1cb5925293f8405fc6cb348588ec97e8611b0", 768, "apt", "systemd"}, - {"fedora-34", "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2", "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea", 768, "dnf", "systemd"}, - {"opensuse-leap-15-1", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2", "40bc72b8ee143364fc401f2c9c9a11ecb7341a29fa84c6f7bf42fc94acf19a02", 512, "zypper", "systemd"}, - {"opensuse-leap-15-2", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2", "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f", 512, "zypper", "systemd"}, - {"opensuse-leap-15-3", "http://mirror.its.dal.ca/opensuse/distribution/leap/15.3/appliances/openSUSE-Leap-15.3-JeOS.x86_64-OpenStack-Cloud.qcow2", "22e0392e4d0becb523d1bc5f709366140b7ee20d6faf26de3d0f9046d1ee15d5", 512, "zypper", "systemd"}, - {"opensuse-tumbleweed", "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2", "79e610bba3ed116556608f031c06e4b9260e3be2b193ce1727914ba213afac3f", 512, "zypper", "systemd"}, - {"oracle-linux-7", "https://yum.oracle.com/templates/OracleLinux/OL7/u9/x86_64/OL7U9_x86_64-olvm-b86.qcow2", "2ef4c10c0f6a0b17844742adc9ede7eb64a2c326e374068b7175f2ecbb1956fb", 512, "yum", "systemd"}, - {"oracle-linux-8", "https://yum.oracle.com/templates/OracleLinux/OL8/u4/x86_64/OL8U4_x86_64-olvm-b85.qcow2", "b86e1f1ea8fc904ed763a85ba12e9f12f4291c019c8435d0e4e6133392182b0b", 768, "dnf", "systemd"}, - {"ubuntu-16-04", "https://cloud-images.ubuntu.com/xenial/20210429/xenial-server-cloudimg-amd64-disk1.img", "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b", 512, "apt", "systemd"}, - {"ubuntu-18-04", "https://cloud-images.ubuntu.com/bionic/20210526/bionic-server-cloudimg-amd64.img", "389ffd5d36bbc7a11bf384fd217cda9388ccae20e5b0cb7d4516733623c96022", 512, "apt", "systemd"}, - {"ubuntu-20-04", "https://cloud-images.ubuntu.com/focal/20210603/focal-server-cloudimg-amd64.img", "1c0969323b058ba8b91fec245527069c2f0502fc119b9138b213b6bfebd965cb", 512, "apt", "systemd"}, - {"ubuntu-20-10", "https://cloud-images.ubuntu.com/groovy/20210604/groovy-server-cloudimg-amd64.img", "2196df5f153faf96443e5502bfdbcaa0baaefbaec614348fec344a241855b0ef", 512, "apt", "systemd"}, - {"ubuntu-21-04", "https://cloud-images.ubuntu.com/hirsute/20210603/hirsute-server-cloudimg-amd64.img", "bf07f36fc99ff521d3426e7d257e28f0c81feebc9780b0c4f4e25ae594ff4d3b", 512, "apt", "systemd"}, +var Distros []Distro = func() []Distro { + var result []Distro + err := hujson.Unmarshal([]byte(distroData), &result) + if err != nil { + log.Fatalf("error decoding distros: %v", err) + } - // NOTE(Xe): We build fresh NixOS images for every test run, so the URL being - // used here is actually the URL of the NixOS channel being built from and the - // shasum is meaningless. This `channel:name` syntax is documented at [1]. - // - // [1]: https://nixos.org/manual/nix/unstable/command-ref/env-common.html - {"nixos-21-05", "channel:nixos-21.05", "lolfakesha", 512, "nix", "systemd"}, - {"nixos-unstable", "channel:nixos-unstable", "lolfakesha", 512, "nix", "systemd"}, -} + return result +}() diff --git a/tstest/integration/vms/distros.hujson b/tstest/integration/vms/distros.hujson new file mode 100644 index 000000000..a2a6ca780 --- /dev/null +++ b/tstest/integration/vms/distros.hujson @@ -0,0 +1,208 @@ +// NOTE(Xe): If you run into issues getting the autoconfig to work, run +// this test with the flag `--distro-regex=alpine-edge`. Connect with a VNC +// client with a command like this: +// +// $ vncviewer :0 +// +// On NixOS you can get away with something like this: +// +// $ env NIXPKGS_ALLOW_UNFREE=1 nix-shell -p tigervnc --run 'vncviewer :0' +// +// Login as root with the password root. Then look in +// /var/log/cloud-init-output.log for what you messed up. +[ + // NOTE(Xe): These images are not official images created by the Alpine Linux + // cloud team because the cloud team hasn't created any official images yet. + // These images were created under the guidance of the cloud team and contain + // few notable differences from what they would end up shipping. The Alpine + // Linux cloud team probably won't have official images up until a year or so + // after this comment is written (2021-06-11), but overall they will be + // compatible with these images. These images were created using the setup in + // this repo: https://github.com/Xe/alpine-image. I hereby promise to not break + // these links. + { + "Name": "alpine-3-13-5", + "URL": "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-3.13.5-cloud-init-within.qcow2", + "SHA256Sum": "a2665c16724e75899723e81d81126bd0254a876e5de286b0b21553734baec287", + "MemoryMegs": 256, + "PackageManager": "apk", + "InitSystem": "openrc" + }, + { + "Name": "alpine-edge", + "URL": "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-edge-2021-05-18-cloud-init-within.qcow2", + "SHA256Sum": "b3bb15311c0bd3beffa1b554f022b75d3b7309b5fdf76fb146fe7c72b83b16d0", + "MemoryMegs": 256, + "PackageManager": "apk", + "InitSystem": "openrc" + }, + + // NOTE(Xe): All of the following images are official images straight from each + // distribution's official documentation. + { + "Name": "amazon-linux", + "URL": "https://cdn.amazonlinux.com/os-images/2.0.20210427.0/kvm/amzn2-kvm-2.0.20210427.0-x86_64.xfs.gpt.qcow2", + "SHA256Sum": "6ef9daef32cec69b2d0088626ec96410cd24afc504d57278bbf2f2ba2b7e529b", + "MemoryMegs": 512, + "PackageManager": "yum", + "InitSystem": "systemd" + }, + { + "Name": "arch", + "URL": "https://mirror.pkgbuild.com/images/v20210515.22945/Arch-Linux-x86_64-cloudimg-20210515.22945.qcow2", + "SHA256Sum": "e4077f5ba3c5d545478f64834bc4852f9f7a2e05950fce8ecd0df84193162a27", + "MemoryMegs": 512, + "PackageManager": "pacman", + "InitSystem": "systemd" + }, + { + "Name": "centos-7", + "URL": "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2003.qcow2c", + "SHA256Sum": "b7555ecf90b24111f2efbc03c1e80f7b38f1e1fc7e1b15d8fee277d1a4575e87", + "MemoryMegs": 512, + "PackageManager": "yum", + "InitSystem": "systemd" + }, + { + "Name": "centos-8", + "URL": "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2", + "SHA256Sum": "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0", + "MemoryMegs": 768, + "PackageManager": "dnf", + "InitSystem": "systemd" + }, + { + "Name": "debian-9", + "URL": "http://cloud.debian.org/images/cloud/OpenStack/9.13.22-20210531/debian-9.13.22-20210531-openstack-amd64.qcow2", + "SHA256Sum": "c36e25f2ab0b5be722180db42ed9928476812f02d053620e1c287f983e9f6f1d", + "MemoryMegs": 512, + "PackageManager": "apt", + "InitSystem": "systemd" + }, + { + "Name": "debian-10", + "URL": "https://cdimage.debian.org/images/cloud/buster/20210329-591/debian-10-generic-amd64-20210329-591.qcow2", + "SHA256Sum": "70c61956095870c4082103d1a7a1cb5925293f8405fc6cb348588ec97e8611b0", + "MemoryMegs": 768, + "PackageManager": "apt", + "InitSystem": "systemd" + }, + { + "Name": "fedora-34", + "URL": "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2", + "SHA256Sum": "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea", + "MemoryMegs": 768, + "PackageManager": "dnf", + "InitSystem": "systemd" + }, + { + "Name": "opensuse-leap-15-1", + "URL": "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2", + "SHA256Sum": "40bc72b8ee143364fc401f2c9c9a11ecb7341a29fa84c6f7bf42fc94acf19a02", + "MemoryMegs": 512, + "PackageManager": "zypper", + "InitSystem": "systemd" + }, + { + "Name": "opensuse-leap-15-2", + "URL": "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2", + "SHA256Sum": "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f", + "MemoryMegs": 512, + "PackageManager": "zypper", + "InitSystem": "systemd" + }, + { + "Name": "opensuse-leap-15-3", + "URL": "http://mirror.its.dal.ca/opensuse/distribution/leap/15.3/appliances/openSUSE-Leap-15.3-JeOS.x86_64-OpenStack-Cloud.qcow2", + "SHA256Sum": "22e0392e4d0becb523d1bc5f709366140b7ee20d6faf26de3d0f9046d1ee15d5", + "MemoryMegs": 512, + "PackageManager": "zypper", + "InitSystem": "systemd" + }, + { + "Name": "opensuse-tumbleweed", + "URL": "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2", + "SHA256Sum": "79e610bba3ed116556608f031c06e4b9260e3be2b193ce1727914ba213afac3f", + "MemoryMegs": 512, + "PackageManager": "zypper", + "InitSystem": "systemd" + }, + { + "Name": "oracle-linux-7", + "URL": "https://yum.oracle.com/templates/OracleLinux/OL7/u9/x86_64/OL7U9_x86_64-olvm-b86.qcow2", + "SHA256Sum": "2ef4c10c0f6a0b17844742adc9ede7eb64a2c326e374068b7175f2ecbb1956fb", + "MemoryMegs": 512, + "PackageManager": "yum", + "InitSystem": "systemd" + }, + { + "Name": "oracle-linux-8", + "URL": "https://yum.oracle.com/templates/OracleLinux/OL8/u4/x86_64/OL8U4_x86_64-olvm-b85.qcow2", + "SHA256Sum": "b86e1f1ea8fc904ed763a85ba12e9f12f4291c019c8435d0e4e6133392182b0b", + "MemoryMegs": 768, + "PackageManager": "dnf", + "InitSystem": "systemd" + }, + { + "Name": "ubuntu-16-04", + "URL": "https://cloud-images.ubuntu.com/xenial/20210429/xenial-server-cloudimg-amd64-disk1.img", + "SHA256Sum": "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b", + "MemoryMegs": 512, + "PackageManager": "apt", + "InitSystem": "systemd" + }, + { + "Name": "ubuntu-18-04", + "URL": "https://cloud-images.ubuntu.com/bionic/20210526/bionic-server-cloudimg-amd64.img", + "SHA256Sum": "389ffd5d36bbc7a11bf384fd217cda9388ccae20e5b0cb7d4516733623c96022", + "MemoryMegs": 512, + "PackageManager": "apt", + "InitSystem": "systemd" + }, + { + "Name": "ubuntu-20-04", + "URL": "https://cloud-images.ubuntu.com/focal/20210603/focal-server-cloudimg-amd64.img", + "SHA256Sum": "1c0969323b058ba8b91fec245527069c2f0502fc119b9138b213b6bfebd965cb", + "MemoryMegs": 512, + "PackageManager": "apt", + "InitSystem": "systemd" + }, + { + "Name": "ubuntu-20-10", + "URL": "https://cloud-images.ubuntu.com/groovy/20210604/groovy-server-cloudimg-amd64.img", + "SHA256Sum": "2196df5f153faf96443e5502bfdbcaa0baaefbaec614348fec344a241855b0ef", + "MemoryMegs": 512, + "PackageManager": "apt", + "InitSystem": "systemd" + }, + { + "Name": "ubuntu-21-04", + "URL": "https://cloud-images.ubuntu.com/hirsute/20210603/hirsute-server-cloudimg-amd64.img", + "SHA256Sum": "bf07f36fc99ff521d3426e7d257e28f0c81feebc9780b0c4f4e25ae594ff4d3b", + "MemoryMegs": 512, + "PackageManager": "apt", + "InitSystem": "systemd" + }, + + // NOTE(Xe): We build fresh NixOS images for every test run, so the URL being + // used here is actually the URL of the NixOS channel being built from and the + // shasum is meaningless. This `channel:name` syntax is documented at [1]. + // + // [1]: https://nixos.org/manual/nix/unstable/command-ref/env-common.html + { + "Name": "nixos-21-05", + "URL": "channel:nixos-21.05", + "SHA256Sum": "lolfakesha", + "MemoryMegs": 512, + "PackageManager": "nix", + "InitSystem": "systemd" + }, + { + "Name": "nixos-unstable", + "URL": "channel:nixos-unstable", + "SHA256Sum": "lolfakesha", + "MemoryMegs": 512, + "PackageManager": "nix", + "InitSystem": "systemd" + } +] diff --git a/tstest/integration/vms/distros_test.go b/tstest/integration/vms/distros_test.go new file mode 100644 index 000000000..c2a969d36 --- /dev/null +++ b/tstest/integration/vms/distros_test.go @@ -0,0 +1,17 @@ +// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package vms + +import ( + "testing" +) + +func TestDistrosGotLoaded(t *testing.T) { + if len(Distros) == 0 { + t.Fatal("no distros were loaded") + } +} diff --git a/tstest/integration/vms/gen/test_codegen.go b/tstest/integration/vms/gen/test_codegen.go new file mode 100644 index 000000000..057cfccbf --- /dev/null +++ b/tstest/integration/vms/gen/test_codegen.go @@ -0,0 +1,55 @@ +// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// build ignore + +package main + +import ( + _ "embed" + "fmt" + "log" + "os" + "time" + + "github.com/dave/jennifer/jen" + "github.com/iancoleman/strcase" + "tailscale.com/tstest/integration/vms" +) + +func main() { + f := jen.NewFile("vms") + f.Comment("Code generated by tstest/integration/vms/gen/test_codegen.go DO NOT EDIT.") + + ptr := jen.Op("*") + + for i, d := range vms.Distros { + f.Func(). + Id("TestRun" + strcase.ToCamel(d.Name)). + Params(jen.Id("t").Add(ptr).Qual("testing", "T")). + BlockFunc(func(g *jen.Group) { + g.Id("t").Dot("Parallel").Call() + g.Id("setupTests").Call(jen.Id("t")) + g.Id("testOneDistribution").Call(jen.Id("t"), jen.Lit(i), jen.Id("Distros").Index(jen.Lit(i))) + }) + } + + os.Remove("top_level_test.go") + fout, err := os.Create("top_level_test.go") + if err != nil { + log.Fatal(err) + } + defer fout.Close() + + fmt.Fprintf(fout, "// Copyright (c) %d Tailscale Inc & AUTHORS All rights reserved.\n", time.Now().Year()) + fout.WriteString("// Use of this source code is governed by a BSD-style\n") + fout.WriteString("// license that can be found in the LICENSE file.\n") + fout.WriteString("\n") + fout.WriteString("// +build linux\n\n") + + err = f.Render(fout) + if err != nil { + log.Fatal(err) + } +} diff --git a/tstest/integration/vms/nixos_test.go b/tstest/integration/vms/nixos_test.go index cc13498f3..1e93035fd 100644 --- a/tstest/integration/vms/nixos_test.go +++ b/tstest/integration/vms/nixos_test.go @@ -169,7 +169,7 @@ func copyUnit(t *testing.T, bins *integration.Binaries) { func (h *Harness) makeNixOSImage(t *testing.T, d Distro, cdir string) string { copyUnit(t, h.bins) dir := t.TempDir() - fname := filepath.Join(dir, d.name+".nix") + fname := filepath.Join(dir, d.Name+".nix") fout, err := os.Create(fname) if err != nil { t.Fatal(err) @@ -196,10 +196,10 @@ func (h *Harness) makeNixOSImage(t *testing.T, d Distro, cdir string) string { os.MkdirAll(outpath, 0755) t.Cleanup(func() { - os.RemoveAll(filepath.Join(outpath, d.name)) // makes the disk image a candidate for GC + os.RemoveAll(filepath.Join(outpath, d.Name)) // makes the disk image a candidate for GC }) - cmd := exec.Command("nixos-generate", "-f", "qcow", "-o", filepath.Join(outpath, d.name), "-c", fname) + cmd := exec.Command("nixos-generate", "-f", "qcow", "-o", filepath.Join(outpath, d.Name), "-c", fname) if *verboseNixOutput { cmd.Stdout = logger.FuncWriter(t.Logf) cmd.Stderr = logger.FuncWriter(t.Logf) @@ -214,16 +214,16 @@ func (h *Harness) makeNixOSImage(t *testing.T, d Distro, cdir string) string { cmd.Stderr = fout defer fout.Close() } - cmd.Env = append(os.Environ(), "NIX_PATH=nixpkgs="+d.url) + cmd.Env = append(os.Environ(), "NIX_PATH=nixpkgs="+d.URL) cmd.Dir = outpath t.Logf("running %s %#v", "nixos-generate", cmd.Args) if err := cmd.Run(); err != nil { - t.Fatalf("error while making NixOS image for %s: %v", d.name, err) + t.Fatalf("error while making NixOS image for %s: %v", d.Name, err) } if !*verboseNixOutput { t.Log("done") } - return filepath.Join(outpath, d.name, "nixos.qcow2") + return filepath.Join(outpath, d.Name, "nixos.qcow2") } diff --git a/tstest/integration/vms/opensuse_leap_15_1_test.go b/tstest/integration/vms/opensuse_leap_15_1_test.go index 4a5bfe41a..e26440434 100644 --- a/tstest/integration/vms/opensuse_leap_15_1_test.go +++ b/tstest/integration/vms/opensuse_leap_15_1_test.go @@ -41,7 +41,7 @@ type openSUSELeap151MetaDataMeta struct { } func hackOpenSUSE151UserData(t *testing.T, d Distro, dir string) bool { - if d.name != "opensuse-leap-15-1" { + if d.Name != "opensuse-leap-15-1" { return false } @@ -54,14 +54,14 @@ func hackOpenSUSE151UserData(t *testing.T, d Distro, dir string) bool { metadata, err := json.Marshal(openSUSELeap151MetaData{ Zone: "nova", - Hostname: d.name, + Hostname: d.Name, LaunchIndex: "0", Meta: openSUSELeap151MetaDataMeta{ Role: "server", DSMode: "local", Essential: "false", }, - Name: d.name, + Name: d.Name, UUID: uuid.New().String(), }) if err != nil { diff --git a/tstest/integration/vms/top_level_test.go b/tstest/integration/vms/top_level_test.go new file mode 100644 index 000000000..d69ed3378 --- /dev/null +++ b/tstest/integration/vms/top_level_test.go @@ -0,0 +1,121 @@ +// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package vms + +import "testing" + +// Code generated by tstest/integration/vms/gen/test_codegen.go DO NOT EDIT. +func TestRunAlpine3135(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 0, Distros[0]) +} +func TestRunAlpineEdge(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 1, Distros[1]) +} +func TestRunAmazonLinux(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 2, Distros[2]) +} +func TestRunArch(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 3, Distros[3]) +} +func TestRunCentos7(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 4, Distros[4]) +} +func TestRunCentos8(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 5, Distros[5]) +} +func TestRunDebian9(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 6, Distros[6]) +} +func TestRunDebian10(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 7, Distros[7]) +} +func TestRunFedora34(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 8, Distros[8]) +} +func TestRunOpensuseLeap151(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 9, Distros[9]) +} +func TestRunOpensuseLeap152(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 10, Distros[10]) +} +func TestRunOpensuseLeap153(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 11, Distros[11]) +} +func TestRunOpensuseTumbleweed(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 12, Distros[12]) +} +func TestRunOracleLinux7(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 13, Distros[13]) +} +func TestRunOracleLinux8(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 14, Distros[14]) +} +func TestRunUbuntu1604(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 15, Distros[15]) +} +func TestRunUbuntu1804(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 16, Distros[16]) +} +func TestRunUbuntu2004(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 17, Distros[17]) +} +func TestRunUbuntu2010(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 18, Distros[18]) +} +func TestRunUbuntu2104(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 19, Distros[19]) +} +func TestRunNixos2105(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 20, Distros[20]) +} +func TestRunNixosUnstable(t *testing.T) { + t.Parallel() + setupTests(t) + testOneDistribution(t, 21, Distros[21]) +} diff --git a/tstest/integration/vms/vm_setup_test.go b/tstest/integration/vms/vm_setup_test.go index 121bfad88..d5aa0f1a1 100644 --- a/tstest/integration/vms/vm_setup_test.go +++ b/tstest/integration/vms/vm_setup_test.go @@ -53,17 +53,17 @@ func (h *Harness) mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir stri mkLayeredQcow(t, tdir, d, h.fetchDistro(t, d)) mkSeed(t, d, sshKey, hostURL, tdir, port) - driveArg := fmt.Sprintf("file=%s,if=virtio", filepath.Join(tdir, d.name+".qcow2")) + driveArg := fmt.Sprintf("file=%s,if=virtio", filepath.Join(tdir, d.Name+".qcow2")) args := []string{ "-machine", "pc-q35-5.1,accel=kvm,usb=off,vmport=off,dump-guest-core=off", "-netdev", fmt.Sprintf("user,hostfwd=::%d-:22,id=net0", port), "-device", "virtio-net-pci,netdev=net0,id=net0,mac=8a:28:5c:30:1f:25", - "-m", fmt.Sprint(d.mem), + "-m", fmt.Sprint(d.MemoryMegs), "-boot", "c", "-drive", driveArg, - "-cdrom", filepath.Join(tdir, d.name, "seed", "seed.iso"), - "-smbios", "type=1,serial=ds=nocloud;h=" + d.name, + "-cdrom", filepath.Join(tdir, d.Name, "seed", "seed.iso"), + "-smbios", "type=1,serial=ds=nocloud;h=" + d.Name, } if *useVNC { @@ -101,7 +101,7 @@ func (h *Harness) mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir stri t.Cleanup(func() { err := cmd.Process.Kill() if err != nil { - t.Errorf("can't kill %s (%d): %v", d.name, cmd.Process.Pid, err) + t.Errorf("can't kill %s (%d): %v", d.Name, cmd.Process.Pid, err) } cmd.Wait() @@ -139,15 +139,15 @@ func fetchFromS3(t *testing.T, fout *os.File, d Distro) bool { d.PartSize = 64 * 1024 * 1024 // 64MB per part }) - t.Logf("fetching s3://%s/%s", bucketName, d.sha256sum) + t.Logf("fetching s3://%s/%s", bucketName, d.SHA256Sum) _, err = dler.Download(fout, &s3.GetObjectInput{ Bucket: aws.String(bucketName), - Key: aws.String(d.sha256sum), + Key: aws.String(d.SHA256Sum), }) if err != nil { fout.Close() - t.Fatalf("can't get s3://%s/%s: %v", bucketName, d.sha256sum, err) + t.Fatalf("can't get s3://%s/%s: %v", bucketName, d.SHA256Sum, err) } err = fout.Close() @@ -169,17 +169,17 @@ func (h *Harness) fetchDistro(t *testing.T, resultDistro Distro) string { } cdir = filepath.Join(cdir, "tailscale", "vm-test") - if strings.HasPrefix(resultDistro.name, "nixos") { + if strings.HasPrefix(resultDistro.Name, "nixos") { return h.makeNixOSImage(t, resultDistro, cdir) } - qcowPath := filepath.Join(cdir, "qcow2", resultDistro.sha256sum) + qcowPath := filepath.Join(cdir, "qcow2", resultDistro.SHA256Sum) _, err = os.Stat(qcowPath) if err == nil { hash := checkCachedImageHash(t, resultDistro, cdir) - if hash != resultDistro.sha256sum { - t.Logf("hash for %s (%s) doesn't match expected %s, re-downloading", resultDistro.name, qcowPath, resultDistro.sha256sum) + if hash != resultDistro.SHA256Sum { + t.Logf("hash for %s (%s) doesn't match expected %s, re-downloading", resultDistro.Name, qcowPath, resultDistro.SHA256Sum) err = errors.New("some fake non-nil error to force a redownload") if err := os.Remove(qcowPath); err != nil { @@ -189,26 +189,26 @@ func (h *Harness) fetchDistro(t *testing.T, resultDistro Distro) string { } if err != nil { - t.Logf("downloading distro image %s to %s", resultDistro.url, qcowPath) + t.Logf("downloading distro image %s to %s", resultDistro.URL, qcowPath) fout, err := os.Create(qcowPath) if err != nil { t.Fatal(err) } if !fetchFromS3(t, fout, resultDistro) { - resp, err := http.Get(resultDistro.url) + resp, err := http.Get(resultDistro.URL) if err != nil { - t.Fatalf("can't fetch qcow2 for %s (%s): %v", resultDistro.name, resultDistro.url, err) + t.Fatalf("can't fetch qcow2 for %s (%s): %v", resultDistro.Name, resultDistro.URL, err) } if resp.StatusCode != http.StatusOK { resp.Body.Close() - t.Fatalf("%s replied %s", resultDistro.url, resp.Status) + t.Fatalf("%s replied %s", resultDistro.URL, resp.Status) } _, err = io.Copy(fout, resp.Body) if err != nil { - t.Fatalf("download of %s failed: %v", resultDistro.url, err) + t.Fatalf("download of %s failed: %v", resultDistro.URL, err) } resp.Body.Close() @@ -219,8 +219,8 @@ func (h *Harness) fetchDistro(t *testing.T, resultDistro Distro) string { hash := checkCachedImageHash(t, resultDistro, cdir) - if hash != resultDistro.sha256sum { - t.Fatalf("hash mismatch, want: %s, got: %s", resultDistro.sha256sum, hash) + if hash != resultDistro.SHA256Sum { + t.Fatalf("hash mismatch, want: %s, got: %s", resultDistro.SHA256Sum, hash) } } } @@ -231,7 +231,7 @@ func (h *Harness) fetchDistro(t *testing.T, resultDistro Distro) string { func checkCachedImageHash(t *testing.T, d Distro, cacheDir string) (gotHash string) { t.Helper() - qcowPath := filepath.Join(cacheDir, "qcow2", d.sha256sum) + qcowPath := filepath.Join(cacheDir, "qcow2", d.SHA256Sum) fin, err := os.Open(qcowPath) if err != nil { @@ -244,8 +244,8 @@ func checkCachedImageHash(t *testing.T, d Distro, cacheDir string) (gotHash stri } hash := hex.EncodeToString(hasher.Sum(nil)) - if hash != d.sha256sum { - t.Fatalf("hash mismatch, got: %q, want: %q", hash, d.sha256sum) + if hash != d.SHA256Sum { + t.Fatalf("hash mismatch, got: %q, want: %q", hash, d.SHA256Sum) } gotHash = hash @@ -255,7 +255,7 @@ func checkCachedImageHash(t *testing.T, d Distro, cacheDir string) (gotHash stri func (h *Harness) copyBinaries(t *testing.T, d Distro, conn *ssh.Client) { bins := h.bins - if strings.HasPrefix(d.name, "nixos") { + if strings.HasPrefix(d.Name, "nixos") { return } @@ -275,7 +275,7 @@ func (h *Harness) copyBinaries(t *testing.T, d Distro, conn *ssh.Client) { // TODO(Xe): revisit this assumption before it breaks the test. copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.defaults", "/etc/default/tailscaled") - switch d.initSystem { + switch d.InitSystem { case "openrc": mkdir(t, cli, "/etc/init.d") copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.openrc", "/etc/init.d/tailscaled") diff --git a/tstest/integration/vms/vms_test.go b/tstest/integration/vms/vms_test.go index e1d6c8904..689cd5f8e 100644 --- a/tstest/integration/vms/vms_test.go +++ b/tstest/integration/vms/vms_test.go @@ -18,6 +18,7 @@ import ( "regexp" "strconv" "strings" + "sync" "testing" "text/template" "time" @@ -57,14 +58,14 @@ func TestDownloadImages(t *testing.T) { bins := integration.BuildTestBinaries(t) - for _, d := range distros { + for _, d := range Distros { distro := d - t.Run(distro.name, func(t *testing.T) { - if !distroRex.Unwrap().MatchString(distro.name) { - t.Skipf("distro name %q doesn't match regex: %s", distro.name, distroRex) + t.Run(distro.Name, func(t *testing.T) { + if !distroRex.Unwrap().MatchString(distro.Name) { + t.Skipf("distro name %q doesn't match regex: %s", distro.Name, distroRex) } - if strings.HasPrefix(distro.name, "nixos") { + if strings.HasPrefix(distro.Name, "nixos") { t.Skip("NixOS is built on the fly, no need to download it") } @@ -98,7 +99,7 @@ func mkLayeredQcow(t *testing.T, tdir string, d Distro, qcowBase string) { run(t, tdir, "qemu-img", "create", "-f", "qcow2", "-o", "backing_file="+qcowBase, - filepath.Join(tdir, d.name+".qcow2"), + filepath.Join(tdir, d.Name+".qcow2"), ) } @@ -112,7 +113,7 @@ var ( func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) { t.Helper() - dir := filepath.Join(tdir, d.name, "seed") + dir := filepath.Join(tdir, d.Name, "seed") os.MkdirAll(dir, 0700) // make meta-data @@ -127,7 +128,7 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) { Hostname string }{ ID: "31337", - Hostname: d.name, + Hostname: d.Name, }) if err != nil { t.Fatal(err) @@ -156,7 +157,7 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) { }{ SSHKey: strings.TrimSpace(sshKey), HostURL: hostURL, - Hostname: d.name, + Hostname: d.Name, Port: port, InstallPre: d.InstallPre(), Password: securePassword, @@ -220,10 +221,11 @@ func getProbablyFreePortNumber() (int, error) { return portNum, nil } -// TestVMIntegrationEndToEnd creates a virtual machine with qemu, installs -// tailscale on it and then ensures that it connects to the network -// successfully. -func TestVMIntegrationEndToEnd(t *testing.T) { +func setupTests(t *testing.T) { + ramsem.once.Do(func() { + ramsem.sem = semaphore.NewWeighted(int64(*vmRamLimit)) + }) + if !*runVMTests { t.Skip("not running integration tests (need --run-vm-tests)") } @@ -239,56 +241,53 @@ func TestVMIntegrationEndToEnd(t *testing.T) { t.Logf("hint: nix-shell -p go -p qemu -p cdrkit --run 'go test --v --timeout=60m --run-vm-tests'") t.Fatalf("missing dependency: %v", err) } +} - ramsem := semaphore.NewWeighted(int64(*vmRamLimit)) - rex := distroRex.Unwrap() +var ramsem struct { + once sync.Once + sem *semaphore.Weighted +} - t.Run("do", func(t *testing.T) { - for n, distro := range distros { - n, distro := n, distro - if rex.MatchString(distro.name) { - t.Logf("%s matches %s", distro.name, rex) - } else { - continue - } +func testOneDistribution(t *testing.T, n int, distro Distro) { + setupTests(t) - t.Run(distro.name, func(t *testing.T) { - ctx, done := context.WithCancel(context.Background()) - t.Cleanup(done) + if distroRex.Unwrap().MatchString(distro.Name) { + t.Logf("%s matches %s", distro.Name, distroRex.Unwrap()) + } else { + t.Skip("regex not matched") + } - t.Parallel() + ctx, done := context.WithCancel(context.Background()) + t.Cleanup(done) - h := newHarness(t) - dir := t.TempDir() + h := newHarness(t) + dir := t.TempDir() - err := ramsem.Acquire(ctx, int64(distro.mem)) - if err != nil { - t.Fatalf("can't acquire ram semaphore: %v", err) - } - defer ramsem.Release(int64(distro.mem)) - - h.mkVM(t, n, distro, h.pubKey, h.loginServerURL, dir) - var ipm ipMapping - - t.Run("wait-for-start", func(t *testing.T) { - waiter := time.NewTicker(time.Second) - defer waiter.Stop() - var ok bool - for { - <-waiter.C - h.ipMu.Lock() - if ipm, ok = h.ipMap[distro.name]; ok { - h.ipMu.Unlock() - break - } - h.ipMu.Unlock() - } - }) - - h.testDistro(t, distro, ipm) - }) + err := ramsem.sem.Acquire(ctx, int64(distro.MemoryMegs)) + if err != nil { + t.Fatalf("can't acquire ram semaphore: %v", err) + } + t.Cleanup(func() { ramsem.sem.Release(int64(distro.MemoryMegs)) }) + + h.mkVM(t, n, distro, h.pubKey, h.loginServerURL, dir) + var ipm ipMapping + + t.Run("wait-for-start", func(t *testing.T) { + waiter := time.NewTicker(time.Second) + defer waiter.Stop() + var ok bool + for { + <-waiter.C + h.ipMu.Lock() + if ipm, ok = h.ipMap[distro.Name]; ok { + h.ipMu.Unlock() + break + } + h.ipMu.Unlock() } }) + + h.testDistro(t, distro, ipm) } func (h *Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) { @@ -339,7 +338,7 @@ func (h *Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) { &expect.BExp{R: `(\#)`}, } - switch d.initSystem { + switch d.InitSystem { case "openrc": // NOTE(Xe): this is a sin, however openrc doesn't really have the concept // of service readiness. If this sleep is removed then tailscale will not be