2021-05-26 18:10:10 +00:00
|
|
|
// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
|
|
// +build linux
|
|
|
|
|
|
|
|
package vms
|
|
|
|
|
|
|
|
import (
|
2021-05-31 21:04:49 +00:00
|
|
|
"context"
|
2021-05-26 18:10:10 +00:00
|
|
|
"crypto/sha256"
|
|
|
|
"encoding/hex"
|
|
|
|
"flag"
|
|
|
|
"fmt"
|
|
|
|
"io"
|
2021-05-28 18:19:44 +00:00
|
|
|
"log"
|
2021-05-26 18:10:10 +00:00
|
|
|
"net"
|
|
|
|
"net/http"
|
|
|
|
"os"
|
|
|
|
"os/exec"
|
|
|
|
"path"
|
|
|
|
"path/filepath"
|
|
|
|
"regexp"
|
2021-05-31 17:14:30 +00:00
|
|
|
"strconv"
|
2021-05-26 18:10:10 +00:00
|
|
|
"strings"
|
|
|
|
"sync"
|
|
|
|
"syscall"
|
|
|
|
"testing"
|
|
|
|
"text/template"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
expect "github.com/google/goexpect"
|
2021-05-31 15:35:01 +00:00
|
|
|
"github.com/pkg/sftp"
|
2021-05-26 18:10:10 +00:00
|
|
|
"golang.org/x/crypto/ssh"
|
2021-05-31 21:04:49 +00:00
|
|
|
"golang.org/x/sync/semaphore"
|
2021-05-28 12:05:17 +00:00
|
|
|
"inet.af/netaddr"
|
2021-05-31 16:00:50 +00:00
|
|
|
"tailscale.com/net/interfaces"
|
2021-05-28 18:19:44 +00:00
|
|
|
"tailscale.com/tstest"
|
2021-05-31 15:35:01 +00:00
|
|
|
"tailscale.com/tstest/integration"
|
2021-05-26 18:10:10 +00:00
|
|
|
"tailscale.com/tstest/integration/testcontrol"
|
|
|
|
)
|
|
|
|
|
2021-05-31 15:35:01 +00:00
|
|
|
const securePassword = "hunter2"
|
|
|
|
|
2021-05-31 21:04:49 +00:00
|
|
|
var runVMTests = flag.Bool("run-vm-tests", false, "if set, run expensive VM based integration tests")
|
|
|
|
var vmRamLimit = flag.Int("ram-limit", 4096, "the maximum number of megabytes of ram that can be used for VMs, must be greater than or equal to 1024")
|
2021-05-31 17:23:38 +00:00
|
|
|
var distroRex *regexValue = func() *regexValue {
|
|
|
|
result := ®exValue{r: regexp.MustCompile(`.*`)}
|
|
|
|
flag.Var(result, "distro-regex", "The regex that matches what distros should be run")
|
|
|
|
return result
|
|
|
|
}()
|
2021-05-26 18:10:10 +00:00
|
|
|
|
|
|
|
type Distro struct {
|
|
|
|
name string // amazon-linux
|
|
|
|
url string // URL to a qcow2 image
|
2021-05-27 18:29:29 +00:00
|
|
|
sha256sum string // hex-encoded sha256 sum of contents of URL
|
2021-05-26 18:10:10 +00:00
|
|
|
mem int // VM memory in megabytes
|
|
|
|
packageManager string // yum/apt/dnf/zypper
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *Distro) InstallPre() string {
|
|
|
|
switch d.packageManager {
|
|
|
|
case "yum":
|
|
|
|
return ` - [ yum, update, gnupg2 ]
|
2021-05-31 17:14:30 +00:00
|
|
|
- [ yum, "-y", install, iptables ]`
|
2021-05-31 15:35:01 +00:00
|
|
|
case "zypper":
|
2021-05-31 17:14:30 +00:00
|
|
|
return ` - [ zypper, in, "-y", iptables ]`
|
2021-05-31 15:35:01 +00:00
|
|
|
|
|
|
|
case "dnf":
|
2021-05-31 17:14:30 +00:00
|
|
|
return ` - [ dnf, install, "-y", iptables ]`
|
2021-05-31 15:35:01 +00:00
|
|
|
|
2021-05-26 18:10:10 +00:00
|
|
|
case "apt":
|
|
|
|
return ` - [ apt-get, update ]
|
2021-05-31 17:14:30 +00:00
|
|
|
- [ apt-get, "-y", install, curl, "apt-transport-https", gnupg2 ]`
|
2021-05-31 15:35:01 +00:00
|
|
|
|
|
|
|
case "apk":
|
|
|
|
return ` - [ apk, "-U", add, curl, "ca-certificates" ]`
|
2021-05-26 18:10:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
|
|
|
|
// fetchDistro fetches a distribution from the internet if it doesn't already exist locally. It
|
|
|
|
// also validates the sha256 sum from a known good hash.
|
|
|
|
func fetchDistro(t *testing.T, resultDistro Distro) {
|
|
|
|
t.Helper()
|
|
|
|
|
|
|
|
cdir, err := os.UserCacheDir()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("can't find cache dir: %v", err)
|
|
|
|
}
|
|
|
|
cdir = filepath.Join(cdir, "tailscale", "vm-test")
|
|
|
|
|
|
|
|
qcowPath := filepath.Join(cdir, "qcow2", resultDistro.sha256sum)
|
|
|
|
|
|
|
|
_, err = os.Stat(qcowPath)
|
|
|
|
if err != nil {
|
|
|
|
t.Logf("downloading distro image %s to %s", resultDistro.url, qcowPath)
|
|
|
|
fout, err := os.Create(qcowPath)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
resp, err := http.Get(resultDistro.url)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("can't fetch qcow2 for %s (%s): %v", resultDistro.name, resultDistro.url, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
|
|
resp.Body.Close()
|
|
|
|
t.Fatalf("%s replied %s", resultDistro.url, resp.Status)
|
|
|
|
}
|
|
|
|
|
|
|
|
_, err = io.Copy(fout, resp.Body)
|
|
|
|
resp.Body.Close()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("download of %s failed: %v", resultDistro.url, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = fout.Close()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("can't close fout: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
fin, err := os.Open(qcowPath)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
hasher := sha256.New()
|
|
|
|
if _, err := io.Copy(hasher, fin); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
hash := hex.EncodeToString(hasher.Sum(nil))
|
|
|
|
|
|
|
|
if hash != resultDistro.sha256sum {
|
|
|
|
t.Logf("got: %q", hash)
|
2021-05-27 18:29:29 +00:00
|
|
|
t.Logf("want: %q", resultDistro.sha256sum)
|
2021-05-26 18:10:10 +00:00
|
|
|
t.Fatal("hash mismatch, someone is doing something nasty")
|
|
|
|
}
|
|
|
|
|
|
|
|
t.Logf("hash check passed (%s)", resultDistro.sha256sum)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// run runs a command or fails the test.
|
|
|
|
func run(t *testing.T, dir, prog string, args ...string) {
|
|
|
|
t.Helper()
|
|
|
|
t.Logf("running: %s %s", prog, strings.Join(args, " "))
|
2021-05-28 18:19:44 +00:00
|
|
|
tstest.FixLogs(t)
|
2021-05-26 18:10:10 +00:00
|
|
|
|
|
|
|
cmd := exec.Command(prog, args...)
|
2021-05-28 18:19:44 +00:00
|
|
|
cmd.Stdout = log.Writer()
|
|
|
|
cmd.Stderr = log.Writer()
|
2021-05-26 18:10:10 +00:00
|
|
|
cmd.Dir = dir
|
|
|
|
if err := cmd.Run(); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-05-31 15:35:01 +00:00
|
|
|
// mkLayeredQcow makes a layered qcow image that allows us to keep the upstream
|
|
|
|
// VM images pristine and only do our changes on an overlay.
|
2021-05-26 18:10:10 +00:00
|
|
|
func mkLayeredQcow(t *testing.T, tdir string, d Distro) {
|
|
|
|
t.Helper()
|
|
|
|
|
|
|
|
cdir, err := os.UserCacheDir()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("can't find cache dir: %v", err)
|
|
|
|
}
|
|
|
|
cdir = filepath.Join(cdir, "tailscale", "vm-test")
|
|
|
|
|
|
|
|
run(t, tdir, "qemu-img", "create",
|
|
|
|
"-f", "qcow2",
|
|
|
|
"-o", "backing_file="+filepath.Join(cdir, "qcow2", d.sha256sum),
|
|
|
|
filepath.Join(tdir, d.name+".qcow2"),
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2021-05-31 15:35:01 +00:00
|
|
|
var (
|
|
|
|
metaDataTempl = template.Must(template.New("meta-data.yaml").Parse(metaDataTemplate))
|
|
|
|
userDataTempl = template.Must(template.New("user-data.yaml").Parse(userDataTemplate))
|
|
|
|
)
|
|
|
|
|
|
|
|
// mkSeed makes the cloud-init seed ISO that is used to configure a VM with
|
|
|
|
// tailscale.
|
2021-05-26 18:10:10 +00:00
|
|
|
func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
|
|
|
|
t.Helper()
|
|
|
|
|
|
|
|
dir := filepath.Join(tdir, d.name, "seed")
|
|
|
|
os.MkdirAll(dir, 0700)
|
|
|
|
|
|
|
|
// make meta-data
|
|
|
|
{
|
|
|
|
fout, err := os.Create(filepath.Join(dir, "meta-data"))
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2021-05-31 15:35:01 +00:00
|
|
|
err = metaDataTempl.Execute(fout, struct {
|
2021-05-26 18:10:10 +00:00
|
|
|
ID string
|
|
|
|
Hostname string
|
|
|
|
}{
|
|
|
|
ID: "31337",
|
|
|
|
Hostname: d.name,
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2021-05-27 18:29:29 +00:00
|
|
|
err = fout.Close()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2021-05-26 18:10:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// make user-data
|
|
|
|
{
|
|
|
|
fout, err := os.Create(filepath.Join(dir, "user-data"))
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2021-05-31 15:35:01 +00:00
|
|
|
err = userDataTempl.Execute(fout, struct {
|
2021-05-26 18:10:10 +00:00
|
|
|
SSHKey string
|
|
|
|
HostURL string
|
|
|
|
Hostname string
|
|
|
|
Port int
|
|
|
|
InstallPre string
|
2021-05-31 15:35:01 +00:00
|
|
|
Password string
|
2021-05-26 18:10:10 +00:00
|
|
|
}{
|
|
|
|
SSHKey: strings.TrimSpace(sshKey),
|
|
|
|
HostURL: hostURL,
|
|
|
|
Hostname: d.name,
|
|
|
|
Port: port,
|
|
|
|
InstallPre: d.InstallPre(),
|
2021-05-31 15:35:01 +00:00
|
|
|
Password: securePassword,
|
2021-05-26 18:10:10 +00:00
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2021-05-27 18:29:29 +00:00
|
|
|
err = fout.Close()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2021-05-26 18:10:10 +00:00
|
|
|
}
|
|
|
|
|
2021-06-03 13:29:07 +00:00
|
|
|
args := []string{
|
2021-05-26 18:10:10 +00:00
|
|
|
"-output", filepath.Join(dir, "seed.iso"),
|
|
|
|
"-volid", "cidata", "-joliet", "-rock",
|
|
|
|
filepath.Join(dir, "meta-data"),
|
|
|
|
filepath.Join(dir, "user-data"),
|
2021-06-03 13:29:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if hackOpenSUSE151UserData(t, d, dir) {
|
|
|
|
args = append(args, filepath.Join(dir, "openstack"))
|
|
|
|
}
|
|
|
|
|
|
|
|
run(t, tdir, "genisoimage", args...)
|
2021-05-26 18:10:10 +00:00
|
|
|
}
|
|
|
|
|
2021-05-31 15:35:01 +00:00
|
|
|
// mkVM makes a KVM-accelerated virtual machine and prepares it for introduction
|
|
|
|
// to the testcontrol server. The function it returns is for killing the virtual
|
|
|
|
// machine when it is time for it to die.
|
2021-05-26 18:10:10 +00:00
|
|
|
func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) func() {
|
|
|
|
t.Helper()
|
|
|
|
|
|
|
|
cdir, err := os.UserCacheDir()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("can't find cache dir: %v", err)
|
|
|
|
}
|
2021-05-31 21:04:49 +00:00
|
|
|
cdir = filepath.Join(cdir, "tailscale", "vm-test")
|
2021-05-26 18:10:10 +00:00
|
|
|
os.MkdirAll(filepath.Join(cdir, "qcow2"), 0755)
|
|
|
|
|
|
|
|
port := 23100 + n
|
|
|
|
|
|
|
|
fetchDistro(t, d)
|
|
|
|
mkLayeredQcow(t, tdir, d)
|
|
|
|
mkSeed(t, d, sshKey, hostURL, tdir, port)
|
|
|
|
|
|
|
|
driveArg := fmt.Sprintf("file=%s,if=virtio", filepath.Join(tdir, d.name+".qcow2"))
|
|
|
|
|
|
|
|
args := []string{
|
|
|
|
"-machine", "pc-q35-5.1,accel=kvm,usb=off,vmport=off,dump-guest-core=off",
|
|
|
|
"-netdev", fmt.Sprintf("user,hostfwd=::%d-:22,id=net0", port),
|
|
|
|
"-device", "virtio-net-pci,netdev=net0,id=net0,mac=8a:28:5c:30:1f:25",
|
|
|
|
"-m", fmt.Sprint(d.mem),
|
|
|
|
"-boot", "c",
|
|
|
|
"-drive", driveArg,
|
|
|
|
"-cdrom", filepath.Join(tdir, d.name, "seed", "seed.iso"),
|
|
|
|
"-vnc", fmt.Sprintf(":%d", n),
|
2021-05-31 21:04:49 +00:00
|
|
|
"-smbios", "type=1,serial=ds=nocloud;h=" + d.name,
|
2021-05-26 18:10:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
t.Logf("running: qemu-system-x86_64 %s", strings.Join(args, " "))
|
|
|
|
|
|
|
|
cmd := exec.Command("qemu-system-x86_64", args...)
|
|
|
|
err = cmd.Start()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
time.Sleep(time.Second)
|
|
|
|
|
|
|
|
if err := cmd.Process.Signal(syscall.Signal(0)); err != nil {
|
|
|
|
t.Fatal("qemu is not running")
|
|
|
|
}
|
|
|
|
|
|
|
|
return func() {
|
|
|
|
err := cmd.Process.Kill()
|
|
|
|
if err != nil {
|
|
|
|
t.Errorf("can't kill %s (%d): %v", d.name, cmd.Process.Pid, err)
|
|
|
|
}
|
2021-06-03 14:58:35 +00:00
|
|
|
|
|
|
|
cmd.Wait()
|
2021-05-26 18:10:10 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-05-31 17:14:30 +00:00
|
|
|
// ipMapping maps a hostname, SSH port and SSH IP together
|
|
|
|
type ipMapping struct {
|
|
|
|
name string
|
|
|
|
port int
|
|
|
|
ip string
|
|
|
|
}
|
|
|
|
|
2021-05-31 15:35:01 +00:00
|
|
|
// TestVMIntegrationEndToEnd creates a virtual machine with qemu, installs
|
|
|
|
// tailscale on it and then ensures that it connects to the network
|
|
|
|
// successfully.
|
2021-05-26 18:10:10 +00:00
|
|
|
func TestVMIntegrationEndToEnd(t *testing.T) {
|
|
|
|
if !*runVMTests {
|
|
|
|
t.Skip("not running integration tests (need -run-vm-tests)")
|
|
|
|
}
|
|
|
|
|
2021-05-31 15:35:01 +00:00
|
|
|
os.Setenv("CGO_ENABLED", "0")
|
|
|
|
|
2021-05-26 18:10:10 +00:00
|
|
|
if _, err := exec.LookPath("qemu-system-x86_64"); err != nil {
|
|
|
|
t.Logf("hint: nix-shell -p go -p qemu -p cdrkit --run 'go test -v -timeout=60m -run-vm-tests'")
|
|
|
|
t.Fatalf("missing dependency: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if _, err := exec.LookPath("genisoimage"); err != nil {
|
|
|
|
t.Logf("hint: nix-shell -p go -p qemu -p cdrkit --run 'go test -v -timeout=60m -run-vm-tests'")
|
|
|
|
t.Fatalf("missing dependency: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
distros := []Distro{
|
2021-05-31 15:35:01 +00:00
|
|
|
// NOTE(Xe): If you run into issues getting the autoconfig to work, comment
|
|
|
|
// out all the other distros and uncomment this one. Connect with a VNC
|
|
|
|
// client with a command like this:
|
|
|
|
//
|
|
|
|
// $ vncviewer :0
|
|
|
|
//
|
|
|
|
// On NixOS you can get away with something like this:
|
|
|
|
//
|
|
|
|
// $ env NIXPKGS_ALLOW_UNFREE=1 nix-shell -p tigervnc --run 'vncviewer :0'
|
|
|
|
//
|
|
|
|
// Login as root with the password root. Then look in
|
|
|
|
// /var/log/cloud-init-output.log for what you messed up.
|
|
|
|
|
|
|
|
// {"alpine-edge", "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-edge-2021-05-18-cloud-init-within.qcow2", "b3bb15311c0bd3beffa1b554f022b75d3b7309b5fdf76fb146fe7c72b83b16d0", 256, "apk"},
|
|
|
|
|
2021-05-27 18:29:29 +00:00
|
|
|
{"amazon-linux", "https://cdn.amazonlinux.com/os-images/2.0.20210427.0/kvm/amzn2-kvm-2.0.20210427.0-x86_64.xfs.gpt.qcow2", "6ef9daef32cec69b2d0088626ec96410cd24afc504d57278bbf2f2ba2b7e529b", 512, "yum"},
|
2021-06-03 13:09:18 +00:00
|
|
|
{"arch", "https://mirror.pkgbuild.com/images/v20210515.22945/Arch-Linux-x86_64-cloudimg-20210515.22945.qcow2", "e4077f5ba3c5d545478f64834bc4852f9f7a2e05950fce8ecd0df84193162a27", 512, "pacman"},
|
2021-05-27 18:29:29 +00:00
|
|
|
{"centos-7", "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2", "1db30c9c272fb37b00111b93dcebff16c278384755bdbe158559e9c240b73b80", 512, "yum"},
|
|
|
|
{"centos-8", "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2", "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0", 768, "dnf"},
|
|
|
|
{"debian-9", "https://cdimage.debian.org/cdimage/openstack/9.13.21-20210511/debian-9.13.21-20210511-openstack-amd64.qcow2", "0667a08e2d947b331aee068db4bbf3a703e03edaf5afa52e23d534adff44b62a", 512, "apt"},
|
|
|
|
{"debian-10", "https://cdimage.debian.org/images/cloud/buster/20210329-591/debian-10-generic-amd64-20210329-591.qcow2", "70c61956095870c4082103d1a7a1cb5925293f8405fc6cb348588ec97e8611b0", 768, "apt"},
|
|
|
|
{"fedora-34", "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2", "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea", 768, "dnf"},
|
2021-05-31 17:14:30 +00:00
|
|
|
{"opensuse-leap-15-1", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2", "3203e256dab5981ca3301408574b63bc522a69972fbe9850b65b54ff44a96e0a", 512, "zypper"},
|
|
|
|
{"opensuse-leap-15-2", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2", "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f", 512, "zypper"},
|
2021-06-01 15:08:45 +00:00
|
|
|
{"opensuse-leap-15-3", "http://mirror.its.dal.ca/opensuse/distribution/leap/15.3/appliances/openSUSE-Leap-15.3-JeOS.x86_64-OpenStack-Cloud.qcow2", "22e0392e4d0becb523d1bc5f709366140b7ee20d6faf26de3d0f9046d1ee15d5", 512, "zypper"},
|
2021-05-28 12:05:17 +00:00
|
|
|
{"opensuse-tumbleweed", "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2", "ba3ecd281045b5019f0fb11378329a644a41870b77631ea647b128cd07eb804b", 512, "zypper"},
|
2021-05-27 18:29:29 +00:00
|
|
|
{"ubuntu-16-04", "https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img", "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b", 512, "apt"},
|
2021-05-28 12:05:17 +00:00
|
|
|
{"ubuntu-18-04", "https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img", "08396cf95c18534a2e3f88289bd92d18eee76f0e75813636b3ab9f1e603816d7", 512, "apt"},
|
2021-05-27 18:29:29 +00:00
|
|
|
{"ubuntu-20-04", "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img", "513158b22ff0f08d0a078d8d60293bcddffdb17094a7809c76c52aba415ecc54", 512, "apt"},
|
2021-05-28 12:05:17 +00:00
|
|
|
{"ubuntu-20-10", "https://cloud-images.ubuntu.com/groovy/current/groovy-server-cloudimg-amd64.img", "e470df72fce4fb8d0ee4ef8af8eed740ee3bf51290515eb42e5c747725e98b6d", 512, "apt"},
|
|
|
|
{"ubuntu-21-04", "https://cloud-images.ubuntu.com/hirsute/current/hirsute-server-cloudimg-amd64.img", "7fab8eda0bcf6f8f6e63845ccf1e29de4706e3359c82d3888835093020fe6f05", 512, "apt"},
|
2021-05-26 18:10:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
dir := t.TempDir()
|
|
|
|
|
2021-05-31 17:23:38 +00:00
|
|
|
rex := distroRex.Unwrap()
|
|
|
|
|
2021-05-28 12:05:17 +00:00
|
|
|
ln, err := net.Listen("tcp", deriveBindhost(t)+":0")
|
2021-05-26 18:10:10 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("can't make TCP listener: %v", err)
|
|
|
|
}
|
|
|
|
defer ln.Close()
|
|
|
|
t.Logf("host:port: %s", ln.Addr())
|
|
|
|
|
|
|
|
cs := &testcontrol.Server{}
|
|
|
|
|
|
|
|
var (
|
|
|
|
ipMu sync.Mutex
|
2021-05-31 21:04:49 +00:00
|
|
|
ipMap = map[string]ipMapping{}
|
2021-05-26 18:10:10 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
mux := http.NewServeMux()
|
|
|
|
mux.Handle("/", cs)
|
|
|
|
|
|
|
|
// This handler will let the virtual machines tell the host information about that VM.
|
|
|
|
// This is used to maintain a list of port->IP address mappings that are known to be
|
|
|
|
// working. This allows later steps to connect over SSH. This returns no response to
|
|
|
|
// clients because no response is needed.
|
|
|
|
mux.HandleFunc("/myip/", func(w http.ResponseWriter, r *http.Request) {
|
|
|
|
ipMu.Lock()
|
|
|
|
defer ipMu.Unlock()
|
|
|
|
|
|
|
|
name := path.Base(r.URL.Path)
|
|
|
|
host, _, _ := net.SplitHostPort(r.RemoteAddr)
|
2021-05-31 17:14:30 +00:00
|
|
|
port, err := strconv.Atoi(name)
|
|
|
|
if err != nil {
|
|
|
|
log.Panicf("bad port: %v", port)
|
|
|
|
}
|
2021-05-31 21:04:49 +00:00
|
|
|
distro := r.UserAgent()
|
|
|
|
ipMap[distro] = ipMapping{distro, port, host}
|
2021-05-26 18:10:10 +00:00
|
|
|
t.Logf("%s: %v", name, host)
|
|
|
|
})
|
|
|
|
|
|
|
|
hs := &http.Server{Handler: mux}
|
|
|
|
go hs.Serve(ln)
|
|
|
|
|
|
|
|
run(t, dir, "ssh-keygen", "-t", "ed25519", "-f", "machinekey", "-N", ``)
|
|
|
|
pubkey, err := os.ReadFile(filepath.Join(dir, "machinekey.pub"))
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("can't read ssh key: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
privateKey, err := os.ReadFile(filepath.Join(dir, "machinekey"))
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("can't read ssh private key: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
signer, err := ssh.ParsePrivateKey(privateKey)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("can't parse private key: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
loginServer := fmt.Sprintf("http://%s", ln.Addr())
|
2021-05-28 12:05:17 +00:00
|
|
|
t.Logf("loginServer: %s", loginServer)
|
2021-05-26 18:10:10 +00:00
|
|
|
|
2021-05-31 21:04:49 +00:00
|
|
|
tstest.FixLogs(t)
|
|
|
|
defer tstest.UnfixLogs(t)
|
2021-05-31 17:23:38 +00:00
|
|
|
|
2021-05-31 21:04:49 +00:00
|
|
|
ramsem := semaphore.NewWeighted(int64(*vmRamLimit))
|
2021-06-03 14:58:35 +00:00
|
|
|
bins := integration.BuildTestBinaries(t)
|
2021-05-26 18:10:10 +00:00
|
|
|
|
2021-05-31 21:04:49 +00:00
|
|
|
t.Run("do", func(t *testing.T) {
|
2021-05-26 18:10:10 +00:00
|
|
|
for n, distro := range distros {
|
|
|
|
n, distro := n, distro
|
2021-05-31 17:23:38 +00:00
|
|
|
if rex.MatchString(distro.name) {
|
|
|
|
t.Logf("%s matches %s", distro.name, rex)
|
|
|
|
} else {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2021-05-26 18:10:10 +00:00
|
|
|
t.Run(distro.name, func(t *testing.T) {
|
2021-05-31 21:04:49 +00:00
|
|
|
ctx, done := context.WithCancel(context.Background())
|
|
|
|
defer done()
|
|
|
|
|
2021-05-26 18:10:10 +00:00
|
|
|
t.Parallel()
|
|
|
|
|
2021-05-31 21:04:49 +00:00
|
|
|
err := ramsem.Acquire(ctx, int64(distro.mem))
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("can't acquire ram semaphore: %v", err)
|
|
|
|
}
|
|
|
|
defer ramsem.Release(int64(distro.mem))
|
|
|
|
|
2021-05-26 18:10:10 +00:00
|
|
|
cancel := mkVM(t, n, distro, string(pubkey), loginServer, dir)
|
2021-05-31 21:04:49 +00:00
|
|
|
defer cancel()
|
|
|
|
var ipm ipMapping
|
|
|
|
|
|
|
|
t.Run("wait-for-start", func(t *testing.T) {
|
|
|
|
waiter := time.NewTicker(time.Second)
|
|
|
|
defer waiter.Stop()
|
|
|
|
var ok bool
|
|
|
|
for {
|
|
|
|
<-waiter.C
|
|
|
|
ipMu.Lock()
|
|
|
|
if ipm, ok = ipMap[distro.name]; ok {
|
|
|
|
ipMu.Unlock()
|
|
|
|
break
|
|
|
|
}
|
|
|
|
ipMu.Unlock()
|
|
|
|
}
|
|
|
|
})
|
|
|
|
|
2021-06-03 14:58:35 +00:00
|
|
|
testDistro(t, loginServer, signer, ipm, bins)
|
2021-05-26 18:10:10 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
})
|
2021-05-31 21:04:49 +00:00
|
|
|
}
|
2021-05-26 18:10:10 +00:00
|
|
|
|
2021-06-03 14:58:35 +00:00
|
|
|
func testDistro(t *testing.T, loginServer string, signer ssh.Signer, ipm ipMapping, bins *integration.Binaries) {
|
2021-05-31 21:04:49 +00:00
|
|
|
t.Helper()
|
|
|
|
port := ipm.port
|
|
|
|
hostport := fmt.Sprintf("127.0.0.1:%d", port)
|
|
|
|
ccfg := &ssh.ClientConfig{
|
|
|
|
User: "root",
|
|
|
|
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer), ssh.Password(securePassword)},
|
|
|
|
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
2021-05-26 18:10:10 +00:00
|
|
|
}
|
|
|
|
|
2021-05-31 21:04:49 +00:00
|
|
|
// NOTE(Xe): This deadline loop helps to make things a bit faster, centos
|
|
|
|
// sometimes is slow at starting its sshd and will sometimes randomly kill
|
|
|
|
// SSH sessions on transition to multi-user.target. I don't know why they
|
|
|
|
// don't use socket activation.
|
|
|
|
const maxRetries = 5
|
|
|
|
var working bool
|
|
|
|
for i := 0; i < maxRetries; i++ {
|
|
|
|
cli, err := ssh.Dial("tcp", hostport, ccfg)
|
|
|
|
if err == nil {
|
|
|
|
working = true
|
|
|
|
cli.Close()
|
|
|
|
break
|
2021-05-26 18:10:10 +00:00
|
|
|
}
|
2021-05-31 15:35:01 +00:00
|
|
|
|
2021-05-31 21:04:49 +00:00
|
|
|
time.Sleep(10 * time.Second)
|
|
|
|
}
|
2021-05-26 18:10:10 +00:00
|
|
|
|
2021-05-31 21:04:49 +00:00
|
|
|
if !working {
|
|
|
|
t.Fatalf("can't connect to %s, tried %d times", hostport, maxRetries)
|
|
|
|
}
|
2021-05-31 15:35:01 +00:00
|
|
|
|
2021-05-31 21:04:49 +00:00
|
|
|
t.Logf("about to ssh into 127.0.0.1:%d", port)
|
|
|
|
cli, err := ssh.Dial("tcp", hostport, ccfg)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2021-06-03 14:58:35 +00:00
|
|
|
copyBinaries(t, cli, bins)
|
2021-05-26 18:10:10 +00:00
|
|
|
|
2021-05-31 21:04:49 +00:00
|
|
|
timeout := 5 * time.Minute
|
2021-05-26 18:10:10 +00:00
|
|
|
|
2021-05-31 21:04:49 +00:00
|
|
|
e, _, err := expect.SpawnSSH(cli, timeout, expect.Verbose(true), expect.VerboseWriter(log.Writer()))
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("%d: can't register a shell session: %v", port, err)
|
|
|
|
}
|
|
|
|
defer e.Close()
|
2021-05-31 15:35:01 +00:00
|
|
|
|
2021-05-31 21:04:49 +00:00
|
|
|
t.Log("opened session")
|
2021-05-26 18:10:10 +00:00
|
|
|
|
2021-05-31 21:04:49 +00:00
|
|
|
_, _, err = e.Expect(regexp.MustCompile(`(\#)`), timeout)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("%d: can't get a shell: %v", port, err)
|
|
|
|
}
|
|
|
|
t.Logf("got shell for %d", port)
|
|
|
|
err = e.Send("systemctl start tailscaled.service\n")
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("can't send command to start tailscaled: %v", err)
|
|
|
|
}
|
|
|
|
_, _, err = e.Expect(regexp.MustCompile(`(\#)`), timeout)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("%d: can't get a shell: %v", port, err)
|
|
|
|
}
|
|
|
|
err = e.Send(fmt.Sprintf("sudo tailscale up --login-server %s\n", loginServer))
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("%d: can't send tailscale up command: %v", port, err)
|
|
|
|
}
|
|
|
|
_, _, err = e.Expect(regexp.MustCompile(`Success.`), timeout)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("not successful: %v", err)
|
2021-05-26 18:10:10 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-03 14:58:35 +00:00
|
|
|
func copyBinaries(t *testing.T, conn *ssh.Client, bins *integration.Binaries) {
|
2021-05-31 15:35:01 +00:00
|
|
|
cli, err := sftp.NewClient(conn)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("can't connect over sftp to copy binaries: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
mkdir(t, cli, "/usr/bin")
|
|
|
|
mkdir(t, cli, "/usr/sbin")
|
|
|
|
mkdir(t, cli, "/etc/systemd/system")
|
|
|
|
mkdir(t, cli, "/etc/default")
|
|
|
|
|
|
|
|
copyFile(t, cli, bins.Daemon, "/usr/sbin/tailscaled")
|
|
|
|
copyFile(t, cli, bins.CLI, "/usr/bin/tailscale")
|
|
|
|
|
|
|
|
// TODO(Xe): revisit this life decision, hopefully before this assumption
|
|
|
|
// breaks the test.
|
|
|
|
copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.defaults", "/etc/default/tailscaled")
|
|
|
|
copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.service", "/etc/systemd/system/tailscaled.service")
|
|
|
|
|
|
|
|
t.Log("tailscale installed!")
|
|
|
|
}
|
|
|
|
|
|
|
|
func mkdir(t *testing.T, cli *sftp.Client, name string) {
|
|
|
|
t.Helper()
|
|
|
|
|
|
|
|
err := cli.MkdirAll(name)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("can't make %s: %v", name, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func copyFile(t *testing.T, cli *sftp.Client, localSrc, remoteDest string) {
|
|
|
|
t.Helper()
|
|
|
|
|
|
|
|
fin, err := os.Open(localSrc)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("can't open: %v", err)
|
|
|
|
}
|
|
|
|
defer fin.Close()
|
|
|
|
|
|
|
|
fi, err := fin.Stat()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("can't stat: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
fout, err := cli.Create(remoteDest)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("can't create output file: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = fout.Chmod(fi.Mode())
|
|
|
|
if err != nil {
|
|
|
|
fout.Close()
|
|
|
|
t.Fatalf("can't chmod fout: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
n, err := io.Copy(fout, fin)
|
|
|
|
if err != nil {
|
|
|
|
fout.Close()
|
|
|
|
t.Fatalf("copy failed: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if fi.Size() != n {
|
|
|
|
t.Fatalf("incorrect number of bytes copied: wanted: %d, got: %d", fi.Size(), n)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = fout.Close()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("can't close fout on remote host: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-05-28 12:05:17 +00:00
|
|
|
func deriveBindhost(t *testing.T) string {
|
|
|
|
t.Helper()
|
|
|
|
|
2021-05-31 16:00:50 +00:00
|
|
|
ifName, err := interfaces.DefaultRouteInterface()
|
2021-05-28 12:05:17 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2021-05-31 16:00:50 +00:00
|
|
|
var ret string
|
|
|
|
err = interfaces.ForeachInterfaceAddress(func(i interfaces.Interface, prefix netaddr.IPPrefix) {
|
|
|
|
if ret != "" || i.Name != ifName {
|
|
|
|
return
|
2021-05-28 12:05:17 +00:00
|
|
|
}
|
2021-05-31 16:00:50 +00:00
|
|
|
ret = prefix.IP().String()
|
|
|
|
})
|
|
|
|
if ret != "" {
|
|
|
|
return ret
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
2021-05-28 12:05:17 +00:00
|
|
|
}
|
|
|
|
t.Fatal("can't find a bindhost")
|
2021-05-31 16:00:50 +00:00
|
|
|
return "unreachable"
|
2021-05-28 12:05:17 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func TestDeriveBindhost(t *testing.T) {
|
|
|
|
t.Log(deriveBindhost(t))
|
|
|
|
}
|
|
|
|
|
2021-05-26 18:10:10 +00:00
|
|
|
const metaDataTemplate = `instance-id: {{.ID}}
|
|
|
|
local-hostname: {{.Hostname}}`
|
|
|
|
|
|
|
|
const userDataTemplate = `#cloud-config
|
|
|
|
#vim:syntax=yaml
|
|
|
|
|
|
|
|
cloud_config_modules:
|
|
|
|
- runcmd
|
|
|
|
|
|
|
|
cloud_final_modules:
|
|
|
|
- [users-groups, always]
|
|
|
|
- [scripts-user, once-per-instance]
|
|
|
|
|
|
|
|
users:
|
2021-05-31 15:35:01 +00:00
|
|
|
- name: root
|
|
|
|
ssh-authorized-keys:
|
|
|
|
- {{.SSHKey}}
|
|
|
|
- name: ts
|
|
|
|
plain_text_passwd: {{.Password}}
|
|
|
|
groups: [ wheel ]
|
|
|
|
sudo: [ "ALL=(ALL) NOPASSWD:ALL" ]
|
|
|
|
shell: /bin/sh
|
|
|
|
ssh-authorized-keys:
|
2021-05-26 18:10:10 +00:00
|
|
|
- {{.SSHKey}}
|
|
|
|
|
|
|
|
write_files:
|
|
|
|
- path: /etc/cloud/cloud.cfg.d/80_disable_network_after_firstboot.cfg
|
|
|
|
content: |
|
|
|
|
# Disable network configuration after first boot
|
|
|
|
network:
|
|
|
|
config: disabled
|
|
|
|
|
|
|
|
runcmd:
|
|
|
|
{{.InstallPre}}
|
|
|
|
- [ curl, "{{.HostURL}}/myip/{{.Port}}", "-H", "User-Agent: {{.Hostname}}" ]
|
|
|
|
`
|