tstest/integration/vms: copy locally built binaries (#2006)

Instead of pulling packages from pkgs.tailscale.com, we should use the tailscale binaries that are local to this git commit. This exposes a bit of the integration testing stack in order to copy the binaries correctly. This commit also bumps our version of github.com/pkg/sftp to the latest commit. If you run into trouble with yaml, be sure to check out the commented-out alpine linux image complete with instructions on how to use it. Updates #1988 Signed-off-by: Christine Dodrill <xe@tailscale.com>
2024-11-29 04:55:31 +00:00 · 2021-05-31 11:35:01 -04:00 · 2021-05-31 11:35:01 -04:00 · 14c1113d2b
commit 14c1113d2b
parent ca455ac84b
5 changed files with 294 additions and 120 deletions
--- a/go.mod
+++ b/go.mod
@ -24,6 +24,7 @@ require (
 	github.com/miekg/dns v1.1.42
 	github.com/pborman/getopt v1.1.0
 	github.com/peterbourgon/ff/v2 v2.0.0
+	github.com/pkg/sftp v1.13.0 // indirect
 	github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
--- a/go.sum
+++ b/go.sum
@ -341,6 +341,7 @@ github.com/klauspost/compress v1.12.2 h1:2KCfW3I9M7nSc5wOqXAlW2v2U6v+w6cbjvbfp+O
 github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@ -470,6 +471,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
+github.com/pkg/sftp v1.13.0 h1:Riw6pgOKK41foc1I1Uu03CjvbLZDXeGpInycM4shXoI=
+github.com/pkg/sftp v1.13.0/go.mod h1:41g+FIPlQUTDCveupEmEA65IoiQFrtgCeDopC4ajGIM=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/polyfloyd/go-errorlint v0.0.0-20201006195004-351e25ade6e3/go.mod h1:wi9BfjxjF/bwiZ701TzmfKu6UKC357IOAtNr0Td0Lvw=
@ -642,6 +645,7 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc=
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
--- a/tstest/integration/integration.go
+++ b/tstest/integration/integration.go
@ -0,0 +1,100 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package integration contains Tailscale integration tests.
+//
+// This package is considered internal and the public API is subject
+// to change without notice.
+package integration
+
+import (
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"tailscale.com/version"
+)
+
+// Binaries are the paths to a tailscaled and tailscale binary.
+// These can be shared by multiple nodes.
+type Binaries struct {
+	Dir    string // temp dir for tailscale & tailscaled
+	Daemon string // tailscaled
+	CLI    string // tailscale
+}
+
+// BuildTestBinaries builds tailscale and tailscaled, failing the test
+// if they fail to compile.
+func BuildTestBinaries(t testing.TB) *Binaries {
+	td := t.TempDir()
+	build(t, td, "tailscale.com/cmd/tailscaled", "tailscale.com/cmd/tailscale")
+	return &Binaries{
+		Dir:    td,
+		Daemon: filepath.Join(td, "tailscaled"+exe()),
+		CLI:    filepath.Join(td, "tailscale"+exe()),
+	}
+}
+
+// buildMu limits our use of "go build" to one at a time, so we don't
+// fight Go's built-in caching trying to do the same build concurrently.
+var buildMu sync.Mutex
+
+func build(t testing.TB, outDir string, targets ...string) {
+	buildMu.Lock()
+	defer buildMu.Unlock()
+
+	t0 := time.Now()
+	defer func() { t.Logf("built %s in %v", targets, time.Since(t0).Round(time.Millisecond)) }()
+
+	goBin := findGo(t)
+	cmd := exec.Command(goBin, "install")
+	if version.IsRace() {
+		cmd.Args = append(cmd.Args, "-race")
+	}
+	cmd.Args = append(cmd.Args, targets...)
+	cmd.Env = append(os.Environ(), "GOARCH="+runtime.GOARCH, "GOBIN="+outDir)
+	errOut, err := cmd.CombinedOutput()
+	if err == nil {
+		return
+	}
+	if strings.Contains(string(errOut), "when GOBIN is set") {
+		// Fallback slow path for cross-compiled binaries.
+		for _, target := range targets {
+			outFile := filepath.Join(outDir, path.Base(target)+exe())
+			cmd := exec.Command(goBin, "build", "-o", outFile, target)
+			cmd.Env = append(os.Environ(), "GOARCH="+runtime.GOARCH)
+			if errOut, err := cmd.CombinedOutput(); err != nil {
+				t.Fatalf("failed to build %v with %v: %v, %s", target, goBin, err, errOut)
+			}
+		}
+		return
+	}
+	t.Fatalf("failed to build %v with %v: %v, %s", targets, goBin, err, errOut)
+}
+
+func findGo(t testing.TB) string {
+	goBin := filepath.Join(runtime.GOROOT(), "bin", "go"+exe())
+	if fi, err := os.Stat(goBin); err != nil {
+		if os.IsNotExist(err) {
+			t.Fatalf("failed to find go at %v", goBin)
+		}
+		t.Fatalf("looking for go binary: %v", err)
+	} else if !fi.Mode().IsRegular() {
+		t.Fatalf("%v is unexpected %v", goBin, fi.Mode())
+	}
+	return goBin
+}
+
+func exe() string {
+	if runtime.GOOS == "windows" {
+		return ".exe"
+	}
+	return ""
+}
--- a/tstest/integration/integration_test.go
+++ b/tstest/integration/integration_test.go
@ -2,7 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

-// Package integration contains Tailscale integration tests.
 package integration

 import (
@ -21,7 +20,6 @@
 	"net/http/httptest"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
@ -44,7 +42,6 @@
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/nettype"
-	"tailscale.com/version"
 )

 var verbose = flag.Bool("verbose", false, "verbose debug logs")
@ -65,7 +62,7 @@ func TestMain(m *testing.M) {

 func TestOneNodeUp_NoAuth(t *testing.T) {
 	t.Parallel()
-	bins := buildTestBinaries(t)
+	bins := BuildTestBinaries(t)

 	env := newTestEnv(t, bins)
 	defer env.Close()
@ -107,7 +104,7 @@ func TestOneNodeUp_NoAuth(t *testing.T) {

 func TestOneNodeUp_Auth(t *testing.T) {
 	t.Parallel()
-	bins := buildTestBinaries(t)
+	bins := BuildTestBinaries(t)

 	env := newTestEnv(t, bins)
 	defer env.Close()
@ -154,7 +151,7 @@ func TestOneNodeUp_Auth(t *testing.T) {

 func TestTwoNodes(t *testing.T) {
 	t.Parallel()
-	bins := buildTestBinaries(t)
+	bins := BuildTestBinaries(t)

 	env := newTestEnv(t, bins)
 	defer env.Close()
@ -198,7 +195,7 @@ func TestTwoNodes(t *testing.T) {

 func TestNodeAddressIPFields(t *testing.T) {
 	t.Parallel()
-	bins := buildTestBinaries(t)
+	bins := BuildTestBinaries(t)

 	env := newTestEnv(t, bins)
 	defer env.Close()
@ -227,31 +224,11 @@ func TestNodeAddressIPFields(t *testing.T) {
 	d1.MustCleanShutdown(t)
 }

-// testBinaries are the paths to a tailscaled and tailscale binary.
-// These can be shared by multiple nodes.
-type testBinaries struct {
-	dir    string // temp dir for tailscale & tailscaled
-	daemon string // tailscaled
-	cli    string // tailscale
-}
-
-// buildTestBinaries builds tailscale and tailscaled, failing the test
-// if they fail to compile.
-func buildTestBinaries(t testing.TB) *testBinaries {
-	td := t.TempDir()
-	build(t, td, "tailscale.com/cmd/tailscaled", "tailscale.com/cmd/tailscale")
-	return &testBinaries{
-		dir:    td,
-		daemon: filepath.Join(td, "tailscaled"+exe()),
-		cli:    filepath.Join(td, "tailscale"+exe()),
-	}
-}
-
 // testEnv contains the test environment (set of servers) used by one
 // or more nodes.
 type testEnv struct {
 	t        testing.TB
-	Binaries *testBinaries
+	Binaries *Binaries

 	LogCatcher       *logCatcher
 	LogCatcherServer *httptest.Server
@ -269,7 +246,7 @@ type testEnv struct {
 // environment.
 //
 // Call Close to shut everything down.
-func newTestEnv(t testing.TB, bins *testBinaries) *testEnv {
+func newTestEnv(t testing.TB, bins *Binaries) *testEnv {
 	if runtime.GOOS == "windows" {
 		t.Skip("not tested/working on Windows yet")
 	}
@ -352,7 +329,7 @@ func (d *Daemon) MustCleanShutdown(t testing.TB) {
 // StartDaemon starts the node's tailscaled, failing if it fails to
 // start.
 func (n *testNode) StartDaemon(t testing.TB) *Daemon {
-	cmd := exec.Command(n.env.Binaries.daemon,
+	cmd := exec.Command(n.env.Binaries.Daemon,
 		"--tun=userspace-networking",
 		"--state="+n.stateFile,
 		"--socket="+n.sockFile,
@ -430,7 +407,7 @@ func (n *testNode) AwaitRunning(t testing.TB) {
 // Tailscale returns a command that runs the tailscale CLI with the provided arguments.
 // It does not start the process.
 func (n *testNode) Tailscale(arg ...string) *exec.Cmd {
-	cmd := exec.Command(n.env.Binaries.cli, "--socket="+n.sockFile)
+	cmd := exec.Command(n.env.Binaries.CLI, "--socket="+n.sockFile)
 	cmd.Args = append(cmd.Args, arg...)
 	cmd.Dir = n.dir
 	return cmd
@ -457,63 +434,6 @@ func (n *testNode) MustStatus(tb testing.TB) *ipnstate.Status {
 	return st
 }

-func exe() string {
-	if runtime.GOOS == "windows" {
-		return ".exe"
-	}
-	return ""
-}
-
-func findGo(t testing.TB) string {
-	goBin := filepath.Join(runtime.GOROOT(), "bin", "go"+exe())
-	if fi, err := os.Stat(goBin); err != nil {
-		if os.IsNotExist(err) {
-			t.Fatalf("failed to find go at %v", goBin)
-		}
-		t.Fatalf("looking for go binary: %v", err)
-	} else if !fi.Mode().IsRegular() {
-		t.Fatalf("%v is unexpected %v", goBin, fi.Mode())
-	}
-	return goBin
-}
-
-// buildMu limits our use of "go build" to one at a time, so we don't
-// fight Go's built-in caching trying to do the same build concurrently.
-var buildMu sync.Mutex
-
-func build(t testing.TB, outDir string, targets ...string) {
-	buildMu.Lock()
-	defer buildMu.Unlock()
-
-	t0 := time.Now()
-	defer func() { t.Logf("built %s in %v", targets, time.Since(t0).Round(time.Millisecond)) }()
-
-	goBin := findGo(t)
-	cmd := exec.Command(goBin, "install")
-	if version.IsRace() {
-		cmd.Args = append(cmd.Args, "-race")
-	}
-	cmd.Args = append(cmd.Args, targets...)
-	cmd.Env = append(os.Environ(), "GOARCH="+runtime.GOARCH, "GOBIN="+outDir)
-	errOut, err := cmd.CombinedOutput()
-	if err == nil {
-		return
-	}
-	if strings.Contains(string(errOut), "when GOBIN is set") {
-		// Fallback slow path for cross-compiled binaries.
-		for _, target := range targets {
-			outFile := filepath.Join(outDir, path.Base(target)+exe())
-			cmd := exec.Command(goBin, "build", "-o", outFile, target)
-			cmd.Env = append(os.Environ(), "GOARCH="+runtime.GOARCH)
-			if errOut, err := cmd.CombinedOutput(); err != nil {
-				t.Fatalf("failed to build %v with %v: %v, %s", target, goBin, err, errOut)
-			}
-		}
-		return
-	}
-	t.Fatalf("failed to build %v with %v: %v, %s", targets, goBin, err, errOut)
-}
-
 // logCatcher is a minimal logcatcher for the logtail upload client.
 type logCatcher struct {
 	mu     sync.Mutex
--- a/tstest/integration/vms/vms_test.go
+++ b/tstest/integration/vms/vms_test.go
@ -28,12 +28,16 @@
 	"time"

 	expect "github.com/google/goexpect"
+	"github.com/pkg/sftp"
 	"golang.org/x/crypto/ssh"
 	"inet.af/netaddr"
 	"tailscale.com/tstest"
+	"tailscale.com/tstest/integration"
 	"tailscale.com/tstest/integration/testcontrol"
 )

+const securePassword = "hunter2"
+
 var runVMTests = flag.Bool("run-vm-tests", false, "if set, run expensive (10G+ ram) VM based integration tests")

 type Distro struct {
@ -48,11 +52,23 @@ func (d *Distro) InstallPre() string {
 	switch d.packageManager {
 	case "yum":
 		return ` - [ yum, update, gnupg2 ]
+ - [ yum, "-y", install, iptables ]
 `
+	case "zypper":
+		return ` - [ zypper, in, "-y", iptables ]
+`
+
+	case "dnf":
+		return ` - [ dnf, install, "-y", iptables ]
+`
+
 	case "apt":
 		return ` - [ apt-get, update ]
 - [ apt-get, "-y", install, curl, "apt-transport-https", gnupg2 ]
 `
+
+	case "apk":
+		return ` - [ apk, "-U", add, curl, "ca-certificates" ]`
 	}

 	return ""
@ -135,8 +151,8 @@ func run(t *testing.T, dir, prog string, args ...string) {
 	}
 }

-// mkLayeredQcow makes a layered qcow image that allows us to keep the upstream VM images
-// pristine and only do our changes on an overlay.
+// mkLayeredQcow makes a layered qcow image that allows us to keep the upstream
+// VM images pristine and only do our changes on an overlay.
 func mkLayeredQcow(t *testing.T, tdir string, d Distro) {
 	t.Helper()

@ -153,7 +169,13 @@ func mkLayeredQcow(t *testing.T, tdir string, d Distro) {
 	)
 }

-// mkSeed makes the cloud-init seed ISO that is used to configure a VM with tailscale.
+var (
+	metaDataTempl = template.Must(template.New("meta-data.yaml").Parse(metaDataTemplate))
+	userDataTempl = template.Must(template.New("user-data.yaml").Parse(userDataTemplate))
+)
+
+// mkSeed makes the cloud-init seed ISO that is used to configure a VM with
+// tailscale.
 func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 	t.Helper()

@ -167,7 +189,7 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 			t.Fatal(err)
 		}

-		err = template.Must(template.New("meta-data.yaml").Parse(metaDataTemplate)).Execute(fout, struct {
+		err = metaDataTempl.Execute(fout, struct {
 			ID       string
 			Hostname string
 		}{
@ -191,18 +213,20 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 			t.Fatal(err)
 		}

-		err = template.Must(template.New("user-data.yaml").Parse(userDataTemplate)).Execute(fout, struct {
+		err = userDataTempl.Execute(fout, struct {
 			SSHKey     string
 			HostURL    string
 			Hostname   string
 			Port       int
 			InstallPre string
+			Password   string
 		}{
 			SSHKey:     strings.TrimSpace(sshKey),
 			HostURL:    hostURL,
 			Hostname:   d.name,
 			Port:       port,
 			InstallPre: d.InstallPre(),
+			Password:   securePassword,
 		})
 		if err != nil {
 			t.Fatal(err)
@ -222,9 +246,9 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
 	)
 }

-// mkVM makes a KVM-accelerated virtual machine and prepares it for introduction to the
-// testcontrol server. The function it returns is for killing the virtual machine when it
-// is time for it to die.
+// mkVM makes a KVM-accelerated virtual machine and prepares it for introduction
+// to the testcontrol server. The function it returns is for killing the virtual
+// machine when it is time for it to die.
 func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) func() {
 	t.Helper()

@ -277,12 +301,16 @@ func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) func() {
 	}
 }

-// TestVMIntegrationEndToEnd creates a virtual machine with mkvm(1X), installs tailscale on it and then ensures that it connects to the network successfully.
+// TestVMIntegrationEndToEnd creates a virtual machine with qemu, installs
+// tailscale on it and then ensures that it connects to the network
+// successfully.
 func TestVMIntegrationEndToEnd(t *testing.T) {
 	if !*runVMTests {
 		t.Skip("not running integration tests (need -run-vm-tests)")
 	}

+	os.Setenv("CGO_ENABLED", "0")
+
 	if _, err := exec.LookPath("qemu-system-x86_64"); err != nil {
 		t.Logf("hint: nix-shell -p go -p qemu -p cdrkit --run 'go test -v -timeout=60m -run-vm-tests'")
 		t.Fatalf("missing dependency: %v", err)
@ -294,13 +322,30 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
 	}

 	distros := []Distro{
+		// NOTE(Xe): If you run into issues getting the autoconfig to work, comment
+		// out all the other distros and uncomment this one. Connect with a VNC
+		// client with a command like this:
+		//
+		//    $ vncviewer :0
+		//
+		// On NixOS you can get away with something like this:
+		//
+		//    $ env NIXPKGS_ALLOW_UNFREE=1 nix-shell -p tigervnc --run 'vncviewer :0'
+		//
+		// Login as root with the password root. Then look in
+		// /var/log/cloud-init-output.log for what you messed up.
+
+		// {"alpine-edge", "https://xena.greedo.xeserv.us/pkg/alpine/img/alpine-edge-2021-05-18-cloud-init-within.qcow2", "b3bb15311c0bd3beffa1b554f022b75d3b7309b5fdf76fb146fe7c72b83b16d0", 256, "apk"},
+
+		// TODO(Xe): This is broken, and I don't know why, see #1988
+		//{"opensuse-leap-15.1", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2", "3203e256dab5981ca3301408574b63bc522a69972fbe9850b65b54ff44a96e0a", 512, "zypper"},
+
 		{"amazon-linux", "https://cdn.amazonlinux.com/os-images/2.0.20210427.0/kvm/amzn2-kvm-2.0.20210427.0-x86_64.xfs.gpt.qcow2", "6ef9daef32cec69b2d0088626ec96410cd24afc504d57278bbf2f2ba2b7e529b", 512, "yum"},
 		{"centos-7", "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2", "1db30c9c272fb37b00111b93dcebff16c278384755bdbe158559e9c240b73b80", 512, "yum"},
 		{"centos-8", "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2", "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0", 768, "dnf"},
 		{"debian-9", "https://cdimage.debian.org/cdimage/openstack/9.13.21-20210511/debian-9.13.21-20210511-openstack-amd64.qcow2", "0667a08e2d947b331aee068db4bbf3a703e03edaf5afa52e23d534adff44b62a", 512, "apt"},
 		{"debian-10", "https://cdimage.debian.org/images/cloud/buster/20210329-591/debian-10-generic-amd64-20210329-591.qcow2", "70c61956095870c4082103d1a7a1cb5925293f8405fc6cb348588ec97e8611b0", 768, "apt"},
 		{"fedora-34", "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2", "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea", 768, "dnf"},
-		{"opensuse-leap-15.1", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2", "3203e256dab5981ca3301408574b63bc522a69972fbe9850b65b54ff44a96e0a", 512, "zypper"},
 		{"opensuse-leap-15.2", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2", "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f", 512, "zypper"},
 		{"opensuse-tumbleweed", "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2", "ba3ecd281045b5019f0fb11378329a644a41870b77631ea647b128cd07eb804b", 512, "zypper"},
 		{"ubuntu-16-04", "https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img", "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b", 512, "apt"},
@ -419,20 +464,39 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
 			port := port
 			t.Run(port, func(t *testing.T) {
 				tstest.FixLogs(t)
-				config := &ssh.ClientConfig{
-					User:            "ts",
-					Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer), ssh.Password("hunter2")},
-					HostKeyCallback: ssh.InsecureIgnoreHostKey(),
-				}
-
-				cli, err := ssh.Dial("tcp", fmt.Sprintf("127.0.0.1:%s", port), config)
-				if err != nil {
-					t.Fatalf("can't dial 127.0.0.1:%s: %v", port, err)
-				}
-				defer cli.Close()
-
 				t.Parallel()
+
+				hostport := fmt.Sprintf("127.0.0.1:%s", port)
+
+				// NOTE(Xe): This retry loop helps to make things a bit faster, centos sometimes is slow at starting its sshd. I don't know why they don't use socket activation.
+				const maxRetries = 5
+				var working bool
+				for i := 0; i < maxRetries; i++ {
+					conn, err := net.Dial("tcp", hostport)
+					if err == nil {
+						working = true
+						conn.Close()
+						break
+					}
+
+					time.Sleep(5 * time.Second)
+				}
+
+				if !working {
+					t.Fatalf("can't connect to %s, tried %d times", hostport, maxRetries)
+				}
+
 				t.Logf("about to ssh into 127.0.0.1:%s", port)
+				cli, err := ssh.Dial("tcp", hostport, &ssh.ClientConfig{
+					User:            "root",
+					Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer), ssh.Password(securePassword)},
+					HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+				})
+				if err != nil {
+					t.Fatal(err)
+				}
+				copyBinaries(t, cli)
+
 				timeout := 5 * time.Minute

 				e, _, err := expect.SpawnSSH(cli, timeout, expect.Verbose(true), expect.VerboseWriter(log.Writer()))
@ -441,18 +505,28 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
 				}
 				defer e.Close()

-				_, _, err = e.Expect(regexp.MustCompile(`(\$|\>)`), timeout)
+				t.Log("opened session")
+
+				_, _, err = e.Expect(regexp.MustCompile(`(\#)`), timeout)
 				if err != nil {
 					t.Fatalf("%s: can't get a shell: %v", port, err)
 				}
 				t.Logf("got shell for %s", port)
+				err = e.Send("systemctl start tailscaled.service\n")
+				if err != nil {
+					t.Fatalf("can't send command to start tailscaled: %v", err)
+				}
+				_, _, err = e.Expect(regexp.MustCompile(`(\#)`), timeout)
+				if err != nil {
+					t.Fatalf("%s: can't get a shell: %v", port, err)
+				}
 				err = e.Send(fmt.Sprintf("sudo tailscale up --login-server %s\n", loginServer))
 				if err != nil {
 					t.Fatalf("%s: can't send tailscale up command: %v", port, err)
 				}
 				_, _, err = e.Expect(regexp.MustCompile(`Success.`), timeout)
 				if err != nil {
-					t.Fatalf("can't extract URL: %v", err)
+					t.Fatalf("not successful: %v", err)
 				}
 			})
 		}
@ -463,6 +537,80 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
 	}
 }

+func copyBinaries(t *testing.T, conn *ssh.Client) {
+	bins := integration.BuildTestBinaries(t)
+
+	cli, err := sftp.NewClient(conn)
+	if err != nil {
+		t.Fatalf("can't connect over sftp to copy binaries: %v", err)
+	}
+
+	mkdir(t, cli, "/usr/bin")
+	mkdir(t, cli, "/usr/sbin")
+	mkdir(t, cli, "/etc/systemd/system")
+	mkdir(t, cli, "/etc/default")
+
+	copyFile(t, cli, bins.Daemon, "/usr/sbin/tailscaled")
+	copyFile(t, cli, bins.CLI, "/usr/bin/tailscale")
+
+	// TODO(Xe): revisit this life decision, hopefully before this assumption
+	// breaks the test.
+	copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.defaults", "/etc/default/tailscaled")
+	copyFile(t, cli, "../../../cmd/tailscaled/tailscaled.service", "/etc/systemd/system/tailscaled.service")
+
+	t.Log("tailscale installed!")
+}
+
+func mkdir(t *testing.T, cli *sftp.Client, name string) {
+	t.Helper()
+
+	err := cli.MkdirAll(name)
+	if err != nil {
+		t.Fatalf("can't make %s: %v", name, err)
+	}
+}
+
+func copyFile(t *testing.T, cli *sftp.Client, localSrc, remoteDest string) {
+	t.Helper()
+
+	fin, err := os.Open(localSrc)
+	if err != nil {
+		t.Fatalf("can't open: %v", err)
+	}
+	defer fin.Close()
+
+	fi, err := fin.Stat()
+	if err != nil {
+		t.Fatalf("can't stat: %v", err)
+	}
+
+	fout, err := cli.Create(remoteDest)
+	if err != nil {
+		t.Fatalf("can't create output file: %v", err)
+	}
+
+	err = fout.Chmod(fi.Mode())
+	if err != nil {
+		fout.Close()
+		t.Fatalf("can't chmod fout: %v", err)
+	}
+
+	n, err := io.Copy(fout, fin)
+	if err != nil {
+		fout.Close()
+		t.Fatalf("copy failed: %v", err)
+	}
+
+	if fi.Size() != n {
+		t.Fatalf("incorrect number of bytes copied: wanted: %d, got: %d", fi.Size(), n)
+	}
+
+	err = fout.Close()
+	if err != nil {
+		t.Fatalf("can't close fout on remote host: %v", err)
+	}
+}
+
 func deriveBindhost(t *testing.T) string {
 	t.Helper()

@ -514,12 +662,15 @@ func TestDeriveBindhost(t *testing.T) {
 - [scripts-user, once-per-instance]

 users:
-  - name: ts
-    plain_text_passwd: hunter2
-    groups: [ wheel ]
-    sudo: [ "ALL=(ALL) NOPASSWD:ALL" ]
-    shell: /bin/sh
-    ssh-authorized-keys:
+ - name: root
+   ssh-authorized-keys:
+    - {{.SSHKey}}
+ - name: ts
+   plain_text_passwd: {{.Password}}
+   groups: [ wheel ]
+   sudo: [ "ALL=(ALL) NOPASSWD:ALL" ]
+   shell: /bin/sh
+   ssh-authorized-keys:
    - {{.SSHKey}}

 write_files:
@ -531,7 +682,5 @@ func TestDeriveBindhost(t *testing.T) {

 runcmd:
 {{.InstallPre}}
- - [ "sh", "-c", "curl https://raw.githubusercontent.com/tailscale/tailscale/Xe/test-install-script-libvirtd/scripts/installer.sh | sh" ]
- - [ systemctl, enable, --now, tailscaled.service ]
 - [ curl, "{{.HostURL}}/myip/{{.Port}}", "-H", "User-Agent: {{.Hostname}}" ]
 `