cmd/containerboot: make a tests table, add more tests.

Also fix a bugs found while adding the tests, oops.

Signed-off-by: David Anderson <danderson@tailscale.com>

cmd/containerboot/kube.go

@@ -18,15 +18,14 @@
 	"log"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strings"
-	"sync"
 	"time"
 )
 
 // findKeyInKubeSecret inspects the kube secret secretName for a data
 // field called "authkey", and returns its value if present.
 func findKeyInKubeSecret(ctx context.Context, secretName string) (string, error) {
-	kubeOnce.Do(initKube)
 	req, err := http.NewRequest("GET", fmt.Sprintf("/api/v1/namespaces/%s/secrets/%s", kubeNamespace, secretName), nil)
 	if err != nil {
 		return "", err
@@ -68,8 +67,6 @@ func findKeyInKubeSecret(ctx context.Context, secretName string) (string, error)
 // storeDeviceID writes deviceID into the "device_id" data field of
 // the kube secret secretName.
 func storeDeviceID(ctx context.Context, secretName, deviceID string) error {
-	kubeOnce.Do(initKube)
-
 	// First check if the secret exists at all. Even if running on
 	// kubernetes, we do not necessarily store state in a k8s secret.
 	req, err := http.NewRequest("GET", fmt.Sprintf("/api/v1/namespaces/%s/secrets/%s", kubeNamespace, secretName), nil)
@@ -109,7 +106,6 @@ func storeDeviceID(ctx context.Context, secretName, deviceID string) error {
 // deleteAuthKey deletes the 'authkey' field of the given kube
 // secret. No-op if there is no authkey in the secret.
 func deleteAuthKey(ctx context.Context, secretName string) error {
-	kubeOnce.Do(initKube)
 	// m is a JSON Patch data structure, see https://jsonpatch.com/ or RFC 6902.
 	m := []struct {
 		Op   string `json:"op"`
@@ -141,14 +137,13 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
 }
 
 var (
-	kubeOnce      sync.Once
 	kubeHost      string
 	kubeNamespace string
 	kubeToken     string
 	kubeHTTP      *http.Transport
 )
 
-func initKube() {
+func initKube(root string) {
 	// If running in Kubernetes, set things up so that doKubeRequest
 	// can talk successfully to the kube apiserver.
 	if os.Getenv("KUBERNETES_SERVICE_HOST") == "" {
@@ -157,19 +152,19 @@ func initKube() {
 
 	kubeHost = os.Getenv("KUBERNETES_SERVICE_HOST") + ":" + os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")
 
-	bs, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
+	bs, err := os.ReadFile(filepath.Join(root, "var/run/secrets/kubernetes.io/serviceaccount/namespace"))
 	if err != nil {
 		log.Fatalf("Error reading kube namespace: %v", err)
 	}
 	kubeNamespace = strings.TrimSpace(string(bs))
 
-	bs, err = os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
+	bs, err = os.ReadFile(filepath.Join(root, "var/run/secrets/kubernetes.io/serviceaccount/token"))
 	if err != nil {
 		log.Fatalf("Error reading kube token: %v", err)
 	}
 	kubeToken = strings.TrimSpace(string(bs))
 
-	bs, err = os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
+	bs, err = os.ReadFile(filepath.Join(root, "var/run/secrets/kubernetes.io/serviceaccount/ca.crt"))
 	if err != nil {
 		log.Fatalf("Error reading kube CA cert: %v", err)
 	}
@@ -185,7 +180,6 @@ func initKube() {
 
 // doKubeRequest sends r to the kube apiserver.
 func doKubeRequest(ctx context.Context, r *http.Request) (*http.Response, error) {
-	kubeOnce.Do(initKube)
 	if kubeHTTP == nil {
 		panic("not in kubernetes")
 	}

cmd/containerboot/main.go

@@ -52,6 +52,7 @@
 	"os"
 	"os/exec"
 	"os/signal"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"syscall"
@@ -81,6 +82,7 @@ func main() {
 		HTTPProxyAddr:   defaultEnv("TS_OUTBOUND_HTTP_PROXY_LISTEN", ""),
 		Socket:          defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"),
 		AuthOnce:        defaultBool("TS_AUTH_ONCE", false),
+		Root:            defaultEnv("TS_TEST_ONLY_ROOT", "/"),
 	}
 
 	if cfg.ProxyTo != "" && cfg.UserspaceMode {
@@ -88,11 +90,11 @@ func main() {
 	}
 
 	if !cfg.UserspaceMode {
-		if err := ensureTunFile(); err != nil {
+		if err := ensureTunFile(cfg.Root); err != nil {
 			log.Fatalf("Unable to create tuntap device file: %v", err)
 		}
 		if cfg.ProxyTo != "" || cfg.Routes != "" {
-			if err := ensureIPForwarding(cfg.ProxyTo, strings.Split(cfg.Routes, ",")); err != nil {
+			if err := ensureIPForwarding(cfg.Root, cfg.ProxyTo, cfg.Routes); err != nil {
 				log.Printf("Failed to enable IP forwarding: %v", err)
 				log.Printf("To run tailscale as a proxy or router container, IP forwarding must be enabled.")
 				if cfg.InKubernetes {
@@ -104,6 +106,10 @@ func main() {
 		}
 	}
 
+	if cfg.InKubernetes {
+		initKube(cfg.Root)
+	}
+
 	// Context is used for all setup stuff until we're in steady
 	// state, so that if something is hanging we eventually time out
 	// and crashloop the container.
@@ -209,10 +215,12 @@ func startAndAuthTailscaled(ctx context.Context, cfg *settings) (*ipnstate.Statu
 		break
 	}
 
+	didLogin := false
 	if !cfg.AuthOnce {
 		if err := tailscaleUp(ctx, cfg); err != nil {
 			return nil, 0, fmt.Errorf("couldn't log in: %v", err)
 		}
+		didLogin = true
 	}
 
 	tsClient := tailscale.LocalClient{
@@ -243,17 +251,20 @@ func startAndAuthTailscaled(ctx context.Context, cfg *settings) (*ipnstate.Statu
 			}
 			log.Printf("No Tailscale IPs assigned yet")
 		case "NeedsLogin":
+			if !didLogin {
 				// Alas, we cannot currently trigger an authkey login from
 				// LocalAPI, so we still have to shell out to the
 				// tailscale CLI for this bit.
 				if err := tailscaleUp(ctx, cfg); err != nil {
 					return nil, 0, fmt.Errorf("couldn't log in: %v", err)
 				}
+				didLogin = true
+			}
 		default:
 			log.Printf("tailscaled in state %q, waiting", st.BackendState)
 		}
 
-		time.Sleep(500 * time.Millisecond)
+		time.Sleep(100 * time.Millisecond)
 	}
 }
 
@@ -271,7 +282,7 @@ func tailscaledArgs(cfg *settings) []string {
 
 	if cfg.UserspaceMode {
 		args = append(args, "--tun=userspace-networking")
-	} else if err := ensureTunFile(); err != nil {
+	} else if err := ensureTunFile(cfg.Root); err != nil {
 		log.Fatalf("ensuring that /dev/net/tun exists: %v", err)
 	}
 
@@ -316,17 +327,17 @@ func tailscaleUp(ctx context.Context, cfg *settings) error {
 
 // ensureTunFile checks that /dev/net/tun exists, creating it if
 // missing.
-func ensureTunFile() error {
+func ensureTunFile(root string) error {
 	// Verify that /dev/net/tun exists, in some container envs it
 	// needs to be mknod-ed.
-	if _, err := os.Stat("/dev/net"); errors.Is(err, fs.ErrNotExist) {
+	if _, err := os.Stat(filepath.Join(root, "dev/net")); errors.Is(err, fs.ErrNotExist) {
-		if err := os.MkdirAll("/dev/net", 0755); err != nil {
+		if err := os.MkdirAll(filepath.Join(root, "dev/net"), 0755); err != nil {
 			return err
 		}
 	}
-	if _, err := os.Stat("/dev/net/tun"); errors.Is(err, fs.ErrNotExist) {
+	if _, err := os.Stat(filepath.Join(root, "dev/net/tun")); errors.Is(err, fs.ErrNotExist) {
 		dev := unix.Mkdev(10, 200) // tuntap major and minor
-		if err := unix.Mknod("/dev/net/tun", 0600|unix.S_IFCHR, int(dev)); err != nil {
+		if err := unix.Mknod(filepath.Join(root, "dev/net/tun"), 0600|unix.S_IFCHR, int(dev)); err != nil {
 			return err
 		}
 	}
@@ -334,10 +345,11 @@ func ensureTunFile() error {
 }
 
 // ensureIPForwarding enables IPv4/IPv6 forwarding for the container.
-func ensureIPForwarding(proxyTo string, routes []string) error {
+func ensureIPForwarding(root, proxyTo, routes string) error {
 	var (
 		v4Forwarding, v6Forwarding bool
 	)
+	if proxyTo != "" {
 		proxyIP, err := netip.ParseAddr(proxyTo)
 		if err != nil {
 			return fmt.Errorf("invalid proxy destination IP: %v", err)
@@ -347,7 +359,9 @@ func ensureIPForwarding(proxyTo string, routes []string) error {
 		} else {
 			v6Forwarding = true
 		}
-	for _, route := range routes {
+	}
+	if routes != "" {
+		for _, route := range strings.Split(routes, ",") {
 			cidr, err := netip.ParsePrefix(route)
 			if err != nil {
 				return fmt.Errorf("invalid subnet route: %v", err)
@@ -358,13 +372,14 @@ func ensureIPForwarding(proxyTo string, routes []string) error {
 				v6Forwarding = true
 			}
 		}
+	}
 
 	var paths []string
 	if v4Forwarding {
-		paths = append(paths, "/proc/sys/net/ipv4/ip_forward")
+		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv4/ip_forward"))
 	}
 	if v6Forwarding {
-		paths = append(paths, "/proc/sys/net/ipv6/conf/all/forwarding")
+		paths = append(paths, filepath.Join(root, "proc/sys/net/ipv6/conf/all/forwarding"))
 	}
 
 	// In some common configurations (e.g. default docker,
@@ -432,6 +447,7 @@ type settings struct {
 	HTTPProxyAddr   string
 	Socket          string
 	AuthOnce        bool
+	Root            string
 }
 
 // defaultEnv returns the value of the given envvar name, or defVal if

cmd/containerboot/main_test.go

@@ -23,20 +23,20 @@
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"golang.org/x/sys/unix"
 	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/tailcfg"
 )
 
 func TestContainerBoot(t *testing.T) {
-	d, err := os.MkdirTemp("", "containerboot")
+	d := t.TempDir()
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	lapi := localAPI{FSRoot: d}
 	if err := lapi.Start(); err != nil {
@@ -50,17 +50,39 @@ func TestContainerBoot(t *testing.T) {
 	}
 	defer kube.Close()
 
-	for _, path := range []string{"var/lib", "usr/bin", "tmp"} {
+	dirs := []string{
+		"var/lib",
+		"usr/bin",
+		"tmp",
+		"dev/net",
+		"proc/sys/net/ipv4",
+		"proc/sys/net/ipv6/conf/all",
+	}
+	for _, path := range dirs {
 		if err := os.MkdirAll(filepath.Join(d, path), 0700); err != nil {
 			t.Fatal(err)
 		}
 	}
-	if err := os.WriteFile(filepath.Join(d, "usr/bin/tailscaled"), fakeTailscaled, 0700); err != nil {
+	files := map[string][]byte{
+		"usr/bin/tailscaled":                    fakeTailscaled,
+		"usr/bin/tailscale":                     fakeTailscale,
+		"usr/bin/iptables":                      fakeTailscale,
+		"usr/bin/ip6tables":                     fakeTailscale,
+		"dev/net/tun":                           []byte(""),
+		"proc/sys/net/ipv4/ip_forward":          []byte("0"),
+		"proc/sys/net/ipv6/conf/all/forwarding": []byte("0"),
+	}
+	resetFiles := func() {
+		for path, content := range files {
+			// Making everything executable is a little weird, but the
+			// stuff that doesn't need to be executable doesn't care if we
+			// do make it executable.
+			if err := os.WriteFile(filepath.Join(d, path), content, 0700); err != nil {
 				t.Fatal(err)
 			}
-	if err := os.WriteFile(filepath.Join(d, "usr/bin/tailscale"), fakeTailscale, 0700); err != nil {
-		t.Fatal(err)
 		}
+	}
+	resetFiles()
 
 	boot := filepath.Join(d, "containerboot")
 	if err := exec.Command("go", "build", "-o", boot, "tailscale.com/cmd/containerboot").Run(); err != nil {
@@ -68,18 +90,326 @@ func TestContainerBoot(t *testing.T) {
 	}
 
 	argFile := filepath.Join(d, "args")
+	tsIPs := []netip.Addr{netip.MustParseAddr("100.64.0.1")}
+	runningSockPath := filepath.Join(d, "tmp/tailscaled.sock")
 
+	// TODO: refactor this 1-2 stuff if we ever need a third
+	// step. Right now all of containerboot's modes either converge
+	// with no further interaction needed, or with one extra step
+	// only.
+	tests := []struct {
+		Name           string
+		Env            map[string]string
+		KubeSecret     map[string]string
+		WantArgs1      []string        // Wait for containerboot to run these commands...
+		Status1        ipnstate.Status // ... then report this status in LocalAPI.
+		WantArgs2      []string        // If non-nil, wait for containerboot to run these additional commands...
+		Status2        ipnstate.Status // ... then report this status in LocalAPI.
+		WantKubeSecret map[string]string
+		WantFiles      map[string]string
+	}{
+		{
+			// Out of the box default: runs in userspace mode, ephemeral storage, interactive login.
+			Name: "no_args",
+			Env:  nil,
+			WantArgs1: []string{
+				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
+			},
+			// The tailscale up call blocks until auth is complete, so
+			// by the time it returns the next converged state is
+			// Running.
+			Status1: ipnstate.Status{
+				BackendState: "Running",
+				TailscaleIPs: tsIPs,
+			},
+		},
+		{
+			// Userspace mode, ephemeral storage, authkey provided on every run.
+			Name: "authkey",
+			Env: map[string]string{
+				"TS_AUTH_KEY": "tskey-key",
+			},
+			WantArgs1: []string{
+				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+			},
+			Status1: ipnstate.Status{
+				BackendState: "Running",
+				TailscaleIPs: tsIPs,
+			},
+		},
+		{
+			Name: "authkey_disk_state",
+			Env: map[string]string{
+				"TS_AUTH_KEY":  "tskey-key",
+				"TS_STATE_DIR": filepath.Join(d, "tmp"),
+			},
+			WantArgs1: []string{
+				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=/tmp --tun=userspace-networking",
+				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+			},
+			Status1: ipnstate.Status{
+				BackendState: "Running",
+				TailscaleIPs: tsIPs,
+			},
+		},
+		{
+			Name: "routes",
+			Env: map[string]string{
+				"TS_AUTH_KEY": "tskey-key",
+				"TS_ROUTES":   "1.2.3.0/24,10.20.30.0/24",
+			},
+			WantArgs1: []string{
+				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=1.2.3.0/24,10.20.30.0/24",
+			},
+			Status1: ipnstate.Status{
+				BackendState: "Running",
+				TailscaleIPs: tsIPs,
+			},
+			WantFiles: map[string]string{
+				"proc/sys/net/ipv4/ip_forward":          "0",
+				"proc/sys/net/ipv6/conf/all/forwarding": "0",
+			},
+		},
+		{
+			Name: "routes_kernel_ipv4",
+			Env: map[string]string{
+				"TS_AUTH_KEY":  "tskey-key",
+				"TS_ROUTES":    "1.2.3.0/24,10.20.30.0/24",
+				"TS_USERSPACE": "false",
+			},
+			WantArgs1: []string{
+				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
+				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=1.2.3.0/24,10.20.30.0/24",
+			},
+			Status1: ipnstate.Status{
+				BackendState: "Running",
+				TailscaleIPs: tsIPs,
+			},
+			WantFiles: map[string]string{
+				"proc/sys/net/ipv4/ip_forward":          "1",
+				"proc/sys/net/ipv6/conf/all/forwarding": "0",
+			},
+		},
+		{
+			Name: "routes_kernel_ipv6",
+			Env: map[string]string{
+				"TS_AUTH_KEY":  "tskey-key",
+				"TS_ROUTES":    "::/64,1::/64",
+				"TS_USERSPACE": "false",
+			},
+			WantArgs1: []string{
+				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
+				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=::/64,1::/64",
+			},
+			Status1: ipnstate.Status{
+				BackendState: "Running",
+				TailscaleIPs: tsIPs,
+			},
+			WantFiles: map[string]string{
+				"proc/sys/net/ipv4/ip_forward":          "0",
+				"proc/sys/net/ipv6/conf/all/forwarding": "1",
+			},
+		},
+		{
+			Name: "routes_kernel_all_families",
+			Env: map[string]string{
+				"TS_AUTH_KEY":  "tskey-key",
+				"TS_ROUTES":    "::/64,1.2.3.0/24",
+				"TS_USERSPACE": "false",
+			},
+			WantArgs1: []string{
+				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
+				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key --advertise-routes=::/64,1.2.3.0/24",
+			},
+			Status1: ipnstate.Status{
+				BackendState: "Running",
+				TailscaleIPs: tsIPs,
+			},
+			WantFiles: map[string]string{
+				"proc/sys/net/ipv4/ip_forward":          "1",
+				"proc/sys/net/ipv6/conf/all/forwarding": "1",
+			},
+		},
+		{
+			Name: "proxy",
+			Env: map[string]string{
+				"TS_AUTH_KEY":  "tskey-key",
+				"TS_DEST_IP":   "1.2.3.4",
+				"TS_USERSPACE": "false",
+			},
+			WantArgs1: []string{
+				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp",
+				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+			},
+			Status1: ipnstate.Status{
+				BackendState: "Running",
+				TailscaleIPs: tsIPs,
+			},
+			WantArgs2: []string{
+				"/usr/bin/iptables -t nat -I PREROUTING 1 -d 100.64.0.1 -j DNAT --to-destination 1.2.3.4",
+			},
+			Status2: ipnstate.Status{
+				BackendState: "Running",
+				TailscaleIPs: tsIPs,
+			},
+		},
+		{
+			Name: "authkey_once",
+			Env: map[string]string{
+				"TS_AUTH_KEY":  "tskey-key",
+				"TS_AUTH_ONCE": "true",
+			},
+			WantArgs1: []string{
+				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+			},
+			Status1: ipnstate.Status{
+				BackendState: "NeedsLogin",
+			},
+			WantArgs2: []string{
+				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+			},
+			Status2: ipnstate.Status{
+				BackendState: "Running",
+				TailscaleIPs: tsIPs,
+			},
+		},
+		{
+			Name: "kube_storage",
+			Env: map[string]string{
+				"KUBERNETES_SERVICE_HOST":       kube.Host,
+				"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port,
+			},
+			KubeSecret: map[string]string{
+				"authkey": "tskey-key",
+			},
+			WantArgs1: []string{
+				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
+				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+			},
+			Status1: ipnstate.Status{
+				BackendState: "Running",
+				TailscaleIPs: tsIPs,
+				Self: &ipnstate.PeerStatus{
+					ID: tailcfg.StableNodeID("myID"),
+				},
+			},
+			WantKubeSecret: map[string]string{
+				"authkey":   "tskey-key",
+				"device_id": "myID",
+			},
+		},
+		{
+			// Same as previous, but deletes the authkey from the kube secret.
+			Name: "kube_storage_auth_once",
+			Env: map[string]string{
+				"KUBERNETES_SERVICE_HOST":       kube.Host,
+				"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port,
+				"TS_AUTH_ONCE":                  "true",
+			},
+			KubeSecret: map[string]string{
+				"authkey": "tskey-key",
+			},
+			WantArgs1: []string{
+				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
+			},
+			Status1: ipnstate.Status{
+				BackendState: "NeedsLogin",
+			},
+			WantArgs2: []string{
+				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
+			},
+			Status2: ipnstate.Status{
+				BackendState: "Running",
+				TailscaleIPs: tsIPs,
+				Self: &ipnstate.PeerStatus{
+					ID: tailcfg.StableNodeID("myID"),
+				},
+			},
+			WantKubeSecret: map[string]string{
+				"device_id": "myID",
+			},
+		},
+		{
+			Name: "proxies",
+			Env: map[string]string{
+				"TS_SOCKS5_SERVER":              "localhost:1080",
+				"TS_OUTBOUND_HTTP_PROXY_LISTEN": "localhost:8080",
+			},
+			WantArgs1: []string{
+				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --socks5-server=localhost:1080 --outbound-http-proxy-listen=localhost:8080",
+				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
+			},
+			// The tailscale up call blocks until auth is complete, so
+			// by the time it returns the next converged state is
+			// Running.
+			Status1: ipnstate.Status{
+				BackendState: "Running",
+				TailscaleIPs: tsIPs,
+			},
+		},
+		{
+			Name: "dns",
+			Env: map[string]string{
+				"TS_ACCEPT_DNS": "true",
+			},
+			WantArgs1: []string{
+				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
+				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=true",
+			},
+			Status1: ipnstate.Status{
+				BackendState: "Running",
+				TailscaleIPs: tsIPs,
+			},
+		},
+		{
+			Name: "extra_args",
+			Env: map[string]string{
+				"TS_EXTRA_ARGS":            "--widget=rotated",
+				"TS_TAILSCALED_EXTRA_ARGS": "--experiments=widgets",
+			},
+			WantArgs1: []string{
+				"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking --experiments=widgets",
+				"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --widget=rotated",
+			},
+			Status1: ipnstate.Status{
+				BackendState: "Running",
+				TailscaleIPs: tsIPs,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
 			lapi.Reset()
 			kube.Reset()
+			os.Remove(argFile)
+			os.Remove(runningSockPath)
+			resetFiles()
+
+			for k, v := range test.KubeSecret {
+				kube.SetSecret(k, v)
+			}
 
 			cmd := exec.Command(boot)
 			cmd.Env = []string{
 				fmt.Sprintf("PATH=%s/usr/bin:%s", d, os.Getenv("PATH")),
 				fmt.Sprintf("TS_TEST_RECORD_ARGS=%s", argFile),
 				fmt.Sprintf("TS_TEST_SOCKET=%s", lapi.Path),
-		fmt.Sprintf("TS_SOCKET=%s", filepath.Join(d, "tmp/tailscaled.sock")),
+				fmt.Sprintf("TS_SOCKET=%s", runningSockPath),
+				fmt.Sprintf("TS_TEST_ONLY_ROOT=%s", d),
+			}
+			for k, v := range test.Env {
+				cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
 			}
 			cbOut := &lockingBuffer{}
+			defer func() {
+				if t.Failed() {
+					t.Logf("containerboot output:\n%s", cbOut.String())
+				}
+			}()
 			cmd.Stderr = cbOut
 			if err := cmd.Start(); err != nil {
 				t.Fatalf("starting containerboot: %v", err)
@@ -89,20 +419,37 @@ func TestContainerBoot(t *testing.T) {
 				cmd.Process.Wait()
 			}()
 
-	want := `
+			waitArgs(t, 2*time.Second, d, argFile, strings.Join(test.WantArgs1, "\n"))
-/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking
+			lapi.SetStatus(test.Status1)
-/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false
+			if test.WantArgs2 != nil {
-`
+				waitArgs(t, 2*time.Second, d, argFile, strings.Join(append(test.WantArgs1, test.WantArgs2...), "\n"))
-	waitArgs(t, 2*time.Second, d, argFile, want)
+				lapi.SetStatus(test.Status2)
-
+			}
-	lapi.SetStatus(ipnstate.Status{
-		BackendState: "Running",
-		TailscaleIPs: []netip.Addr{
-			netip.MustParseAddr("100.64.0.1"),
-		},
-	})
-
 			waitLogLine(t, 2*time.Second, cbOut, "Startup complete, waiting for shutdown signal")
+
+			if test.WantKubeSecret != nil {
+				got := kube.Secret()
+				if diff := cmp.Diff(got, test.WantKubeSecret); diff != "" {
+					t.Fatalf("unexpected kube secret data (-got+want):\n%s", diff)
+				}
+			} else {
+				got := kube.Secret()
+				if len(got) != 0 {
+					t.Fatalf("kube secret unexpectedly not empty, got %#v", got)
+				}
+			}
+
+			for path, want := range test.WantFiles {
+				gotBs, err := os.ReadFile(filepath.Join(d, path))
+				if err != nil {
+					t.Fatalf("reading wanted file %q: %v", path, err)
+				}
+				if got := strings.TrimSpace(string(gotBs)); got != want {
+					t.Errorf("wrong file contents for %q, got %q want %q", path, got, want)
+				}
+			}
+		})
+	}
 }
 
 type lockingBuffer struct {
@@ -260,7 +607,7 @@ func (l *localAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 // that something unexpected happened.
 type kubeServer struct {
 	FSRoot     string
-	Addr   string // populated by Start
+	Host, Port string // populated by Start
 
 	srv *httptest.Server
 
@@ -305,7 +652,8 @@ func (k *kubeServer) Start() error {
 	}
 
 	k.srv = httptest.NewTLSServer(k)
-	k.Addr = k.srv.Listener.Addr().String()
+	k.Host = k.srv.Listener.Addr().(*net.TCPAddr).IP.String()
+	k.Port = strconv.Itoa(k.srv.Listener.Addr().(*net.TCPAddr).Port)
 
 	var cert bytes.Buffer
 	if err := pem.Encode(&cert, &pem.Block{Type: "CERTIFICATE", Bytes: k.srv.Certificate().Raw}); err != nil {

cmd/containerboot/test_tailscale.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 #
-# This is a fake tailscale CLI that records its arguments and exits successfully.
+# This is a fake tailscale CLI (and also iptables and ip6tables) that
+# records its arguments and exits successfully.
 #
 # It is used by main_test.go to test the behavior of containerboot.