ssh/tailssh: add integration test

Updates tailscale/corp#11854

Signed-off-by: Percy Wegmann <percy@tailscale.com>

.gitignore

@@ -9,6 +9,7 @@
 
 cmd/tailscale/tailscale
 cmd/tailscaled/tailscaled
+ssh/tailssh/testcontainers/tailscaled
 
 # Test binary, built with `go test -c`
 *.test

Makefile

@@ -108,6 +108,18 @@ publishdevnameserver: ## Build and publish k8s-nameserver image to location spec
 	@test "${REPO}" != "ghcr.io/tailscale/k8s-nameserver" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-nameserver" && exit 1)
 	TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-nameserver ./build_docker.sh
 
+.PHONY: sshintegrationtest
+sshintegrationtest: ## Run the SSH integration tests in various Docker containers
+	@GOOS=linux GOARCH=amd64 go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \
+	GOOS=linux GOARCH=amd64 go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \
+	echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \
+	echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \
+	echo "Testing on ubuntu:mantic" && docker build --build-arg="BASE=ubuntu:mantic" -t ssh-ubuntu-mantic ssh/tailssh/testcontainers && \
+	echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers && \
+	echo "Testing on fedora:38" && docker build --build-arg="BASE=dokken/fedora-38" -t ssh-fedora-38 ssh/tailssh/testcontainers && \
+	echo "Testing on fedora:39" && docker build --build-arg="BASE=dokken/fedora-39" -t ssh-fedora-39 ssh/tailssh/testcontainers && \
+	echo "Testing on fedora:40" && docker build --build-arg="BASE=dokken/fedora-40" -t ssh-fedora-40 ssh/tailssh/testcontainers
+
 help: ## Show this help
 	@echo "\nSpecify a command. The choices are:\n"
 	@grep -hE '^[0-9a-zA-Z_-]+:.*?## .*$$' ${MAKEFILE_LIST} | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[0;36m%-20s\033[m %s\n", $$1, $$2}'

ssh/tailssh/incubator.go

@@ -26,6 +26,7 @@
 	"sort"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"syscall"
 
 	"github.com/creack/pty"
@@ -115,6 +116,10 @@ func (ss *sshSession) newIncubatorCommand() (cmd *exec.Cmd) {
 		"--tty-name=",     // updated in-place by startWithPTY
 	}
 
+	if debugTest.Load() {
+		incubatorArgs = append(incubatorArgs, "--debug-test")
+	}
+
 	if isSFTP {
 		incubatorArgs = append(incubatorArgs, "--sftp")
 	} else {
@@ -146,7 +151,8 @@ func (ss *sshSession) newIncubatorCommand() (cmd *exec.Cmd) {
 	return exec.CommandContext(ss.ctx, ss.conn.srv.tailscaledPath, incubatorArgs...)
 }
 
-const debugIncubator = false
+var debugIncubator bool
+var debugTest atomic.Bool
 
 type stdRWC struct{}
 
@@ -177,6 +183,7 @@ type incubatorArgs struct {
 	isShell      bool
 	loginCmdPath string
 	cmdArgs      []string
+	debugTest    bool
 }
 
 func parseIncubatorArgs(args []string) (a incubatorArgs) {
@@ -193,6 +200,7 @@ func parseIncubatorArgs(args []string) (a incubatorArgs) {
 	flags.BoolVar(&a.isShell, "shell", false, "is launching a shell (with no cmds)")
 	flags.BoolVar(&a.isSFTP, "sftp", false, "run sftp server (cmd is ignored)")
 	flags.StringVar(&a.loginCmdPath, "login-cmd", "", "the path to `login` cmd")
+	flags.BoolVar(&a.debugTest, "debug-test", false, "should debug in test mode")
 	flags.Parse(args)
 	a.cmdArgs = flags.Args()
 	return a
@@ -229,6 +237,16 @@ func beIncubator(args []string) error {
 		if sl, err := syslog.New(syslog.LOG_INFO|syslog.LOG_DAEMON, "tailscaled-ssh"); err == nil {
 			logf = log.New(sl, "", 0).Printf
 		}
+	} else if ia.debugTest {
+		// In testing, we don't always have syslog, log to a temp file
+		if logFile, err := os.OpenFile("/tmp/tailscalessh.log", os.O_APPEND|os.O_WRONLY, 0666); err == nil {
+			lf := log.New(logFile, "", 0)
+			logf = func(msg string, args ...any) {
+				lf.Printf(msg, args...)
+				logFile.Sync()
+			}
+			defer logFile.Close()
+		}
 	}
 
 	euid := os.Geteuid()

ssh/tailssh/tailssh_integration_test.go

@@ -0,0 +1,393 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build integrationtest
+// +build integrationtest
+
+package tailssh
+
+import (
+	"bufio"
+	"crypto/ecdsa"
+	"crypto/ed25519"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/netip"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/pkg/sftp"
+	gossh "github.com/tailscale/golang-x-crypto/ssh"
+	"golang.org/x/crypto/ssh"
+	"tailscale.com/net/tsdial"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
+	"tailscale.com/types/netmap"
+)
+
+// This file contains integration tests of the SSH functionality. These tests
+// exercise everything except for the authentication logic.
+//
+// The tests make the following assumptions about the environment:
+//
+// - OS is one of MacOS or Linux
+// - Test is being run as root (e.g. go test -tags integrationtest -c . && sudo ./tailssh.test -test.run TestIntegration)
+// - TAILSCALED_PATH environment variable points at tailscaled binary
+// - User "testuser" exists
+// - "testuser" is in groups "groupone" and "grouptwo"
+
+func TestMain(m *testing.M) {
+	// Create our log file.
+	file, err := os.OpenFile("/tmp/tailscalessh.log", os.O_CREATE|os.O_WRONLY, 0666)
+	if err != nil {
+		log.Fatal(err)
+	}
+	file.Close()
+
+	// Tail our log file.
+	cmd := exec.Command("tail", "-f", "/tmp/tailscalessh.log")
+
+	r, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+
+	scanner := bufio.NewScanner(r)
+	go func() {
+		for scanner.Scan() {
+			line := scanner.Text()
+			log.Println(line)
+		}
+	}()
+
+	err = cmd.Start()
+	if err != nil {
+		return
+	}
+
+	m.Run()
+}
+
+func TestIntegrationSSH(t *testing.T) {
+	debugTest.Store(true)
+	t.Cleanup(func() {
+		debugTest.Store(false)
+	})
+
+	homeDir := "/home/testuser"
+	if runtime.GOOS == "darwin" {
+		homeDir = "/Users/testuser"
+	}
+
+	tests := []struct {
+		cmd  string
+		want []string
+	}{
+		{
+			cmd:  "id",
+			want: []string{"testuser", "groupone", "grouptwo"},
+		},
+		{
+			cmd:  "pwd",
+			want: []string{homeDir},
+		},
+	}
+
+	for _, test := range tests {
+		// run every test both without and with a shell
+		for _, shell := range []bool{false, true} {
+			shellQualifier := "no_shell"
+			if shell {
+				shellQualifier = "shell"
+			}
+
+			t.Run(fmt.Sprintf("%s_%s", test.cmd, shellQualifier), func(t *testing.T) {
+				s := testSession(t)
+
+				if shell {
+					err := s.RequestPty("xterm", 40, 80, ssh.TerminalModes{
+						ssh.ECHO:          1,
+						ssh.TTY_OP_ISPEED: 14400,
+						ssh.TTY_OP_OSPEED: 14400,
+					})
+					if err != nil {
+						t.Fatalf("unable to request shell: %s", err)
+					}
+				}
+
+				got := s.run(t, test.cmd)
+				for _, want := range test.want {
+					if !strings.Contains(got, want) {
+						t.Errorf("%q does not contain %q", got, want)
+					}
+				}
+			})
+		}
+	}
+}
+
+func TestIntegrationSFTP(t *testing.T) {
+	debugTest.Store(true)
+	t.Cleanup(func() {
+		debugTest.Store(false)
+	})
+
+	filePath := "/tmp/sftptest.dat"
+	wantText := "hello world"
+
+	cl := testClient(t)
+	scl, err := sftp.NewClient(cl)
+	if err != nil {
+		t.Fatalf("can't get sftp client: %s", err)
+	}
+
+	file, err := scl.Create(filePath)
+	if err != nil {
+		t.Fatalf("can't create file: %s", err)
+	}
+	_, err = file.Write([]byte(wantText))
+	if err != nil {
+		t.Fatalf("can't write to file: %s", err)
+	}
+	err = file.Close()
+	if err != nil {
+		t.Fatalf("can't close file: %s", err)
+	}
+
+	file, err = scl.OpenFile(filePath, os.O_RDONLY)
+	if err != nil {
+		t.Fatalf("can't open file: %s", err)
+	}
+	defer file.Close()
+	gotText, err := io.ReadAll(file)
+	if err != nil {
+		t.Fatalf("can't read file: %s", err)
+	}
+	if diff := cmp.Diff(string(gotText), wantText); diff != "" {
+		t.Fatalf("unexpected file contents (-got +want):\n%s", diff)
+	}
+
+	s := testSessionFor(t, cl)
+	got := s.run(t, "ls -l "+filePath)
+	if !strings.Contains(got, "testuser") {
+		t.Fatalf("unexpected file owner user: %s", got)
+	} else if !strings.Contains(got, "testuser") {
+		t.Fatalf("unexpected file owner group: %s", got)
+	}
+}
+
+type session struct {
+	*ssh.Session
+
+	stdin  io.WriteCloser
+	stdout io.ReadCloser
+	stderr io.ReadCloser
+}
+
+func (s *session) run(t *testing.T, cmdString string) string {
+	t.Helper()
+
+	err := s.Start(cmdString)
+	if err != nil {
+		t.Fatalf("unable to start command: %s", err)
+	}
+
+	ch := make(chan []byte)
+	go func() {
+		for {
+			b := make([]byte, 1)
+			n, err := s.stdout.Read(b)
+			if n > 0 {
+				ch <- b
+			}
+			if err == io.EOF {
+				return
+			}
+		}
+	}()
+
+	// Read first byte in blocking fashion.
+	_got := <-ch
+
+	// Read subsequent bytes in non-blocking fashion.
+readLoop:
+	for {
+		select {
+		case b := <-ch:
+			_got = append(_got, b...)
+		case <-time.After(25 * time.Millisecond):
+			break readLoop
+		}
+	}
+
+	return string(_got)
+}
+
+func testClient(t *testing.T) *ssh.Client {
+	t.Helper()
+
+	username := "testuser"
+	srv := &server{
+		lb:             &testBackend{localUser: username},
+		logf:           log.Printf,
+		tailscaledPath: os.Getenv("TAILSCALED_PATH"),
+		timeNow:        time.Now,
+	}
+
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { l.Close() })
+
+	go func() {
+		conn, err := l.Accept()
+		if err == nil {
+			go srv.HandleSSHConn(&addressFakingConn{conn})
+		}
+	}()
+
+	cl, err := ssh.Dial("tcp", l.Addr().String(), &ssh.ClientConfig{
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+	t.Cleanup(func() { cl.Close() })
+
+	return cl
+}
+
+func testSession(t *testing.T) *session {
+	cl := testClient(t)
+	return testSessionFor(t, cl)
+}
+
+func testSessionFor(t *testing.T, cl *ssh.Client) *session {
+	s, err := cl.NewSession()
+	if err != nil {
+		log.Fatal(err)
+	}
+	t.Cleanup(func() { s.Close() })
+
+	stdinReader, stdinWriter := io.Pipe()
+	stdoutReader, stdoutWriter := io.Pipe()
+	stderrReader, stderrWriter := io.Pipe()
+	s.Stdin = stdinReader
+	s.Stdout = io.MultiWriter(stdoutWriter, os.Stdout)
+	s.Stderr = io.MultiWriter(stderrWriter, os.Stderr)
+	return &session{
+		Session: s,
+		stdin:   stdinWriter,
+		stdout:  stdoutReader,
+		stderr:  stderrReader,
+	}
+}
+
+// testBackend implements ipnLocalBackend
+type testBackend struct {
+	localUser string
+}
+
+func (tb *testBackend) GetSSH_HostKeys() ([]gossh.Signer, error) {
+	var result []gossh.Signer
+	for _, typ := range []string{"ed25519", "ecdsa", "rsa"} {
+		var priv any
+		var err error
+		switch typ {
+		case "ed25519":
+			_, priv, err = ed25519.GenerateKey(rand.Reader)
+		case "ecdsa":
+			curve := elliptic.P256()
+			priv, err = ecdsa.GenerateKey(curve, rand.Reader)
+		case "rsa":
+			const keySize = 2048
+			priv, err = rsa.GenerateKey(rand.Reader, keySize)
+		}
+		if err != nil {
+			return nil, err
+		}
+		mk, err := x509.MarshalPKCS8PrivateKey(priv)
+		if err != nil {
+			return nil, err
+		}
+		hostKey := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: mk})
+		signer, err := gossh.ParsePrivateKey(hostKey)
+		if err != nil {
+			return nil, err
+		}
+		result = append(result, signer)
+	}
+	return result, nil
+}
+
+func (tb *testBackend) ShouldRunSSH() bool {
+	return true
+}
+
+func (tb *testBackend) NetMap() *netmap.NetworkMap {
+	return &netmap.NetworkMap{
+		SSHPolicy: &tailcfg.SSHPolicy{
+			Rules: []*tailcfg.SSHRule{
+				&tailcfg.SSHRule{
+					Principals: []*tailcfg.SSHPrincipal{{Any: true}},
+					Action:     &tailcfg.SSHAction{Accept: true},
+					SSHUsers:   map[string]string{"*": tb.localUser},
+				},
+			},
+		},
+	}
+}
+
+func (tb *testBackend) WhoIs(ipp netip.AddrPort) (n tailcfg.NodeView, u tailcfg.UserProfile, ok bool) {
+	return (&tailcfg.Node{}).View(), tailcfg.UserProfile{
+		LoginName: tb.localUser + "@example.com",
+	}, true
+}
+
+func (tb *testBackend) DoNoiseRequest(req *http.Request) (*http.Response, error) {
+	return nil, nil
+}
+
+func (tb *testBackend) Dialer() *tsdial.Dialer {
+	return nil
+}
+
+func (tb *testBackend) TailscaleVarRoot() string {
+	return ""
+}
+
+func (tb *testBackend) NodeKey() key.NodePublic {
+	return key.NodePublic{}
+}
+
+type addressFakingConn struct {
+	net.Conn
+}
+
+func (conn *addressFakingConn) LocalAddr() net.Addr {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("100.100.100.101"),
+		Port: 22,
+	}
+}
+
+func (conn *addressFakingConn) RemoteAddr() net.Addr {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("100.100.100.102"),
+		Port: 10002,
+	}
+}

ssh/tailssh/testcontainers/Dockerfile

@@ -0,0 +1,18 @@
+ARG BASE
+FROM ${BASE}
+
+RUN groupadd -g 10000 groupone
+RUN groupadd -g 10001 grouptwo
+RUN useradd -g 10000 -G 10001 -u 10002 -m testuser
+COPY . .
+
+# First run tests normally.
+RUN TAILSCALED_PATH=`pwd`tailscaled ./tailssh.test -test.run TestIntegration
+
+# Then remove the login command and make sure tests still pass.
+RUN rm `which login`
+RUN TAILSCALED_PATH=`pwd`tailscaled ./tailssh.test -test.run TestIntegration
+
+# Then run tests as non-root user testuser.
+RUN chown testuser:groupone /tmp/tailscalessh.log
+RUN TAILSCALED_PATH=`pwd`tailscaled su -m testuser -c "./tailssh.test -test.run TestIntegration"