From 843afe7c537a6633af9955619b3218f02cbc29eb Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Sun, 28 Apr 2024 10:42:10 -0500 Subject: [PATCH] ssh/tailssh: add integration test Updates tailscale/corp#11854 Signed-off-by: Percy Wegmann --- .gitignore | 1 + Makefile | 12 + ssh/tailssh/incubator.go | 20 +- ssh/tailssh/tailssh_integration_test.go | 393 ++++++++++++++++++++++++ ssh/tailssh/testcontainers/Dockerfile | 18 ++ 5 files changed, 443 insertions(+), 1 deletion(-) create mode 100644 ssh/tailssh/tailssh_integration_test.go create mode 100644 ssh/tailssh/testcontainers/Dockerfile diff --git a/.gitignore b/.gitignore index bea5627bc..8cd5cdf98 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ cmd/tailscale/tailscale cmd/tailscaled/tailscaled +ssh/tailssh/testcontainers/tailscaled # Test binary, built with `go test -c` *.test diff --git a/Makefile b/Makefile index 31c4fd80d..20ee97e9b 100644 --- a/Makefile +++ b/Makefile @@ -108,6 +108,18 @@ publishdevnameserver: ## Build and publish k8s-nameserver image to location spec @test "${REPO}" != "ghcr.io/tailscale/k8s-nameserver" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-nameserver" && exit 1) TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-nameserver ./build_docker.sh +.PHONY: sshintegrationtest +sshintegrationtest: ## Run the SSH integration tests in various Docker containers + @GOOS=linux GOARCH=amd64 go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \ + GOOS=linux GOARCH=amd64 go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \ + echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \ + echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \ + echo "Testing on ubuntu:mantic" && docker build --build-arg="BASE=ubuntu:mantic" -t ssh-ubuntu-mantic ssh/tailssh/testcontainers && \ + echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers && \ + echo "Testing on fedora:38" && docker build --build-arg="BASE=dokken/fedora-38" -t ssh-fedora-38 ssh/tailssh/testcontainers && \ + echo "Testing on fedora:39" && docker build --build-arg="BASE=dokken/fedora-39" -t ssh-fedora-39 ssh/tailssh/testcontainers && \ + echo "Testing on fedora:40" && docker build --build-arg="BASE=dokken/fedora-40" -t ssh-fedora-40 ssh/tailssh/testcontainers + help: ## Show this help @echo "\nSpecify a command. The choices are:\n" @grep -hE '^[0-9a-zA-Z_-]+:.*?## .*$$' ${MAKEFILE_LIST} | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[0;36m%-20s\033[m %s\n", $$1, $$2}' diff --git a/ssh/tailssh/incubator.go b/ssh/tailssh/incubator.go index 266e4e518..547995e56 100644 --- a/ssh/tailssh/incubator.go +++ b/ssh/tailssh/incubator.go @@ -26,6 +26,7 @@ "sort" "strconv" "strings" + "sync/atomic" "syscall" "github.com/creack/pty" @@ -115,6 +116,10 @@ func (ss *sshSession) newIncubatorCommand() (cmd *exec.Cmd) { "--tty-name=", // updated in-place by startWithPTY } + if debugTest.Load() { + incubatorArgs = append(incubatorArgs, "--debug-test") + } + if isSFTP { incubatorArgs = append(incubatorArgs, "--sftp") } else { @@ -146,7 +151,8 @@ func (ss *sshSession) newIncubatorCommand() (cmd *exec.Cmd) { return exec.CommandContext(ss.ctx, ss.conn.srv.tailscaledPath, incubatorArgs...) } -const debugIncubator = false +var debugIncubator bool +var debugTest atomic.Bool type stdRWC struct{} @@ -177,6 +183,7 @@ type incubatorArgs struct { isShell bool loginCmdPath string cmdArgs []string + debugTest bool } func parseIncubatorArgs(args []string) (a incubatorArgs) { @@ -193,6 +200,7 @@ func parseIncubatorArgs(args []string) (a incubatorArgs) { flags.BoolVar(&a.isShell, "shell", false, "is launching a shell (with no cmds)") flags.BoolVar(&a.isSFTP, "sftp", false, "run sftp server (cmd is ignored)") flags.StringVar(&a.loginCmdPath, "login-cmd", "", "the path to `login` cmd") + flags.BoolVar(&a.debugTest, "debug-test", false, "should debug in test mode") flags.Parse(args) a.cmdArgs = flags.Args() return a @@ -229,6 +237,16 @@ func beIncubator(args []string) error { if sl, err := syslog.New(syslog.LOG_INFO|syslog.LOG_DAEMON, "tailscaled-ssh"); err == nil { logf = log.New(sl, "", 0).Printf } + } else if ia.debugTest { + // In testing, we don't always have syslog, log to a temp file + if logFile, err := os.OpenFile("/tmp/tailscalessh.log", os.O_APPEND|os.O_WRONLY, 0666); err == nil { + lf := log.New(logFile, "", 0) + logf = func(msg string, args ...any) { + lf.Printf(msg, args...) + logFile.Sync() + } + defer logFile.Close() + } } euid := os.Geteuid() diff --git a/ssh/tailssh/tailssh_integration_test.go b/ssh/tailssh/tailssh_integration_test.go new file mode 100644 index 000000000..888158d68 --- /dev/null +++ b/ssh/tailssh/tailssh_integration_test.go @@ -0,0 +1,393 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build integrationtest +// +build integrationtest + +package tailssh + +import ( + "bufio" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "fmt" + "io" + "log" + "net" + "net/http" + "net/netip" + "os" + "os/exec" + "runtime" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/pkg/sftp" + gossh "github.com/tailscale/golang-x-crypto/ssh" + "golang.org/x/crypto/ssh" + "tailscale.com/net/tsdial" + "tailscale.com/tailcfg" + "tailscale.com/types/key" + "tailscale.com/types/netmap" +) + +// This file contains integration tests of the SSH functionality. These tests +// exercise everything except for the authentication logic. +// +// The tests make the following assumptions about the environment: +// +// - OS is one of MacOS or Linux +// - Test is being run as root (e.g. go test -tags integrationtest -c . && sudo ./tailssh.test -test.run TestIntegration) +// - TAILSCALED_PATH environment variable points at tailscaled binary +// - User "testuser" exists +// - "testuser" is in groups "groupone" and "grouptwo" + +func TestMain(m *testing.M) { + // Create our log file. + file, err := os.OpenFile("/tmp/tailscalessh.log", os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + log.Fatal(err) + } + file.Close() + + // Tail our log file. + cmd := exec.Command("tail", "-f", "/tmp/tailscalessh.log") + + r, err := cmd.StdoutPipe() + if err != nil { + return + } + + scanner := bufio.NewScanner(r) + go func() { + for scanner.Scan() { + line := scanner.Text() + log.Println(line) + } + }() + + err = cmd.Start() + if err != nil { + return + } + + m.Run() +} + +func TestIntegrationSSH(t *testing.T) { + debugTest.Store(true) + t.Cleanup(func() { + debugTest.Store(false) + }) + + homeDir := "/home/testuser" + if runtime.GOOS == "darwin" { + homeDir = "/Users/testuser" + } + + tests := []struct { + cmd string + want []string + }{ + { + cmd: "id", + want: []string{"testuser", "groupone", "grouptwo"}, + }, + { + cmd: "pwd", + want: []string{homeDir}, + }, + } + + for _, test := range tests { + // run every test both without and with a shell + for _, shell := range []bool{false, true} { + shellQualifier := "no_shell" + if shell { + shellQualifier = "shell" + } + + t.Run(fmt.Sprintf("%s_%s", test.cmd, shellQualifier), func(t *testing.T) { + s := testSession(t) + + if shell { + err := s.RequestPty("xterm", 40, 80, ssh.TerminalModes{ + ssh.ECHO: 1, + ssh.TTY_OP_ISPEED: 14400, + ssh.TTY_OP_OSPEED: 14400, + }) + if err != nil { + t.Fatalf("unable to request shell: %s", err) + } + } + + got := s.run(t, test.cmd) + for _, want := range test.want { + if !strings.Contains(got, want) { + t.Errorf("%q does not contain %q", got, want) + } + } + }) + } + } +} + +func TestIntegrationSFTP(t *testing.T) { + debugTest.Store(true) + t.Cleanup(func() { + debugTest.Store(false) + }) + + filePath := "/tmp/sftptest.dat" + wantText := "hello world" + + cl := testClient(t) + scl, err := sftp.NewClient(cl) + if err != nil { + t.Fatalf("can't get sftp client: %s", err) + } + + file, err := scl.Create(filePath) + if err != nil { + t.Fatalf("can't create file: %s", err) + } + _, err = file.Write([]byte(wantText)) + if err != nil { + t.Fatalf("can't write to file: %s", err) + } + err = file.Close() + if err != nil { + t.Fatalf("can't close file: %s", err) + } + + file, err = scl.OpenFile(filePath, os.O_RDONLY) + if err != nil { + t.Fatalf("can't open file: %s", err) + } + defer file.Close() + gotText, err := io.ReadAll(file) + if err != nil { + t.Fatalf("can't read file: %s", err) + } + if diff := cmp.Diff(string(gotText), wantText); diff != "" { + t.Fatalf("unexpected file contents (-got +want):\n%s", diff) + } + + s := testSessionFor(t, cl) + got := s.run(t, "ls -l "+filePath) + if !strings.Contains(got, "testuser") { + t.Fatalf("unexpected file owner user: %s", got) + } else if !strings.Contains(got, "testuser") { + t.Fatalf("unexpected file owner group: %s", got) + } +} + +type session struct { + *ssh.Session + + stdin io.WriteCloser + stdout io.ReadCloser + stderr io.ReadCloser +} + +func (s *session) run(t *testing.T, cmdString string) string { + t.Helper() + + err := s.Start(cmdString) + if err != nil { + t.Fatalf("unable to start command: %s", err) + } + + ch := make(chan []byte) + go func() { + for { + b := make([]byte, 1) + n, err := s.stdout.Read(b) + if n > 0 { + ch <- b + } + if err == io.EOF { + return + } + } + }() + + // Read first byte in blocking fashion. + _got := <-ch + + // Read subsequent bytes in non-blocking fashion. +readLoop: + for { + select { + case b := <-ch: + _got = append(_got, b...) + case <-time.After(25 * time.Millisecond): + break readLoop + } + } + + return string(_got) +} + +func testClient(t *testing.T) *ssh.Client { + t.Helper() + + username := "testuser" + srv := &server{ + lb: &testBackend{localUser: username}, + logf: log.Printf, + tailscaledPath: os.Getenv("TAILSCALED_PATH"), + timeNow: time.Now, + } + + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { l.Close() }) + + go func() { + conn, err := l.Accept() + if err == nil { + go srv.HandleSSHConn(&addressFakingConn{conn}) + } + }() + + cl, err := ssh.Dial("tcp", l.Addr().String(), &ssh.ClientConfig{ + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + }) + if err != nil { + log.Fatal(err) + } + t.Cleanup(func() { cl.Close() }) + + return cl +} + +func testSession(t *testing.T) *session { + cl := testClient(t) + return testSessionFor(t, cl) +} + +func testSessionFor(t *testing.T, cl *ssh.Client) *session { + s, err := cl.NewSession() + if err != nil { + log.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + + stdinReader, stdinWriter := io.Pipe() + stdoutReader, stdoutWriter := io.Pipe() + stderrReader, stderrWriter := io.Pipe() + s.Stdin = stdinReader + s.Stdout = io.MultiWriter(stdoutWriter, os.Stdout) + s.Stderr = io.MultiWriter(stderrWriter, os.Stderr) + return &session{ + Session: s, + stdin: stdinWriter, + stdout: stdoutReader, + stderr: stderrReader, + } +} + +// testBackend implements ipnLocalBackend +type testBackend struct { + localUser string +} + +func (tb *testBackend) GetSSH_HostKeys() ([]gossh.Signer, error) { + var result []gossh.Signer + for _, typ := range []string{"ed25519", "ecdsa", "rsa"} { + var priv any + var err error + switch typ { + case "ed25519": + _, priv, err = ed25519.GenerateKey(rand.Reader) + case "ecdsa": + curve := elliptic.P256() + priv, err = ecdsa.GenerateKey(curve, rand.Reader) + case "rsa": + const keySize = 2048 + priv, err = rsa.GenerateKey(rand.Reader, keySize) + } + if err != nil { + return nil, err + } + mk, err := x509.MarshalPKCS8PrivateKey(priv) + if err != nil { + return nil, err + } + hostKey := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: mk}) + signer, err := gossh.ParsePrivateKey(hostKey) + if err != nil { + return nil, err + } + result = append(result, signer) + } + return result, nil +} + +func (tb *testBackend) ShouldRunSSH() bool { + return true +} + +func (tb *testBackend) NetMap() *netmap.NetworkMap { + return &netmap.NetworkMap{ + SSHPolicy: &tailcfg.SSHPolicy{ + Rules: []*tailcfg.SSHRule{ + &tailcfg.SSHRule{ + Principals: []*tailcfg.SSHPrincipal{{Any: true}}, + Action: &tailcfg.SSHAction{Accept: true}, + SSHUsers: map[string]string{"*": tb.localUser}, + }, + }, + }, + } +} + +func (tb *testBackend) WhoIs(ipp netip.AddrPort) (n tailcfg.NodeView, u tailcfg.UserProfile, ok bool) { + return (&tailcfg.Node{}).View(), tailcfg.UserProfile{ + LoginName: tb.localUser + "@example.com", + }, true +} + +func (tb *testBackend) DoNoiseRequest(req *http.Request) (*http.Response, error) { + return nil, nil +} + +func (tb *testBackend) Dialer() *tsdial.Dialer { + return nil +} + +func (tb *testBackend) TailscaleVarRoot() string { + return "" +} + +func (tb *testBackend) NodeKey() key.NodePublic { + return key.NodePublic{} +} + +type addressFakingConn struct { + net.Conn +} + +func (conn *addressFakingConn) LocalAddr() net.Addr { + return &net.TCPAddr{ + IP: net.ParseIP("100.100.100.101"), + Port: 22, + } +} + +func (conn *addressFakingConn) RemoteAddr() net.Addr { + return &net.TCPAddr{ + IP: net.ParseIP("100.100.100.102"), + Port: 10002, + } +} diff --git a/ssh/tailssh/testcontainers/Dockerfile b/ssh/tailssh/testcontainers/Dockerfile new file mode 100644 index 000000000..7832a5853 --- /dev/null +++ b/ssh/tailssh/testcontainers/Dockerfile @@ -0,0 +1,18 @@ +ARG BASE +FROM ${BASE} + +RUN groupadd -g 10000 groupone +RUN groupadd -g 10001 grouptwo +RUN useradd -g 10000 -G 10001 -u 10002 -m testuser +COPY . . + +# First run tests normally. +RUN TAILSCALED_PATH=`pwd`tailscaled ./tailssh.test -test.run TestIntegration + +# Then remove the login command and make sure tests still pass. +RUN rm `which login` +RUN TAILSCALED_PATH=`pwd`tailscaled ./tailssh.test -test.run TestIntegration + +# Then run tests as non-root user testuser. +RUN chown testuser:groupone /tmp/tailscalessh.log +RUN TAILSCALED_PATH=`pwd`tailscaled su -m testuser -c "./tailssh.test -test.run TestIntegration"