Make simple initial test case

This commit makes the initial SSH test a bit simpler:

- Use the same pattern/functions for all clients as other tests
- Only test within _one_ namespace/user to confirm the base case
- Use retry function, same as taildrop, there is some funky going on
  there...

Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>

integration/cli_test.go

@@ -8,6 +8,7 @@ import (
 
 	v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
 	"github.com/juanfont/headscale/integration/hsic"
+	"github.com/juanfont/headscale/integration/tsic"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -37,7 +38,7 @@ func TestNamespaceCommand(t *testing.T) {
 		"namespace2": 0,
 	}
 
-	err = scenario.CreateHeadscaleEnv(spec, hsic.WithTestName("clins"))
+	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("clins"))
 	assert.NoError(t, err)
 
 	headscale, err := scenario.Headscale()
@@ -118,7 +119,7 @@ func TestPreAuthKeyCommand(t *testing.T) {
 		namespace: 0,
 	}
 
-	err = scenario.CreateHeadscaleEnv(spec, hsic.WithTestName("clipak"))
+	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("clipak"))
 	assert.NoError(t, err)
 
 	headscale, err := scenario.Headscale()
@@ -258,7 +259,7 @@ func TestPreAuthKeyCommandWithoutExpiry(t *testing.T) {
 		namespace: 0,
 	}
 
-	err = scenario.CreateHeadscaleEnv(spec, hsic.WithTestName("clipaknaexp"))
+	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("clipaknaexp"))
 	assert.NoError(t, err)
 
 	headscale, err := scenario.Headscale()
@@ -323,7 +324,7 @@ func TestPreAuthKeyCommandReusableEphemeral(t *testing.T) {
 		namespace: 0,
 	}
 
-	err = scenario.CreateHeadscaleEnv(spec, hsic.WithTestName("clipakresueeph"))
+	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("clipakresueeph"))
 	assert.NoError(t, err)
 
 	headscale, err := scenario.Headscale()

integration/general_test.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/juanfont/headscale/integration/hsic"
+	"github.com/juanfont/headscale/integration/tsic"
 	"github.com/rs/zerolog/log"
 )
 
@@ -24,7 +25,7 @@ func TestPingAllByIP(t *testing.T) {
 		"namespace2": len(TailscaleVersions),
 	}
 
-	err = scenario.CreateHeadscaleEnv(spec, hsic.WithTestName("pingallbyip"))
+	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("pingallbyip"))
 	if err != nil {
 		t.Errorf("failed to create headscale environment: %s", err)
 	}
@@ -80,7 +81,7 @@ func TestPingAllByHostname(t *testing.T) {
 		"namespace4": len(TailscaleVersions) - 1,
 	}
 
-	err = scenario.CreateHeadscaleEnv(spec, hsic.WithTestName("pingallbyname"))
+	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("pingallbyname"))
 	if err != nil {
 		t.Errorf("failed to create headscale environment: %s", err)
 	}
@@ -148,7 +149,7 @@ func TestTaildrop(t *testing.T) {
 		"taildrop": len(TailscaleVersions) - 1,
 	}
 
-	err = scenario.CreateHeadscaleEnv(spec, hsic.WithTestName("taildrop"))
+	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("taildrop"))
 	if err != nil {
 		t.Errorf("failed to create headscale environment: %s", err)
 	}
@@ -276,7 +277,7 @@ func TestResolveMagicDNS(t *testing.T) {
 		"magicdns2": len(TailscaleVersions) - 1,
 	}
 
-	err = scenario.CreateHeadscaleEnv(spec, hsic.WithTestName("magicdns"))
+	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("magicdns"))
 	if err != nil {
 		t.Errorf("failed to create headscale environment: %s", err)
 	}

integration/scenario.go

@@ -229,6 +229,7 @@ func (s *Scenario) CreateTailscaleNodesInNamespace(
 	namespaceStr string,
 	requestedVersion string,
 	count int,
+	opts ...tsic.Option,
 ) error {
 	if namespace, ok := s.namespaces[namespaceStr]; ok {
 		for i := 0; i < count; i++ {
@@ -247,6 +248,11 @@ func (s *Scenario) CreateTailscaleNodesInNamespace(
 
 			namespace.createWaitGroup.Add(1)
 
+			opts = append(opts,
+				tsic.WithHeadscaleTLS(cert),
+				tsic.WithHeadscaleName(hostname),
+			)
+
 			go func() {
 				defer namespace.createWaitGroup.Done()
 
@@ -255,8 +261,7 @@ func (s *Scenario) CreateTailscaleNodesInNamespace(
 					s.pool,
 					version,
 					s.network,
-					tsic.WithHeadscaleTLS(cert),
+					opts...,
-					tsic.WithHeadscaleName(hostname),
 				)
 				if err != nil {
 					// return fmt.Errorf("failed to add tailscale node: %w", err)
@@ -341,7 +346,11 @@ func (s *Scenario) WaitForTailscaleSync() error {
 // CreateHeadscaleEnv is a conventient method returning a set up Headcale
 // test environment with nodes of all versions, joined to the server with X
 // namespaces.
-func (s *Scenario) CreateHeadscaleEnv(namespaces map[string]int, opts ...hsic.Option) error {
+func (s *Scenario) CreateHeadscaleEnv(
+	namespaces map[string]int,
+	tsOpts []tsic.Option,
+	opts ...hsic.Option,
+) error {
 	headscale, err := s.Headscale(opts...)
 	if err != nil {
 		return err
@@ -353,7 +362,7 @@ func (s *Scenario) CreateHeadscaleEnv(namespaces map[string]int, opts ...hsic.Op
 			return err
 		}
 
-		err = s.CreateTailscaleNodesInNamespace(namespaceName, "all", clientCount)
+		err = s.CreateTailscaleNodesInNamespace(namespaceName, "all", clientCount, tsOpts...)
 		if err != nil {
 			return err
 		}

integration/ssh_test.go

@@ -2,14 +2,29 @@ package integration
 
 import (
 	"fmt"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/juanfont/headscale"
 )
 
-func TestSSHIntoAll(t *testing.T) {
+func TestSSHOneNamespaceAllToAll(t *testing.T) {
 	IntegrationSkip(t)
 
+	retry := func(times int, sleepInverval time.Duration, doWork func() (string, error)) (string, error) {
+		var err error
+		for attempts := 0; attempts < times; attempts++ {
+			result, err := doWork()
+			if err == nil {
+				return result, nil
+			}
+			time.Sleep(sleepInverval)
+		}
+
+		return "", err
+	}
+
 	scenario, err := NewScenario()
 	if err != nil {
 		t.Errorf("failed to create scenario: %s", err)
@@ -17,14 +32,12 @@ func TestSSHIntoAll(t *testing.T) {
 
 	spec := &HeadscaleSpec{
 		namespaces: map[string]int{
-			// Omit versions before 1.24 because they don't support SSH
+			"namespace1": len(TailscaleVersions) - 5,
-			"namespace1": len(TailscaleVersions) - 4,
-			"namespace2": len(TailscaleVersions) - 4,
 		},
 		enableSSH: true,
 		acl: &headscale.ACLPolicy{
 			Groups: map[string][]string{
-				"group:integration-test": {"namespace1", "namespace2"},
+				"group:integration-test": {"namespace1"},
 			},
 			ACLs: []headscale.ACL{
 				{
@@ -48,68 +61,73 @@ func TestSSHIntoAll(t *testing.T) {
 	if err != nil {
 		t.Errorf("failed to create headscale environment: %s", err)
 	}
+
+	allClients, err := scenario.ListTailscaleClients()
+	if err != nil {
+		t.Errorf("failed to get clients: %s", err)
+	}
+
 	err = scenario.WaitForTailscaleSync()
 	if err != nil {
 		t.Errorf("failed wait for tailscale clients to be in sync: %s", err)
 	}
 
-	for namespace := range spec.namespaces {
+	_, err = scenario.ListTailscaleClientsFQDNs()
-		// This will essentially fetch and cache all the FQDNs for the given namespace
+	if err != nil {
-		nsFQDNs, err := scenario.ListTailscaleClientsFQDNs(namespace)
+		t.Errorf("failed to get FQDNs: %s", err)
-		if err != nil {
+	}
-			t.Errorf("failed to get FQDNs: %s", err)
-		}
 
-		nsClients, err := scenario.ListTailscaleClients(namespace)
+	success := 0
-		if err != nil {
-			t.Errorf("failed to get clients: %s", err)
-		}
 
-		for _, client := range nsClients {
+	for _, client := range allClients {
-			currentClientFqdn, _ := client.FQDN()
+		for _, peer := range allClients {
-			sshTargets := removeFromSlice(nsFQDNs, currentClientFqdn)
+			if client.Hostname() == peer.Hostname() {
-
+				continue
-			for _, target := range sshTargets {
-				t.Run(
-					fmt.Sprintf("%s-%s", currentClientFqdn, target),
-					func(t *testing.T) {
-						command := []string{
-							"ssh", "-o StrictHostKeyChecking=no",
-							fmt.Sprintf("%s@%s", "ssh-it-user", target),
-							"'hostname'",
-						}
-
-						result, err := client.Execute(command)
-						if err != nil {
-							t.Errorf("failed to execute command over SSH: %s", err)
-						}
-
-						if result != target {
-							t.Logf("result=%s, target=%s", result, target)
-							t.Fail()
-						}
-
-						t.Logf("Result for %s: %s\n", target, result)
-					},
-				)
 			}
 
-			// t.Logf("%s wants to SSH into %+v", currentClientFqdn, sshTargets)
+			clientFQDN, _ := client.FQDN()
+			peerFQDN, _ := peer.FQDN()
+
+			t.Run(
+				fmt.Sprintf("%s-%s", clientFQDN, peerFQDN),
+				func(t *testing.T) {
+					command := []string{
+						"ssh", "-o StrictHostKeyChecking=no", "-o ConnectTimeout=1",
+						fmt.Sprintf("%s@%s", "ssh-it-user", peer.Hostname()),
+						"'hostname'",
+					}
+
+					result, err := retry(10, 1*time.Second, func() (string, error) {
+						return client.Execute(command)
+					})
+					if err != nil {
+						t.Errorf("failed to execute command over SSH: %s", err)
+					}
+
+					if strings.Contains(peer.ID(), result) {
+						t.Logf(
+							"failed to get correct container ID from %s, expected: %s, got: %s",
+							peer.Hostname(),
+							peer.ID(),
+							result,
+						)
+						t.Fail()
+					} else {
+						success++
+					}
+				},
+			)
 		}
 	}
 
+	t.Logf(
+		"%d successful pings out of %d",
+		success,
+		(len(allClients)*len(allClients))-len(allClients),
+	)
+
 	err = scenario.Shutdown()
 	if err != nil {
 		t.Errorf("failed to tear down scenario: %s", err)
 	}
 }
-
-func removeFromSlice(haystack []string, needle string) []string {
-	for i, value := range haystack {
-		if needle == value {
-			return append(haystack[:i], haystack[i+1:]...)
-		}
-	}
-
-	return haystack
-}

integration/tsic/tsic.go

@@ -47,6 +47,7 @@ type TailscaleInContainer struct {
 	// optional config
 	headscaleCert     []byte
 	headscaleHostname string
+	withSSH           bool
 }
 
 type Option = func(c *TailscaleInContainer)
@@ -83,6 +84,12 @@ func WithHeadscaleName(hsName string) Option {
 	}
 }
 
+func WithSSH() Option {
+	return func(tsic *TailscaleInContainer) {
+		tsic.withSSH = true
+	}
+}
+
 func New(
 	pool *dockertest.Pool,
 	version string,
@@ -219,6 +226,10 @@ func (t *TailscaleInContainer) Up(
 		t.hostname,
 	}
 
+	if t.withSSH {
+		command = append(command, "--ssh")
+	}
+
 	if _, _, err := t.Execute(command); err != nil {
 		return fmt.Errorf("failed to join tailscale client: %w", err)
 	}