introduce a version subset we must test against

Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>

cmd/build-docker-img/main.go

@@ -21,7 +21,7 @@ func main() {
 		log.Fatalf("failed to create or get network: %s", err)
 	}
 
-	for _, version := range integration.TailscaleVersions {
+	for _, version := range integration.AllVersions {
 		log.Printf("creating container image for Tailscale (%s)", version)
 
 		tsClient, err := tsic.New(

integration/auth_oidc_test.go

@@ -50,7 +50,7 @@ func TestOIDCAuthenticationPingAll(t *testing.T) {
 	defer scenario.Shutdown()
 
 	spec := map[string]int{
-		"user1": len(TailscaleVersions),
+		"user1": len(MustTestVersions),
 	}
 
 	oidcConfig, err := scenario.runMockOIDC(defaultAccessTTL)
@@ -105,7 +105,7 @@ func TestOIDCExpireNodesBasedOnTokenExpiry(t *testing.T) {
 	defer scenario.Shutdown()
 
 	spec := map[string]int{
-		"user1": len(TailscaleVersions),
+		"user1": len(MustTestVersions),
 	}
 
 	oidcConfig, err := scenario.runMockOIDC(shortAccessTTL)

integration/auth_web_flow_test.go

@@ -37,8 +37,8 @@ func TestAuthWebFlowAuthenticationPingAll(t *testing.T) {
 	defer scenario.Shutdown()
 
 	spec := map[string]int{
-		"user1": len(TailscaleVersions),
+		"user1": len(MustTestVersions),
-		"user2": len(TailscaleVersions),
+		"user2": len(MustTestVersions),
 	}
 
 	err = scenario.CreateHeadscaleEnv(spec, hsic.WithTestName("webauthping"))
@@ -74,8 +74,8 @@ func TestAuthWebFlowLogoutAndRelogin(t *testing.T) {
 	defer scenario.Shutdown()
 
 	spec := map[string]int{
-		"user1": len(TailscaleVersions),
+		"user1": len(MustTestVersions),
-		"user2": len(TailscaleVersions),
+		"user2": len(MustTestVersions),
 	}
 
 	err = scenario.CreateHeadscaleEnv(spec, hsic.WithTestName("weblogout"))

integration/embedded_derp_test.go

@@ -33,7 +33,7 @@ func TestDERPServerScenario(t *testing.T) {
 	defer scenario.Shutdown()
 
 	spec := map[string]int{
-		"user1": len(TailscaleVersions),
+		"user1": len(MustTestVersions),
 	}
 
 	headscaleConfig := map[string]string{}
@@ -164,7 +164,7 @@ func (s *EmbeddedDERPServerScenario) CreateTailscaleIsolatedNodesInUser(
 
 			version := requestedVersion
 			if requestedVersion == "all" {
-				version = TailscaleVersions[clientN%len(TailscaleVersions)]
+				version = MustTestVersions[clientN%len(MustTestVersions)]
 			}
 
 			cert := hsServer.GetCert()

integration/general_test.go

@@ -25,8 +25,8 @@ func TestPingAllByIP(t *testing.T) {
 	defer scenario.Shutdown()
 
 	spec := map[string]int{
-		"user1": len(TailscaleVersions),
+		"user1": len(MustTestVersions),
-		"user2": len(TailscaleVersions),
+		"user2": len(MustTestVersions),
 	}
 
 	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("pingallbyip"))
@@ -58,8 +58,8 @@ func TestAuthKeyLogoutAndRelogin(t *testing.T) {
 	defer scenario.Shutdown()
 
 	spec := map[string]int{
-		"user1": len(TailscaleVersions),
+		"user1": len(MustTestVersions),
-		"user2": len(TailscaleVersions),
+		"user2": len(MustTestVersions),
 	}
 
 	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("pingallbyip"))
@@ -165,8 +165,8 @@ func TestEphemeral(t *testing.T) {
 	defer scenario.Shutdown()
 
 	spec := map[string]int{
-		"user1": len(TailscaleVersions),
+		"user1": len(MustTestVersions),
-		"user2": len(TailscaleVersions),
+		"user2": len(MustTestVersions),
 	}
 
 	headscale, err := scenario.Headscale(hsic.WithTestName("ephemeral"))
@@ -249,8 +249,8 @@ func TestPingAllByHostname(t *testing.T) {
 
 	spec := map[string]int{
 		// Omit 1.16.2 (-1) because it does not have the FQDN field
-		"user3": len(TailscaleVersions) - 1,
+		"user3": len(MustTestVersions) - 1,
-		"user4": len(TailscaleVersions) - 1,
+		"user4": len(MustTestVersions) - 1,
 	}
 
 	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("pingallbyname"))
@@ -297,7 +297,7 @@ func TestTaildrop(t *testing.T) {
 
 	spec := map[string]int{
 		// Omit 1.16.2 (-1) because it does not have the FQDN field
-		"taildrop": len(TailscaleVersions) - 1,
+		"taildrop": len(MustTestVersions) - 1,
 	}
 
 	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("taildrop"))
@@ -408,8 +408,8 @@ func TestResolveMagicDNS(t *testing.T) {
 
 	spec := map[string]int{
 		// Omit 1.16.2 (-1) because it does not have the FQDN field
-		"magicdns1": len(TailscaleVersions) - 1,
+		"magicdns1": len(MustTestVersions) - 1,
-		"magicdns2": len(TailscaleVersions) - 1,
+		"magicdns2": len(MustTestVersions) - 1,
 	}
 
 	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("magicdns"))
@@ -474,7 +474,7 @@ func TestExpireNode(t *testing.T) {
 	defer scenario.Shutdown()
 
 	spec := map[string]int{
-		"user1": len(TailscaleVersions),
+		"user1": len(MustTestVersions),
 	}
 
 	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("expirenode"))
@@ -501,7 +501,7 @@ func TestExpireNode(t *testing.T) {
 		assertNoErr(t, err)
 
 		// Assert that we have the original count - self
-		assert.Len(t, status.Peers(), len(TailscaleVersions)-1)
+		assert.Len(t, status.Peers(), len(MustTestVersions)-1)
 	}
 
 	headscale, err := scenario.Headscale()
@@ -536,7 +536,7 @@ func TestExpireNode(t *testing.T) {
 
 		if client.Hostname() != machine.Name {
 			// Assert that we have the original count - self - expired node
-			assert.Len(t, status.Peers(), len(TailscaleVersions)-2)
+			assert.Len(t, status.Peers(), len(MustTestVersions)-2)
 		}
 	}
 }

integration/scenario.go

@@ -49,7 +49,7 @@ var (
 	tailscaleVersions2019 = []string{
 		"1.28",
 		"1.26",
-		"1.24",
+		"1.24", // Tailscale SSH
 		"1.22",
 		"1.20",
 		"1.18",
@@ -63,7 +63,7 @@ var (
 	// "1.8.7",
 	// }.
 
-	// TailscaleVersions represents a list of Tailscale versions the suite
+	// AllVersions represents a list of Tailscale versions the suite
 	// uses to test compatibility with the ControlServer.
 	//
 	// The list contains two special cases, "head" and "unstable" which
@@ -72,10 +72,21 @@ var (
 	//
 	// The rest of the version represents Tailscale versions that can be
 	// found in Tailscale's apt repository.
-	TailscaleVersions = append(
+	AllVersions = append(
 		tailscaleVersions2021,
 		tailscaleVersions2019...,
 	)
+
+	// MustTestVersions is the minimum set of versions we should test.
+	// At the moment, this is arbitrarily choosen as:
+	//
+	// - Two unstable (HEAD and unstable)
+	// - Two latest versions
+	// - Two oldest versions
+	MustTestVersions = append(
+		tailscaleVersions2021[0:4],
+		tailscaleVersions2019[len(tailscaleVersions2019)-2:len(tailscaleVersions2019)]...,
+	)
 )
 
 // User represents a User in the ControlServer and a map of TailscaleClient's
@@ -277,7 +288,7 @@ func (s *Scenario) CreateTailscaleNodesInUser(
 		for i := 0; i < count; i++ {
 			version := requestedVersion
 			if requestedVersion == "all" {
-				version = TailscaleVersions[i%len(TailscaleVersions)]
+				version = MustTestVersions[i%len(MustTestVersions)]
 			}
 
 			headscale, err := s.Headscale()

integration/ssh_test.go

@@ -102,7 +102,7 @@ func TestSSHOneUserAllToAll(t *testing.T) {
 				},
 			},
 		},
-		len(TailscaleVersions)-5,
+		len(MustTestVersions)-2,
 	)
 	defer scenario.Shutdown()
 
@@ -151,7 +151,7 @@ func TestSSHMultipleUsersAllToAll(t *testing.T) {
 				},
 			},
 		},
-		len(TailscaleVersions)-5,
+		len(MustTestVersions)-2,
 	)
 	defer scenario.Shutdown()
 
@@ -197,7 +197,7 @@ func TestSSHNoSSHConfigured(t *testing.T) {
 			},
 			SSHs: []policy.SSH{},
 		},
-		len(TailscaleVersions)-5,
+		len(MustTestVersions)-2,
 	)
 	defer scenario.Shutdown()
 
@@ -246,7 +246,7 @@ func TestSSHIsBlockedInACL(t *testing.T) {
 				},
 			},
 		},
-		len(TailscaleVersions)-5,
+		len(MustTestVersions)-2,
 	)
 	defer scenario.Shutdown()
 
@@ -302,7 +302,7 @@ func TestSSUserOnlyIsolation(t *testing.T) {
 				},
 			},
 		},
-		len(TailscaleVersions)-5,
+		len(MustTestVersions)-2,
 	)
 	defer scenario.Shutdown()