Reuse Ping function for DERP ping

fixed linting

.github/workflows/test-integration-v2-TestDERPServerScenario.yaml

@@ -0,0 +1,57 @@
+# DO NOT EDIT, generated with cmd/gh-action-integration-generator/main.go
+# To regenerate, run "go generate" in cmd/gh-action-integration-generator/
+
+name: Integration Test v2 - TestDERPServerScenario
+
+on: [pull_request]
+
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v34
+        with:
+          files: |
+            *.nix
+            go.*
+            **/*.go
+            integration_test/
+            config-example.yaml
+
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
+
+      - name: Run general integration tests
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: |
+            nix develop --command -- docker run \
+              --tty --rm \
+              --volume ~/.cache/hs-integration-go:/go \
+              --name headscale-test-suite \
+              --volume $PWD:$PWD -w $PWD/integration \
+              --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
+              golang:1 \
+                go test ./... \
+                  -tags ts2019 \
+                  -failfast \
+                  -timeout 120m \
+                  -parallel 1 \
+                  -run "^TestDERPServerScenario$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

integration/control.go

@@ -2,12 +2,14 @@ package integration
 
 import (
 	v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
+	"github.com/ory/dockertest/v3"
 )
 
 type ControlServer interface {
 	Shutdown() error
 	SaveLog(string) error
 	Execute(command []string) (string, error)
+	ConnectToNetwork(network *dockertest.Network) error
 	GetHealthEndpoint() string
 	GetEndpoint() string
 	WaitForReady() error

integration/embedded_derp_test.go

@@ -0,0 +1,236 @@
+package integration
+
+import (
+	"fmt"
+	"log"
+	"net/url"
+	"testing"
+
+	"github.com/juanfont/headscale"
+	"github.com/juanfont/headscale/integration/dockertestutil"
+	"github.com/juanfont/headscale/integration/hsic"
+	"github.com/juanfont/headscale/integration/tsic"
+	"github.com/ory/dockertest/v3"
+)
+
+type EmbeddedDERPServerScenario struct {
+	*Scenario
+
+	tsicNetworks map[string]*dockertest.Network
+}
+
+func TestDERPServerScenario(t *testing.T) {
+	IntegrationSkip(t)
+	// t.Parallel()
+
+	baseScenario, err := NewScenario()
+	if err != nil {
+		t.Errorf("failed to create scenario: %s", err)
+	}
+
+	scenario := EmbeddedDERPServerScenario{
+		Scenario:     baseScenario,
+		tsicNetworks: map[string]*dockertest.Network{},
+	}
+
+	spec := map[string]int{
+		"user1": len(TailscaleVersions),
+	}
+
+	headscaleConfig := map[string]string{}
+	headscaleConfig["HEADSCALE_DERP_URLS"] = ""
+	headscaleConfig["HEADSCALE_DERP_SERVER_ENABLED"] = "true"
+	headscaleConfig["HEADSCALE_DERP_SERVER_REGION_ID"] = "999"
+	headscaleConfig["HEADSCALE_DERP_SERVER_REGION_CODE"] = "headscale"
+	headscaleConfig["HEADSCALE_DERP_SERVER_REGION_NAME"] = "Headscale Embedded DERP"
+	headscaleConfig["HEADSCALE_DERP_SERVER_STUN_LISTEN_ADDR"] = "0.0.0.0:3478"
+
+	err = scenario.CreateHeadscaleEnv(
+		spec,
+		hsic.WithConfigEnv(headscaleConfig),
+		hsic.WithTestName("derpserver"),
+		hsic.WithExtraPorts([]string{"3478/udp"}),
+		hsic.WithTLS(),
+		hsic.WithHostnameAsServerURL(),
+	)
+
+	if err != nil {
+		t.Errorf("failed to create headscale environment: %s", err)
+	}
+
+	allClients, err := scenario.ListTailscaleClients()
+	if err != nil {
+		t.Errorf("failed to get clients: %s", err)
+	}
+
+	allIps, err := scenario.ListTailscaleClientsIPs()
+	if err != nil {
+		t.Errorf("failed to get clients: %s", err)
+	}
+
+	err = scenario.WaitForTailscaleSync()
+	if err != nil {
+		t.Errorf("failed wait for tailscale clients to be in sync: %s", err)
+	}
+
+	allHostnames, err := scenario.ListTailscaleClientsFQDNs()
+	if err != nil {
+		t.Errorf("failed to get FQDNs: %s", err)
+	}
+
+	success := pingDerpAllHelper(t, allClients, allHostnames)
+
+	t.Logf("%d successful pings out of %d", success, len(allClients)*len(allIps))
+
+	err = scenario.Shutdown()
+	if err != nil {
+		t.Errorf("failed to tear down scenario: %s", err)
+	}
+}
+
+func (s *EmbeddedDERPServerScenario) CreateHeadscaleEnv(
+	users map[string]int,
+	opts ...hsic.Option,
+) error {
+	hsServer, err := s.Headscale(opts...)
+	if err != nil {
+		return err
+	}
+
+	headscaleEndpoint := hsServer.GetEndpoint()
+	headscaleURL, err := url.Parse(headscaleEndpoint)
+	if err != nil {
+		return err
+	}
+
+	headscaleURL.Host = fmt.Sprintf("%s:%s", hsServer.GetHostname(), headscaleURL.Port())
+
+	err = hsServer.WaitForReady()
+	if err != nil {
+		return err
+	}
+
+	hash, err := headscale.GenerateRandomStringDNSSafe(scenarioHashLength)
+	if err != nil {
+		return err
+	}
+
+	for userName, clientCount := range users {
+		err = s.CreateUser(userName)
+		if err != nil {
+			return err
+		}
+
+		err = s.CreateTailscaleIsolatedNodesInUser(
+			hash,
+			userName,
+			"all",
+			clientCount,
+		)
+		if err != nil {
+			return err
+		}
+
+		key, err := s.CreatePreAuthKey(userName, true, false)
+		if err != nil {
+			return err
+		}
+
+		err = s.RunTailscaleUp(userName, headscaleURL.String(), key.GetKey())
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *EmbeddedDERPServerScenario) CreateTailscaleIsolatedNodesInUser(
+	hash string,
+	userStr string,
+	requestedVersion string,
+	count int,
+	opts ...tsic.Option,
+) error {
+	hsServer, err := s.Headscale()
+	if err != nil {
+		return err
+	}
+
+	if user, ok := s.users[userStr]; ok {
+		for clientN := 0; clientN < count; clientN++ {
+			networkName := fmt.Sprintf("tsnet-%s-%s-%d",
+				hash,
+				userStr,
+				clientN,
+			)
+			network, err := dockertestutil.GetFirstOrCreateNetwork(
+				s.pool,
+				networkName,
+			)
+			if err != nil {
+				return fmt.Errorf("failed to create or get %s network: %w", networkName, err)
+			}
+
+			s.tsicNetworks[networkName] = network
+
+			err = hsServer.ConnectToNetwork(network)
+			if err != nil {
+				return fmt.Errorf("failed to connect headscale to %s network: %w", networkName, err)
+			}
+
+			version := requestedVersion
+			if requestedVersion == "all" {
+				version = TailscaleVersions[clientN%len(TailscaleVersions)]
+			}
+
+			cert := hsServer.GetCert()
+
+			user.createWaitGroup.Add(1)
+
+			opts = append(opts,
+				tsic.WithHeadscaleTLS(cert),
+			)
+
+			go func() {
+				defer user.createWaitGroup.Done()
+
+				// TODO(kradalby): error handle this
+				tsClient, err := tsic.New(
+					s.pool,
+					version,
+					network,
+					opts...,
+				)
+				if err != nil {
+					// return fmt.Errorf("failed to add tailscale node: %w", err)
+					log.Printf("failed to create tailscale node: %s", err)
+				}
+
+				err = tsClient.WaitForReady()
+				if err != nil {
+					// return fmt.Errorf("failed to add tailscale node: %w", err)
+					log.Printf("failed to wait for tailscaled: %s", err)
+				}
+
+				user.Clients[tsClient.Hostname()] = tsClient
+			}()
+		}
+		user.createWaitGroup.Wait()
+
+		return nil
+	}
+
+	return fmt.Errorf("failed to add tailscale node: %w", errNoUserAvailable)
+}
+
+func (s *EmbeddedDERPServerScenario) Shutdown() error {
+	for _, network := range s.tsicNetworks {
+		err := s.pool.RemoveNetwork(network)
+		if err != nil {
+			return err
+		}
+	}
+
+	return s.Scenario.Shutdown()
+}

integration/hsic/hsic.go

@@ -15,6 +15,7 @@ import (
 	"math/big"
 	"net"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/davecgh/go-spew/spew"
@@ -23,6 +24,7 @@ import (
 	"github.com/juanfont/headscale/integration/dockertestutil"
 	"github.com/juanfont/headscale/integration/integrationutil"
 	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
 )
 
 const (
@@ -52,6 +54,8 @@ type HeadscaleInContainer struct {
 
 	// optional config
 	port             int
+	extraPorts       []string
+	hostPortBindings map[string][]string
 	aclPolicy        *headscale.ACLPolicy
 	env              map[string]string
 	tlsCert          []byte
@@ -77,7 +81,7 @@ func WithACLPolicy(acl *headscale.ACLPolicy) Option {
 // WithTLS creates certificates and enables HTTPS.
 func WithTLS() Option {
 	return func(hsic *HeadscaleInContainer) {
-		cert, key, err := createCertificate()
+		cert, key, err := createCertificate(hsic.hostname)
 		if err != nil {
 			log.Fatalf("failed to create certificates for headscale test: %s", err)
 		}
@@ -108,6 +112,19 @@ func WithPort(port int) Option {
 	}
 }
 
+// WithExtraPorts exposes additional ports on the container (e.g. 3478/udp for STUN).
+func WithExtraPorts(ports []string) Option {
+	return func(hsic *HeadscaleInContainer) {
+		hsic.extraPorts = ports
+	}
+}
+
+func WithHostPortBindings(bindings map[string][]string) Option {
+	return func(hsic *HeadscaleInContainer) {
+		hsic.hostPortBindings = bindings
+	}
+}
+
 // WithTestName sets a name for the test, this will be reflected
 // in the Docker container name.
 func WithTestName(testName string) Option {
@@ -173,6 +190,16 @@ func New(
 
 	portProto := fmt.Sprintf("%d/tcp", hsic.port)
 
+	serverURL, err := url.Parse(hsic.env["HEADSCALE_SERVER_URL"])
+	if err != nil {
+		return nil, err
+	}
+
+	if len(hsic.tlsCert) != 0 && len(hsic.tlsKey) != 0 {
+		serverURL.Scheme = "https"
+		hsic.env["HEADSCALE_SERVER_URL"] = serverURL.String()
+	}
+
 	headscaleBuildOptions := &dockertest.BuildOptions{
 		Dockerfile: "Dockerfile.debug",
 		ContextDir: dockerContextPath,
@@ -187,7 +214,7 @@ func New(
 
 	runOptions := &dockertest.RunOptions{
 		Name:         hsic.hostname,
-		ExposedPorts: []string{portProto},
+		ExposedPorts: append([]string{portProto}, hsic.extraPorts...),
 		Networks:     []*dockertest.Network{network},
 		// Cmd:          []string{"headscale", "serve"},
 		// TODO(kradalby): Get rid of this hack, we currently need to give us some
@@ -196,6 +223,18 @@ func New(
 		Env:        env,
 	}
 
+	if len(hsic.hostPortBindings) > 0 {
+		runOptions.PortBindings = map[docker.Port][]docker.PortBinding{}
+		for port, hostPorts := range hsic.hostPortBindings {
+			runOptions.PortBindings[docker.Port(port)] = []docker.PortBinding{}
+			for _, hostPort := range hostPorts {
+				runOptions.PortBindings[docker.Port(port)] = append(
+					runOptions.PortBindings[docker.Port(port)],
+					docker.PortBinding{HostPort: hostPort})
+			}
+		}
+	}
+
 	// dockertest isnt very good at handling containers that has already
 	// been created, this is an attempt to make sure this container isnt
 	// present.
@@ -256,6 +295,10 @@ func New(
 	return hsic, nil
 }
 
+func (t *HeadscaleInContainer) ConnectToNetwork(network *dockertest.Network) error {
+	return t.container.ConnectToNetwork(network)
+}
+
 func (t *HeadscaleInContainer) hasTLS() bool {
 	return len(t.tlsCert) != 0 && len(t.tlsKey) != 0
 }
@@ -456,7 +499,7 @@ func (t *HeadscaleInContainer) WriteFile(path string, data []byte) error {
 }
 
 // nolint
-func createCertificate() ([]byte, []byte, error) {
+func createCertificate(hostname string) ([]byte, []byte, error) {
 	// From:
 	// https://shaneutt.com/blog/golang-ca-and-signed-cert-go/
 
@@ -468,7 +511,7 @@ func createCertificate() ([]byte, []byte, error) {
 			Locality:     []string{"Leiden"},
 		},
 		NotBefore: time.Now(),
-		NotAfter:  time.Now().Add(30 * time.Minute),
+		NotAfter:  time.Now().Add(60 * time.Minute),
 		IsCA:      true,
 		ExtKeyUsage: []x509.ExtKeyUsage{
 			x509.ExtKeyUsageClientAuth,
@@ -486,16 +529,17 @@ func createCertificate() ([]byte, []byte, error) {
 	cert := &x509.Certificate{
 		SerialNumber: big.NewInt(1658),
 		Subject: pkix.Name{
+			CommonName:   hostname,
 			Organization: []string{"Headscale testing INC"},
 			Country:      []string{"NL"},
 			Locality:     []string{"Leiden"},
 		},
-		IPAddresses:  []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
 		NotBefore:    time.Now(),
-		NotAfter:     time.Now().Add(30 * time.Minute),
+		NotAfter:     time.Now().Add(60 * time.Minute),
 		SubjectKeyId: []byte{1, 2, 3, 4, 6},
 		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
 		KeyUsage:     x509.KeyUsageDigitalSignature,
+		DNSNames:     []string{hostname},
 	}
 
 	certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)

integration/tailscale.go

@@ -4,6 +4,7 @@ import (
 	"net/netip"
 	"net/url"
 
+	"github.com/juanfont/headscale/integration/dockertestutil"
 	"github.com/juanfont/headscale/integration/tsic"
 	"tailscale.com/ipn/ipnstate"
 )
@@ -13,7 +14,7 @@ type TailscaleClient interface {
 	Hostname() string
 	Shutdown() error
 	Version() string
-	Execute(command []string) (string, string, error)
+	Execute(command []string, options ...dockertestutil.ExecuteCommandOption) (string, string, error)
 	Up(loginServer, authKey string) error
 	UpWithLoginURL(loginServer string) (*url.URL, error)
 	Logout() error

integration/tsic/tsic.go

@@ -29,6 +29,7 @@ const (
 
 var (
 	errTailscalePingFailed             = errors.New("ping failed")
+	errTailscalePingNotDERP            = errors.New("ping not via DERP")
 	errTailscaleNotLoggedIn            = errors.New("tailscale not logged in")
 	errTailscaleWrongPeerCount         = errors.New("wrong peer count")
 	errTailscaleCannotUpWithoutAuthkey = errors.New("cannot up without authkey")
@@ -56,6 +57,7 @@ type TailscaleInContainer struct {
 	withSSH           bool
 	withTags          []string
 	withEntrypoint    []string
+	withExtraHosts    []string
 	workdir           string
 }
 
@@ -124,6 +126,12 @@ func WithDockerWorkdir(dir string) Option {
 	}
 }
 
+func WithExtraHosts(hosts []string) Option {
+	return func(tsic *TailscaleInContainer) {
+		tsic.withExtraHosts = hosts
+	}
+}
+
 // WithDockerEntrypoint allows the docker entrypoint of the container
 // to be overridden. This is a dangerous option which can make
 // the container not work as intended as a typo might prevent
@@ -169,11 +177,12 @@ func New(
 
 	tailscaleOptions := &dockertest.RunOptions{
 		Name:     hostname,
-		Networks: []*dockertest.Network{network},
+		Networks: []*dockertest.Network{tsic.network},
 		// Cmd: []string{
 		// 	"tailscaled", "--tun=tsdev",
 		// },
 		Entrypoint: tsic.withEntrypoint,
+		ExtraHosts: tsic.withExtraHosts,
 	}
 
 	if tsic.headscaleHostname != "" {
@@ -248,11 +257,13 @@ func (t *TailscaleInContainer) ID() string {
 // result of stdout as a string.
 func (t *TailscaleInContainer) Execute(
 	command []string,
+	options ...dockertestutil.ExecuteCommandOption,
 ) (string, string, error) {
 	stdout, stderr, err := dockertestutil.ExecuteCommand(
 		t.container,
 		command,
 		[]string{},
+		options...,
 	)
 	if err != nil {
 		log.Printf("command stderr: %s\n", stderr)
@@ -477,7 +488,7 @@ func (t *TailscaleInContainer) WaitForPeers(expected int) error {
 }
 
 type (
-	// PingOption repreent optional settings that can be given
+	// PingOption represent optional settings that can be given
 	// to ping another host.
 	PingOption = func(args *pingArgs)
 
@@ -535,7 +546,12 @@ func (t *TailscaleInContainer) Ping(hostnameOrIP string, opts ...PingOption) err
 	command = append(command, hostnameOrIP)
 
 	return t.pool.Retry(func() error {
-		result, _, err := t.Execute(command)
+		result, _, err := t.Execute(
+			command,
+			dockertestutil.ExecuteCommandTimeout(
+				time.Duration(int64(args.timeout)*int64(args.count)),
+			),
+		)
 		if err != nil {
 			log.Printf(
 				"failed to run ping command from %s to %s, err: %s",
@@ -547,10 +563,22 @@ func (t *TailscaleInContainer) Ping(hostnameOrIP string, opts ...PingOption) err
 			return err
 		}
 
-		if !strings.Contains(result, "pong") && !strings.Contains(result, "is local") {
+		if strings.Contains(result, "is local") {
+			return nil
+		}
+
+		if !strings.Contains(result, "pong") {
 			return backoff.Permanent(errTailscalePingFailed)
 		}
 
+		if !args.direct {
+			if strings.Contains(result, "via DERP") {
+				return nil
+			} else {
+				return backoff.Permanent(errTailscalePingNotDERP)
+			}
+		}
+
 		return nil
 	})
 }

integration/utils.go

@@ -2,6 +2,14 @@ package integration
 
 import (
 	"testing"
+	"time"
+
+	"github.com/juanfont/headscale/integration/tsic"
+)
+
+const (
+	derpPingTimeout = 2 * time.Second
+	derpPingCount   = 10
 )
 
 func pingAllHelper(t *testing.T, clients []TailscaleClient, addrs []string) int {
@@ -22,6 +30,52 @@ func pingAllHelper(t *testing.T, clients []TailscaleClient, addrs []string) int
 	return success
 }
 
+func pingDerpAllHelper(t *testing.T, clients []TailscaleClient, addrs []string) int {
+	t.Helper()
+	success := 0
+
+	for _, client := range clients {
+		for _, addr := range addrs {
+			if isSelfClient(client, addr) {
+				continue
+			}
+
+			err := client.Ping(
+				addr,
+				tsic.WithPingTimeout(derpPingTimeout),
+				tsic.WithPingCount(derpPingCount),
+				tsic.WithPingUntilDirect(false),
+			)
+			if err != nil {
+				t.Errorf("failed to ping %s from %s: %s", addr, client.Hostname(), err)
+			} else {
+				success++
+			}
+		}
+	}
+
+	return success
+}
+
+func isSelfClient(client TailscaleClient, addr string) bool {
+	if addr == client.Hostname() {
+		return true
+	}
+
+	ips, err := client.IPs()
+	if err != nil {
+		return false
+	}
+
+	for _, ip := range ips {
+		if ip.String() == addr {
+			return true
+		}
+	}
+
+	return false
+}
+
 // pingAllNegativeHelper is intended to have 1 or more nodes timeing out from the ping,
 // it counts failures instead of successes.
 // func pingAllNegativeHelper(t *testing.T, clients []TailscaleClient, addrs []string) int {