remove old release flow

Signed-off-by: Kristoffer Dalby <kristoffer@dalby.cc>

.github/workflows/release-docker.yml

@@ -1,138 +0,0 @@
----
-name: Release Docker
-
-on:
-  push:
-    tags:
-      - "*" # triggers only if push new tag version
-  workflow_dispatch:
-
-jobs:
-  docker-release:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Set up QEMU for multiple platforms
-        uses: docker/setup-qemu-action@master
-        with:
-          platforms: arm64,amd64
-      - name: Cache Docker layers
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v3
-        with:
-          # list of Docker images to use as base name for tags
-          images: |
-            ${{ secrets.DOCKERHUB_USERNAME }}/headscale
-            ghcr.io/${{ github.repository_owner }}/headscale
-          tags: |
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
-            type=sha
-            type=raw,value=develop
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Login to GHCR
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build and push
-        id: docker_build
-        uses: docker/build-push-action@v2
-        with:
-          push: true
-          context: .
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          platforms: linux/amd64,linux/arm64
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache-new
-          build-args: |
-            VERSION=${{ steps.meta.outputs.version }}
-      - name: Prepare cache for next build
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
-
-  docker-debug-release:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Set up QEMU for multiple platforms
-        uses: docker/setup-qemu-action@master
-        with:
-          platforms: arm64,amd64
-      - name: Cache Docker layers
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildx-cache-debug
-          key: ${{ runner.os }}-buildx-debug-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-debug-
-      - name: Docker meta
-        id: meta-debug
-        uses: docker/metadata-action@v3
-        with:
-          # list of Docker images to use as base name for tags
-          images: |
-            ${{ secrets.DOCKERHUB_USERNAME }}/headscale
-            ghcr.io/${{ github.repository_owner }}/headscale
-          flavor: |
-            suffix=-debug,onlatest=true
-          tags: |
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
-            type=sha
-            type=raw,value=develop
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Login to GHCR
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build and push
-        id: docker_build
-        uses: docker/build-push-action@v2
-        with:
-          push: true
-          context: .
-          file: Dockerfile.debug
-          tags: ${{ steps.meta-debug.outputs.tags }}
-          labels: ${{ steps.meta-debug.outputs.labels }}
-          platforms: linux/amd64,linux/arm64
-          cache-from: type=local,src=/tmp/.buildx-cache-debug
-          cache-to: type=local,dest=/tmp/.buildx-cache-debug-new
-          build-args: |
-            VERSION=${{ steps.meta-debug.outputs.version }}
-      - name: Prepare cache for next build
-        run: |
-          rm -rf /tmp/.buildx-cache-debug
-          mv /tmp/.buildx-cache-debug-new /tmp/.buildx-cache-debug

.github/workflows/test-integration-derp.yml

@@ -1,35 +0,0 @@
-name: Integration Test DERP
-
-on: [pull_request]
-
-jobs:
-  integration-test-derp:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-
-      - name: Set Swap Space
-        uses: pierotofy/set-swap-space@master
-        with:
-          swap-size-gb: 10
-
-      - name: Get changed files
-        id: changed-files
-        uses: tj-actions/changed-files@v34
-        with:
-          files: |
-            *.nix
-            go.*
-            **/*.go
-            integration_test/
-            config-example.yaml
-
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
-
-      - name: Run Embedded DERP server integration tests
-        if: steps.changed-files.outputs.any_changed == 'true'
-        run: nix develop --command -- make test_integration_derp

.github/workflows/test-integration-v2-TestACLAllowStarDst.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestACLAllowUser80Dst.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestACLAllowUserDst.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestACLDenyAllPort80.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestACLDevice1CanAccessDevice2.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestACLHostsInNetMapTable.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestACLNamedHostsCanReach.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestACLNamedHostsCanReachBySubnet.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestAuthKeyLogoutAndRelogin.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestAuthWebFlowAuthenticationPingAll.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestAuthWebFlowLogoutAndRelogin.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestCreateTailscale.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestDERPServerScenario.yaml

@@ -0,0 +1,63 @@
+# DO NOT EDIT, generated with cmd/gh-action-integration-generator/main.go
+# To regenerate, run "go generate" in cmd/gh-action-integration-generator/
+
+name: Integration Test v2 - TestDERPServerScenario
+
+on: [pull_request]
+
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v34
+        with:
+          files: |
+            *.nix
+            go.*
+            **/*.go
+            integration_test/
+            config-example.yaml
+
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
+
+      - name: Run general integration tests
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: |
+            nix develop --command -- docker run \
+              --tty --rm \
+              --volume ~/.cache/hs-integration-go:/go \
+              --name headscale-test-suite \
+              --volume $PWD:$PWD -w $PWD/integration \
+              --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
+              golang:1 \
+                go test ./... \
+                  -tags ts2019 \
+                  -failfast \
+                  -timeout 120m \
+                  -parallel 1 \
+                  -run "^TestDERPServerScenario$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestEnablingRoutes.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestEphemeral.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestExpireNode.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestHeadscale.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestOIDCAuthenticationPingAll.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestOIDCExpireNodesBasedOnTokenExpiry.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestPingAllByHostname.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestPingAllByIP.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestPreAuthKeyCommand.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestPreAuthKeyCommandReusableEphemeral.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestPreAuthKeyCommandWithoutExpiry.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestResolveMagicDNS.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestSSHIsBlockedInACL.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestSSHMultipleUsersAllToAll.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestSSHNoSSHConfigured.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestSSHOneUserAllToAll.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestSSUserOnlyIsolation.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestTaildrop.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestTailscaleNodesJoiningHeadcale.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.github/workflows/test-integration-v2-TestUserCommand.yaml

@@ -55,3 +55,9 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

.goreleaser.yml

@@ -32,20 +32,11 @@ builds:
 
 archives:
   - id: golang-cross
-    builds:
-      - darwin-amd64
-      - darwin-arm64
-      - freebsd-amd64
-      - linux-386
-      - linux-amd64
-      - linux-arm64
-      - linux-arm-5
-      - linux-arm-6
-      - linux-arm-7
-    name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
+    name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}{{ with .Mips }}_{{ . }}{{ end }}{{ if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}'
     format: binary
 
 source:
+  rlcp: true
   enabled: true
   name_template: "{{ .ProjectName }}_{{ .Version }}"
   format: tar.gz
@@ -73,7 +64,6 @@ nfpms:
     bindir: /usr/bin
     formats:
       - deb
-      # - rpm
     contents:
       - src: ./config-example.yaml
         dst: /etc/headscale/config.yaml
@@ -90,6 +80,36 @@ nfpms:
       postinstall: ./docs/packaging/postinstall.sh
       postremove: ./docs/packaging/postremove.sh
 
+kos:
+  - id: ghcr
+    build: headscale
+    base_image: gcr.io/distroless/base-debian11
+    repository: ghcr.io/juanfont/headscale
+    platforms:
+      - linux/amd64
+      - linux/386
+      - linux/arm64
+      - linux/arm/v7
+      - linux/arm/v6
+      - linux/arm/v5
+    tags:
+      - latest
+      - '{{.Tag}}'
+  - id: dockerhub
+    build: headscale
+    base_image: gcr.io/distroless/base-debian11
+    repository: headscale/headscale
+    platforms:
+      - linux/amd64
+      - linux/386
+      - linux/arm64
+      - linux/arm/v7
+      - linux/arm/v6
+      - linux/arm/v5
+    tags:
+      - latest
+      - '{{.Tag}}'
+
 checksum:
   name_template: "checksums.txt"
 snapshot:

CHANGELOG.md

@@ -4,6 +4,15 @@
 
 ### Changes
 
+- Add environment flags to enable pprof (profiling) [#1382](https://github.com/juanfont/headscale/pull/1382)
+  - Profiles are continously generated in our integration tests.
+
+## 0.22.1 (2023-04-20)
+
+### Changes
+
+- Fix issue where SystemD could not bind to port 80 [#1365](https://github.com/juanfont/headscale/pull/1365)
+
 ## 0.22.0 (2023-04-20)
 
 ### Changes

Dockerfile

@@ -1,4 +1,7 @@
-# Builder image
+# This Dockerfile and the images produced are for testing headscale,
+# and are in no way endorsed by Headscale's maintainers as an
+# official nor supported release or distribution.
+
 FROM docker.io/golang:1.20-bullseye AS build
 ARG VERSION=dev
 ENV GOPATH /go

Dockerfile.debug

@@ -1,4 +1,7 @@
-# Builder image
+# This Dockerfile and the images produced are for testing headscale,
+# and are in no way endorsed by Headscale's maintainers as an
+# official nor supported release or distribution.
+
 FROM docker.io/golang:1.20-bullseye AS build
 ARG VERSION=dev
 ENV GOPATH /go

Dockerfile.tailscale

@@ -1,3 +1,7 @@
+# This Dockerfile and the images produced are for testing headscale,
+# and are in no way endorsed by Headscale's maintainers as an
+# official nor supported release or distribution.
+
 FROM ubuntu:latest
 
 ARG TAILSCALE_VERSION=*

Dockerfile.tailscale-HEAD

@@ -1,3 +1,7 @@
+# This Dockerfile and the images produced are for testing headscale,
+# and are in no way endorsed by Headscale's maintainers as an
+# official nor supported release or distribution.
+
 FROM golang:latest
 
 RUN apt-get update \

Makefile

@@ -38,16 +38,6 @@ test_integration_cli:
 		-v /var/run/docker.sock:/var/run/docker.sock golang:1 \
 		go run gotest.tools/gotestsum@latest -- $(TAGS) -failfast -timeout 30m -count=1 -run IntegrationCLI ./...
 
-test_integration_derp:
-	docker network rm $$(docker network ls --filter name=headscale --quiet) || true
-	docker network create headscale-test || true
-	docker run -t --rm \
-		--network headscale-test \
-		-v ~/.cache/hs-integration-go:/go \
-		-v $$PWD:$$PWD -w $$PWD \
-		-v /var/run/docker.sock:/var/run/docker.sock golang:1 \
-		go run gotest.tools/gotestsum@latest -- $(TAGS) -failfast -timeout 30m -count=1 -run IntegrationDERP ./...
-
 test_integration_v2_general:
 	docker run \
 		-t --rm \

acls.go

@@ -163,23 +163,20 @@ func (h *Headscale) UpdateACLRules() error {
 // generateACLPeerCacheMap takes a list of Tailscale filter rules and generates a map
 // of which Sources ("*" and IPs) can access destinations. This is to speed up the
 // process of generating MapResponses when deciding which Peers to inform nodes about.
-func generateACLPeerCacheMap(rules []tailcfg.FilterRule) map[string]map[string]struct{} {
-	aclCachePeerMap := make(map[string]map[string]struct{})
+func generateACLPeerCacheMap(rules []tailcfg.FilterRule) map[string][]string {
+	aclCachePeerMap := make(map[string][]string)
 	for _, rule := range rules {
 		for _, srcIP := range rule.SrcIPs {
 			for _, ip := range expandACLPeerAddr(srcIP) {
 				if data, ok := aclCachePeerMap[ip]; ok {
 					for _, dstPort := range rule.DstPorts {
-						for _, dstIP := range expandACLPeerAddr(dstPort.IP) {
-							data[dstIP] = struct{}{}
-						}
+						data = append(data, dstPort.IP)
 					}
+					aclCachePeerMap[ip] = data
 				} else {
-					dstPortsMap := make(map[string]struct{}, len(rule.DstPorts))
+					dstPortsMap := make([]string, 0)
 					for _, dstPort := range rule.DstPorts {
-						for _, dstIP := range expandACLPeerAddr(dstPort.IP) {
-							dstPortsMap[dstIP] = struct{}{}
-						}
+						dstPortsMap = append(dstPortsMap, dstPort.IP)
 					}
 					aclCachePeerMap[ip] = dstPortsMap
 				}

app.go

@@ -87,7 +87,7 @@ type Headscale struct {
 	aclPolicy         *ACLPolicy
 	aclRules          []tailcfg.FilterRule
 	aclPeerCacheMapRW sync.RWMutex
-	aclPeerCacheMap   map[string]map[string]struct{}
+	aclPeerCacheMap   map[string][]string
 	sshPolicy         *tailcfg.SSHPolicy
 
 	lastStateChange *xsync.MapOf[string, time.Time]
@@ -820,7 +820,6 @@ func (h *Headscale) Serve() error {
 
 				// And we're done:
 				cancel()
-				os.Exit(0)
 			}
 		}
 	}

cmd/gh-action-integration-generator/main.go

@@ -76,6 +76,12 @@ jobs:
         with:
           name: logs
           path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"
 `),
 	)
 )

cmd/headscale/headscale.go

@@ -6,11 +6,25 @@ import (
 
 	"github.com/efekarakus/termcolor"
 	"github.com/juanfont/headscale/cmd/headscale/cli"
+	"github.com/pkg/profile"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 )
 
 func main() {
+	if _, enableProfile := os.LookupEnv("HEADSCALE_PROFILING_ENABLED"); enableProfile {
+		if profilePath, ok := os.LookupEnv("HEADSCALE_PROFILING_PATH"); ok {
+			err := os.MkdirAll(profilePath, os.ModePerm)
+			if err != nil {
+				log.Fatal().Err(err).Msg("failed to create profiling directory")
+			}
+
+			defer profile.Start(profile.ProfilePath(profilePath)).Stop()
+		} else {
+			defer profile.Start().Stop()
+		}
+	}
+
 	var colors bool
 	switch l := termcolor.SupportLevel(os.Stderr); l {
 	case termcolor.Level16M:

docs/packaging/headscale.systemd.service

@@ -16,7 +16,7 @@ WorkingDirectory=/var/lib/headscale
 ReadWritePaths=/var/lib/headscale /var/run
 
 AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_CHOWN
-CapabilityBoundingSet=CAP_CHOWN
+CapabilityBoundingSet=CAP_NET_BIND_SERVICE CAP_CHOWN
 LockPersonality=true
 NoNewPrivileges=true
 PrivateDevices=true

flake.nix

@@ -36,7 +36,7 @@
 
             # When updating go.mod or go.sum, a new sha will need to be calculated,
             # update this if you have a mismatch after doing a change to thos files.
-            vendorSha256 = "sha256-+JxS4Q6rTpdBwms2nkVDY/Kluv2qu2T0BaOIjfeX85M=";
+            vendorSha256 = "sha256-Gu0RhzXnrZ5705X/1CbTfmZlIG9PkxBZzs1bqWQPqWg=";
 
             ldflags = [ "-s" "-w" "-X github.com/juanfont/headscale/cmd/headscale/cli.Version=v${version}" ];
           };
@@ -100,6 +100,10 @@
             nfpm
             gotestsum
 
+            # 'dot' is needed for pprof graphs
+            # go tool pprof -http=: <source>
+            graphviz
+
             # Protobuf dependencies
             protobuf
             protoc-gen-go

go.mod

@@ -4,7 +4,6 @@ go 1.20
 
 require (
 	github.com/AlecAivazis/survey/v2 v2.3.6
-	github.com/ccding/go-stun/stun v0.0.0-20200514191101-4dc67bcdb029
 	github.com/cenkalti/backoff/v4 v4.2.0
 	github.com/coreos/go-oidc/v3 v3.5.0
 	github.com/davecgh/go-spew v1.1.1
@@ -20,6 +19,7 @@ require (
 	github.com/ory/dockertest/v3 v3.9.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/philip-bui/grpc-zerolog v1.0.1
+	github.com/pkg/profile v1.7.0
 	github.com/prometheus/client_golang v1.14.0
 	github.com/prometheus/common v0.42.0
 	github.com/pterm/pterm v0.12.58
@@ -64,6 +64,7 @@ require (
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/felixge/fgprof v0.9.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.4.0 // indirect
 	github.com/glebarez/go-sqlite v1.20.3 // indirect
@@ -75,6 +76,7 @@ require (
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/go-github v17.0.0+incompatible // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gookit/color v1.5.3 // indirect

go.sum

@@ -74,8 +74,6 @@ github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkU
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/ccding/go-stun/stun v0.0.0-20200514191101-4dc67bcdb029 h1:POmUHfxXdeyM8Aomg4tKDcwATCFuW+cYLkj6pwsw9pc=
-github.com/ccding/go-stun/stun v0.0.0-20200514191101-4dc67bcdb029/go.mod h1:Rpr5n9cGHYdM3S3IK8ROSUUUYjQOu+MSUCZDcJbYWi8=
 github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4=
 github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -129,6 +127,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
+github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
@@ -238,7 +238,9 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
 github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
@@ -272,6 +274,7 @@ github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u
 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
 github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
 github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
@@ -384,6 +387,8 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
+github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo=
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -669,6 +674,7 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

integration/control.go

@@ -2,12 +2,15 @@ package integration
 
 import (
 	v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
+	"github.com/ory/dockertest/v3"
 )
 
 type ControlServer interface {
 	Shutdown() error
 	SaveLog(string) error
+	SaveProfile(string) error
 	Execute(command []string) (string, error)
+	ConnectToNetwork(network *dockertest.Network) error
 	GetHealthEndpoint() string
 	GetEndpoint() string
 	WaitForReady() error

integration/embedded_derp_test.go

@@ -0,0 +1,236 @@
+package integration
+
+import (
+	"fmt"
+	"log"
+	"net/url"
+	"testing"
+
+	"github.com/juanfont/headscale"
+	"github.com/juanfont/headscale/integration/dockertestutil"
+	"github.com/juanfont/headscale/integration/hsic"
+	"github.com/juanfont/headscale/integration/tsic"
+	"github.com/ory/dockertest/v3"
+)
+
+type EmbeddedDERPServerScenario struct {
+	*Scenario
+
+	tsicNetworks map[string]*dockertest.Network
+}
+
+func TestDERPServerScenario(t *testing.T) {
+	IntegrationSkip(t)
+	// t.Parallel()
+
+	baseScenario, err := NewScenario()
+	if err != nil {
+		t.Errorf("failed to create scenario: %s", err)
+	}
+
+	scenario := EmbeddedDERPServerScenario{
+		Scenario:     baseScenario,
+		tsicNetworks: map[string]*dockertest.Network{},
+	}
+
+	spec := map[string]int{
+		"user1": len(TailscaleVersions),
+	}
+
+	headscaleConfig := map[string]string{}
+	headscaleConfig["HEADSCALE_DERP_URLS"] = ""
+	headscaleConfig["HEADSCALE_DERP_SERVER_ENABLED"] = "true"
+	headscaleConfig["HEADSCALE_DERP_SERVER_REGION_ID"] = "999"
+	headscaleConfig["HEADSCALE_DERP_SERVER_REGION_CODE"] = "headscale"
+	headscaleConfig["HEADSCALE_DERP_SERVER_REGION_NAME"] = "Headscale Embedded DERP"
+	headscaleConfig["HEADSCALE_DERP_SERVER_STUN_LISTEN_ADDR"] = "0.0.0.0:3478"
+
+	err = scenario.CreateHeadscaleEnv(
+		spec,
+		hsic.WithConfigEnv(headscaleConfig),
+		hsic.WithTestName("derpserver"),
+		hsic.WithExtraPorts([]string{"3478/udp"}),
+		hsic.WithTLS(),
+		hsic.WithHostnameAsServerURL(),
+	)
+
+	if err != nil {
+		t.Errorf("failed to create headscale environment: %s", err)
+	}
+
+	allClients, err := scenario.ListTailscaleClients()
+	if err != nil {
+		t.Errorf("failed to get clients: %s", err)
+	}
+
+	allIps, err := scenario.ListTailscaleClientsIPs()
+	if err != nil {
+		t.Errorf("failed to get clients: %s", err)
+	}
+
+	err = scenario.WaitForTailscaleSync()
+	if err != nil {
+		t.Errorf("failed wait for tailscale clients to be in sync: %s", err)
+	}
+
+	allHostnames, err := scenario.ListTailscaleClientsFQDNs()
+	if err != nil {
+		t.Errorf("failed to get FQDNs: %s", err)
+	}
+
+	success := pingDerpAllHelper(t, allClients, allHostnames)
+
+	t.Logf("%d successful pings out of %d", success, len(allClients)*len(allIps))
+
+	err = scenario.Shutdown()
+	if err != nil {
+		t.Errorf("failed to tear down scenario: %s", err)
+	}
+}
+
+func (s *EmbeddedDERPServerScenario) CreateHeadscaleEnv(
+	users map[string]int,
+	opts ...hsic.Option,
+) error {
+	hsServer, err := s.Headscale(opts...)
+	if err != nil {
+		return err
+	}
+
+	headscaleEndpoint := hsServer.GetEndpoint()
+	headscaleURL, err := url.Parse(headscaleEndpoint)
+	if err != nil {
+		return err
+	}
+
+	headscaleURL.Host = fmt.Sprintf("%s:%s", hsServer.GetHostname(), headscaleURL.Port())
+
+	err = hsServer.WaitForReady()
+	if err != nil {
+		return err
+	}
+
+	hash, err := headscale.GenerateRandomStringDNSSafe(scenarioHashLength)
+	if err != nil {
+		return err
+	}
+
+	for userName, clientCount := range users {
+		err = s.CreateUser(userName)
+		if err != nil {
+			return err
+		}
+
+		err = s.CreateTailscaleIsolatedNodesInUser(
+			hash,
+			userName,
+			"all",
+			clientCount,
+		)
+		if err != nil {
+			return err
+		}
+
+		key, err := s.CreatePreAuthKey(userName, true, false)
+		if err != nil {
+			return err
+		}
+
+		err = s.RunTailscaleUp(userName, headscaleURL.String(), key.GetKey())
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *EmbeddedDERPServerScenario) CreateTailscaleIsolatedNodesInUser(
+	hash string,
+	userStr string,
+	requestedVersion string,
+	count int,
+	opts ...tsic.Option,
+) error {
+	hsServer, err := s.Headscale()
+	if err != nil {
+		return err
+	}
+
+	if user, ok := s.users[userStr]; ok {
+		for clientN := 0; clientN < count; clientN++ {
+			networkName := fmt.Sprintf("tsnet-%s-%s-%d",
+				hash,
+				userStr,
+				clientN,
+			)
+			network, err := dockertestutil.GetFirstOrCreateNetwork(
+				s.pool,
+				networkName,
+			)
+			if err != nil {
+				return fmt.Errorf("failed to create or get %s network: %w", networkName, err)
+			}
+
+			s.tsicNetworks[networkName] = network
+
+			err = hsServer.ConnectToNetwork(network)
+			if err != nil {
+				return fmt.Errorf("failed to connect headscale to %s network: %w", networkName, err)
+			}
+
+			version := requestedVersion
+			if requestedVersion == "all" {
+				version = TailscaleVersions[clientN%len(TailscaleVersions)]
+			}
+
+			cert := hsServer.GetCert()
+
+			user.createWaitGroup.Add(1)
+
+			opts = append(opts,
+				tsic.WithHeadscaleTLS(cert),
+			)
+
+			go func() {
+				defer user.createWaitGroup.Done()
+
+				// TODO(kradalby): error handle this
+				tsClient, err := tsic.New(
+					s.pool,
+					version,
+					network,
+					opts...,
+				)
+				if err != nil {
+					// return fmt.Errorf("failed to add tailscale node: %w", err)
+					log.Printf("failed to create tailscale node: %s", err)
+				}
+
+				err = tsClient.WaitForReady()
+				if err != nil {
+					// return fmt.Errorf("failed to add tailscale node: %w", err)
+					log.Printf("failed to wait for tailscaled: %s", err)
+				}
+
+				user.Clients[tsClient.Hostname()] = tsClient
+			}()
+		}
+		user.createWaitGroup.Wait()
+
+		return nil
+	}
+
+	return fmt.Errorf("failed to add tailscale node: %w", errNoUserAvailable)
+}
+
+func (s *EmbeddedDERPServerScenario) Shutdown() error {
+	for _, network := range s.tsicNetworks {
+		err := s.pool.RemoveNetwork(network)
+		if err != nil {
+			return err
+		}
+	}
+
+	return s.Scenario.Shutdown()
+}

integration/hsic/hsic.go

@@ -15,6 +15,10 @@ import (
 	"math/big"
 	"net"
 	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"strings"
 	"time"
 
 	"github.com/davecgh/go-spew/spew"
@@ -23,6 +27,7 @@ import (
 	"github.com/juanfont/headscale/integration/dockertestutil"
 	"github.com/juanfont/headscale/integration/integrationutil"
 	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
 )
 
 const (
@@ -52,6 +57,8 @@ type HeadscaleInContainer struct {
 
 	// optional config
 	port             int
+	extraPorts       []string
+	hostPortBindings map[string][]string
 	aclPolicy        *headscale.ACLPolicy
 	env              map[string]string
 	tlsCert          []byte
@@ -77,7 +84,7 @@ func WithACLPolicy(acl *headscale.ACLPolicy) Option {
 // WithTLS creates certificates and enables HTTPS.
 func WithTLS() Option {
 	return func(hsic *HeadscaleInContainer) {
-		cert, key, err := createCertificate()
+		cert, key, err := createCertificate(hsic.hostname)
 		if err != nil {
 			log.Fatalf("failed to create certificates for headscale test: %s", err)
 		}
@@ -108,6 +115,19 @@ func WithPort(port int) Option {
 	}
 }
 
+// WithExtraPorts exposes additional ports on the container (e.g. 3478/udp for STUN).
+func WithExtraPorts(ports []string) Option {
+	return func(hsic *HeadscaleInContainer) {
+		hsic.extraPorts = ports
+	}
+}
+
+func WithHostPortBindings(bindings map[string][]string) Option {
+	return func(hsic *HeadscaleInContainer) {
+		hsic.hostPortBindings = bindings
+	}
+}
+
 // WithTestName sets a name for the test, this will be reflected
 // in the Docker container name.
 func WithTestName(testName string) Option {
@@ -173,12 +193,25 @@ func New(
 
 	portProto := fmt.Sprintf("%d/tcp", hsic.port)
 
+	serverURL, err := url.Parse(hsic.env["HEADSCALE_SERVER_URL"])
+	if err != nil {
+		return nil, err
+	}
+
+	if len(hsic.tlsCert) != 0 && len(hsic.tlsKey) != 0 {
+		serverURL.Scheme = "https"
+		hsic.env["HEADSCALE_SERVER_URL"] = serverURL.String()
+	}
+
 	headscaleBuildOptions := &dockertest.BuildOptions{
 		Dockerfile: "Dockerfile.debug",
 		ContextDir: dockerContextPath,
 	}
 
-	env := []string{}
+	env := []string{
+		"HEADSCALE_PROFILING_ENABLED=1",
+		"HEADSCALE_PROFILING_PATH=/tmp/profile",
+	}
 	for key, value := range hsic.env {
 		env = append(env, fmt.Sprintf("%s=%s", key, value))
 	}
@@ -187,15 +220,27 @@ func New(
 
 	runOptions := &dockertest.RunOptions{
 		Name:         hsic.hostname,
-		ExposedPorts: []string{portProto},
+		ExposedPorts: append([]string{portProto}, hsic.extraPorts...),
 		Networks:     []*dockertest.Network{network},
 		// Cmd:          []string{"headscale", "serve"},
 		// TODO(kradalby): Get rid of this hack, we currently need to give us some
 		// to inject the headscale configuration further down.
-		Entrypoint: []string{"/bin/bash", "-c", "/bin/sleep 3 ; headscale serve"},
+		Entrypoint: []string{"/bin/bash", "-c", "/bin/sleep 3 ; headscale serve ; /bin/sleep 30"},
 		Env:        env,
 	}
 
+	if len(hsic.hostPortBindings) > 0 {
+		runOptions.PortBindings = map[docker.Port][]docker.PortBinding{}
+		for port, hostPorts := range hsic.hostPortBindings {
+			runOptions.PortBindings[docker.Port(port)] = []docker.PortBinding{}
+			for _, hostPort := range hostPorts {
+				runOptions.PortBindings[docker.Port(port)] = append(
+					runOptions.PortBindings[docker.Port(port)],
+					docker.PortBinding{HostPort: hostPort})
+			}
+		}
+	}
+
 	// dockertest isnt very good at handling containers that has already
 	// been created, this is an attempt to make sure this container isnt
 	// present.
@@ -256,12 +301,43 @@ func New(
 	return hsic, nil
 }
 
+func (t *HeadscaleInContainer) ConnectToNetwork(network *dockertest.Network) error {
+	return t.container.ConnectToNetwork(network)
+}
+
 func (t *HeadscaleInContainer) hasTLS() bool {
 	return len(t.tlsCert) != 0 && len(t.tlsKey) != 0
 }
 
 // Shutdown stops and cleans up the Headscale container.
 func (t *HeadscaleInContainer) Shutdown() error {
+	err := t.SaveLog("/tmp/control")
+	if err != nil {
+		log.Printf(
+			"Failed to save log from control: %s",
+			fmt.Errorf("failed to save log from control: %w", err),
+		)
+	}
+
+	// Send a interrupt signal to the "headscale" process inside the container
+	// allowing it to shut down gracefully and flush the profile to disk.
+	// The container will live for a bit longer due to the sleep at the end.
+	err = t.SendInterrupt()
+	if err != nil {
+		log.Printf(
+			"Failed to send graceful interrupt to control: %s",
+			fmt.Errorf("failed to send graceful interrupt to control: %w", err),
+		)
+	}
+
+	err = t.SaveProfile("/tmp/control")
+	if err != nil {
+		log.Printf(
+			"Failed to save profile from control: %s",
+			fmt.Errorf("failed to save profile from control: %w", err),
+		)
+	}
+
 	return t.pool.Purge(t.container)
 }
 
@@ -271,6 +347,24 @@ func (t *HeadscaleInContainer) SaveLog(path string) error {
 	return dockertestutil.SaveLog(t.pool, t.container, path)
 }
 
+func (t *HeadscaleInContainer) SaveProfile(savePath string) error {
+	tarFile, err := t.FetchPath("/tmp/profile")
+	if err != nil {
+		return err
+	}
+
+	err = os.WriteFile(
+		path.Join(savePath, t.hostname+".pprof.tar"),
+		tarFile,
+		os.ModePerm,
+	)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // Execute runs a command inside the Headscale container and returns the
 // result of stdout as a string.
 func (t *HeadscaleInContainer) Execute(
@@ -455,8 +549,28 @@ func (t *HeadscaleInContainer) WriteFile(path string, data []byte) error {
 	return integrationutil.WriteFileToContainer(t.pool, t.container, path, data)
 }
 
+// FetchPath gets a path from inside the Headscale container and returns a tar
+// file as byte array.
+func (t *HeadscaleInContainer) FetchPath(path string) ([]byte, error) {
+	return integrationutil.FetchPathFromContainer(t.pool, t.container, path)
+}
+
+func (t *HeadscaleInContainer) SendInterrupt() error {
+	pid, err := t.Execute([]string{"pidof", "headscale"})
+	if err != nil {
+		return err
+	}
+
+	_, err = t.Execute([]string{"kill", "-2", strings.Trim(pid, "'\n")})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // nolint
-func createCertificate() ([]byte, []byte, error) {
+func createCertificate(hostname string) ([]byte, []byte, error) {
 	// From:
 	// https://shaneutt.com/blog/golang-ca-and-signed-cert-go/
 
@@ -468,7 +582,7 @@ func createCertificate() ([]byte, []byte, error) {
 			Locality:     []string{"Leiden"},
 		},
 		NotBefore: time.Now(),
-		NotAfter:  time.Now().Add(30 * time.Minute),
+		NotAfter:  time.Now().Add(60 * time.Minute),
 		IsCA:      true,
 		ExtKeyUsage: []x509.ExtKeyUsage{
 			x509.ExtKeyUsageClientAuth,
@@ -486,16 +600,17 @@ func createCertificate() ([]byte, []byte, error) {
 	cert := &x509.Certificate{
 		SerialNumber: big.NewInt(1658),
 		Subject: pkix.Name{
+			CommonName:   hostname,
 			Organization: []string{"Headscale testing INC"},
 			Country:      []string{"NL"},
 			Locality:     []string{"Leiden"},
 		},
-		IPAddresses:  []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
 		NotBefore:    time.Now(),
-		NotAfter:     time.Now().Add(30 * time.Minute),
+		NotAfter:     time.Now().Add(60 * time.Minute),
 		SubjectKeyId: []byte{1, 2, 3, 4, 6},
 		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
 		KeyUsage:     x509.KeyUsageDigitalSignature,
+		DNSNames:     []string{hostname},
 	}
 
 	certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)

integration/integrationutil/util.go

@@ -72,3 +72,24 @@ func WriteFileToContainer(
 
 	return nil
 }
+
+func FetchPathFromContainer(
+	pool *dockertest.Pool,
+	container *dockertest.Resource,
+	path string,
+) ([]byte, error) {
+	buf := bytes.NewBuffer([]byte{})
+
+	err := pool.Client.DownloadFromContainer(
+		container.Container.ID,
+		docker.DownloadFromContainerOptions{
+			OutputStream: buf,
+			Path:         path,
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}

integration/scenario.go

@@ -149,15 +149,7 @@ func NewScenario() (*Scenario, error) {
 // environment running the tests.
 func (s *Scenario) Shutdown() error {
 	s.controlServers.Range(func(_ string, control ControlServer) bool {
-		err := control.SaveLog("/tmp/control")
-		if err != nil {
-			log.Printf(
-				"Failed to save log from control: %s",
-				fmt.Errorf("failed to save log from control: %w", err),
-			)
-		}
-
-		err = control.Shutdown()
+		err := control.Shutdown()
 		if err != nil {
 			log.Printf(
 				"Failed to shut down control: %s",

integration/tailscale.go

@@ -4,6 +4,7 @@ import (
 	"net/netip"
 	"net/url"
 
+	"github.com/juanfont/headscale/integration/dockertestutil"
 	"github.com/juanfont/headscale/integration/tsic"
 	"tailscale.com/ipn/ipnstate"
 )
@@ -13,7 +14,7 @@ type TailscaleClient interface {
 	Hostname() string
 	Shutdown() error
 	Version() string
-	Execute(command []string) (string, string, error)
+	Execute(command []string, options ...dockertestutil.ExecuteCommandOption) (string, string, error)
 	Up(loginServer, authKey string) error
 	UpWithLoginURL(loginServer string) (*url.URL, error)
 	Logout() error

integration/tsic/tsic.go

@@ -29,6 +29,7 @@ const (
 
 var (
 	errTailscalePingFailed             = errors.New("ping failed")
+	errTailscalePingNotDERP            = errors.New("ping not via DERP")
 	errTailscaleNotLoggedIn            = errors.New("tailscale not logged in")
 	errTailscaleWrongPeerCount         = errors.New("wrong peer count")
 	errTailscaleCannotUpWithoutAuthkey = errors.New("cannot up without authkey")
@@ -56,6 +57,7 @@ type TailscaleInContainer struct {
 	withSSH           bool
 	withTags          []string
 	withEntrypoint    []string
+	withExtraHosts    []string
 	workdir           string
 }
 
@@ -124,6 +126,12 @@ func WithDockerWorkdir(dir string) Option {
 	}
 }
 
+func WithExtraHosts(hosts []string) Option {
+	return func(tsic *TailscaleInContainer) {
+		tsic.withExtraHosts = hosts
+	}
+}
+
 // WithDockerEntrypoint allows the docker entrypoint of the container
 // to be overridden. This is a dangerous option which can make
 // the container not work as intended as a typo might prevent
@@ -169,11 +177,12 @@ func New(
 
 	tailscaleOptions := &dockertest.RunOptions{
 		Name:     hostname,
-		Networks: []*dockertest.Network{network},
+		Networks: []*dockertest.Network{tsic.network},
 		// Cmd: []string{
 		// 	"tailscaled", "--tun=tsdev",
 		// },
 		Entrypoint: tsic.withEntrypoint,
+		ExtraHosts: tsic.withExtraHosts,
 	}
 
 	if tsic.headscaleHostname != "" {
@@ -248,11 +257,13 @@ func (t *TailscaleInContainer) ID() string {
 // result of stdout as a string.
 func (t *TailscaleInContainer) Execute(
 	command []string,
+	options ...dockertestutil.ExecuteCommandOption,
 ) (string, string, error) {
 	stdout, stderr, err := dockertestutil.ExecuteCommand(
 		t.container,
 		command,
 		[]string{},
+		options...,
 	)
 	if err != nil {
 		log.Printf("command stderr: %s\n", stderr)
@@ -477,7 +488,7 @@ func (t *TailscaleInContainer) WaitForPeers(expected int) error {
 }
 
 type (
-	// PingOption repreent optional settings that can be given
+	// PingOption represent optional settings that can be given
 	// to ping another host.
 	PingOption = func(args *pingArgs)
 
@@ -535,7 +546,12 @@ func (t *TailscaleInContainer) Ping(hostnameOrIP string, opts ...PingOption) err
 	command = append(command, hostnameOrIP)
 
 	return t.pool.Retry(func() error {
-		result, _, err := t.Execute(command)
+		result, _, err := t.Execute(
+			command,
+			dockertestutil.ExecuteCommandTimeout(
+				time.Duration(int64(args.timeout)*int64(args.count)),
+			),
+		)
 		if err != nil {
 			log.Printf(
 				"failed to run ping command from %s to %s, err: %s",
@@ -547,10 +563,22 @@ func (t *TailscaleInContainer) Ping(hostnameOrIP string, opts ...PingOption) err
 			return err
 		}
 
-		if !strings.Contains(result, "pong") && !strings.Contains(result, "is local") {
+		if strings.Contains(result, "is local") {
+			return nil
+		}
+
+		if !strings.Contains(result, "pong") {
 			return backoff.Permanent(errTailscalePingFailed)
 		}
 
+		if !args.direct {
+			if strings.Contains(result, "via DERP") {
+				return nil
+			} else {
+				return backoff.Permanent(errTailscalePingNotDERP)
+			}
+		}
+
 		return nil
 	})
 }

integration/utils.go

@@ -2,6 +2,14 @@ package integration
 
 import (
 	"testing"
+	"time"
+
+	"github.com/juanfont/headscale/integration/tsic"
+)
+
+const (
+	derpPingTimeout = 2 * time.Second
+	derpPingCount   = 10
 )
 
 func pingAllHelper(t *testing.T, clients []TailscaleClient, addrs []string) int {
@@ -22,6 +30,52 @@ func pingAllHelper(t *testing.T, clients []TailscaleClient, addrs []string) int
 	return success
 }
 
+func pingDerpAllHelper(t *testing.T, clients []TailscaleClient, addrs []string) int {
+	t.Helper()
+	success := 0
+
+	for _, client := range clients {
+		for _, addr := range addrs {
+			if isSelfClient(client, addr) {
+				continue
+			}
+
+			err := client.Ping(
+				addr,
+				tsic.WithPingTimeout(derpPingTimeout),
+				tsic.WithPingCount(derpPingCount),
+				tsic.WithPingUntilDirect(false),
+			)
+			if err != nil {
+				t.Errorf("failed to ping %s from %s: %s", addr, client.Hostname(), err)
+			} else {
+				success++
+			}
+		}
+	}
+
+	return success
+}
+
+func isSelfClient(client TailscaleClient, addr string) bool {
+	if addr == client.Hostname() {
+		return true
+	}
+
+	ips, err := client.IPs()
+	if err != nil {
+		return false
+	}
+
+	for _, ip := range ips {
+		if ip.String() == addr {
+			return true
+		}
+	}
+
+	return false
+}
+
 // pingAllNegativeHelper is intended to have 1 or more nodes timeing out from the ping,
 // it counts failures instead of successes.
 // func pingAllNegativeHelper(t *testing.T, clients []TailscaleClient, addrs []string) int {

integration_embedded_derp_test.go

@@ -1,453 +0,0 @@
-// nolint
-package headscale
-
-import (
-	"bytes"
-	"context"
-	"crypto/tls"
-	"encoding/json"
-	"fmt"
-	"log"
-	"net/http"
-	"os"
-	"path"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/ccding/go-stun/stun"
-	v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
-	"github.com/ory/dockertest/v3"
-	"github.com/ory/dockertest/v3/docker"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/suite"
-)
-
-const (
-	headscaleDerpHostname = "headscale-derp"
-	userName              = "derpuser"
-	totalContainers       = 3
-)
-
-type IntegrationDERPTestSuite struct {
-	suite.Suite
-	stats *suite.SuiteInformation
-
-	pool              dockertest.Pool
-	network           dockertest.Network
-	containerNetworks map[int]dockertest.Network // so we keep the containers isolated
-	headscale         dockertest.Resource
-	saveLogs          bool
-
-	tailscales    map[string]dockertest.Resource
-	joinWaitGroup sync.WaitGroup
-}
-
-func TestIntegrationDERPTestSuite(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration tests due to short flag")
-	}
-
-	saveLogs, err := GetEnvBool("HEADSCALE_INTEGRATION_SAVE_LOG")
-	if err != nil {
-		saveLogs = false
-	}
-
-	s := new(IntegrationDERPTestSuite)
-
-	s.tailscales = make(map[string]dockertest.Resource)
-	s.containerNetworks = make(map[int]dockertest.Network)
-	s.saveLogs = saveLogs
-
-	suite.Run(t, s)
-
-	// HandleStats, which allows us to check if we passed and save logs
-	// is called after TearDown, so we cannot tear down containers before
-	// we have potentially saved the logs.
-	if s.saveLogs {
-		for _, tailscale := range s.tailscales {
-			if err := s.pool.Purge(&tailscale); err != nil {
-				log.Printf("Could not purge resource: %s\n", err)
-			}
-		}
-
-		if !s.stats.Passed() {
-			err := s.saveLog(&s.headscale, "test_output")
-			if err != nil {
-				log.Printf("Could not save log: %s\n", err)
-			}
-		}
-		if err := s.pool.Purge(&s.headscale); err != nil {
-			log.Printf("Could not purge resource: %s\n", err)
-		}
-
-		for _, network := range s.containerNetworks {
-			if err := network.Close(); err != nil {
-				log.Printf("Could not close network: %s\n", err)
-			}
-		}
-	}
-}
-
-func (s *IntegrationDERPTestSuite) SetupSuite() {
-	if ppool, err := dockertest.NewPool(""); err == nil {
-		s.pool = *ppool
-	} else {
-		s.FailNow(fmt.Sprintf("Could not connect to docker: %s", err), "")
-	}
-
-	network, err := GetFirstOrCreateNetwork(&s.pool, headscaleNetwork)
-	if err != nil {
-		s.FailNow(fmt.Sprintf("Failed to create or get network: %s", err), "")
-	}
-	s.network = network
-
-	for i := 0; i < totalContainers; i++ {
-		if pnetwork, err := s.pool.CreateNetwork(fmt.Sprintf("headscale-derp-%d", i)); err == nil {
-			s.containerNetworks[i] = *pnetwork
-		} else {
-			s.FailNow(fmt.Sprintf("Could not create network: %s", err), "")
-		}
-	}
-
-	headscaleBuildOptions := &dockertest.BuildOptions{
-		Dockerfile: "Dockerfile",
-		ContextDir: ".",
-	}
-
-	currentPath, err := os.Getwd()
-	if err != nil {
-		s.FailNow(fmt.Sprintf("Could not determine current path: %s", err), "")
-	}
-
-	headscaleOptions := &dockertest.RunOptions{
-		Name: headscaleDerpHostname,
-		Mounts: []string{
-			fmt.Sprintf(
-				"%s/integration_test/etc_embedded_derp:/etc/headscale",
-				currentPath,
-			),
-		},
-		Cmd:          []string{"headscale", "serve"},
-		Networks:     []*dockertest.Network{&s.network},
-		ExposedPorts: []string{"8443/tcp", "3478/udp"},
-		PortBindings: map[docker.Port][]docker.PortBinding{
-			"8443/tcp": {{HostPort: "8443"}},
-			"3478/udp": {{HostPort: "3478"}},
-		},
-	}
-
-	err = s.pool.RemoveContainerByName(headscaleDerpHostname)
-	if err != nil {
-		s.FailNow(
-			fmt.Sprintf(
-				"Could not remove existing container before building test: %s",
-				err,
-			),
-			"",
-		)
-	}
-
-	log.Println("Creating headscale container for DERP integration tests")
-	if pheadscale, err := s.pool.BuildAndRunWithBuildOptions(headscaleBuildOptions, headscaleOptions, DockerRestartPolicy); err == nil {
-		s.headscale = *pheadscale
-	} else {
-		s.FailNow(fmt.Sprintf("Could not start headscale container: %s", err), "")
-	}
-	log.Println("Created headscale container for embedded DERP tests")
-
-	log.Println("Creating tailscale containers for embedded DERP tests")
-
-	for i := 0; i < totalContainers; i++ {
-		version := tailscaleVersions[i%len(tailscaleVersions)]
-		hostname, container := s.tailscaleContainer(
-			fmt.Sprint(i),
-			version,
-			s.containerNetworks[i],
-		)
-		s.tailscales[hostname] = *container
-	}
-
-	log.Println("Waiting for headscale to be ready for embedded DERP tests")
-	hostEndpoint := fmt.Sprintf("%s:%s",
-		s.headscale.GetIPInNetwork(&s.network),
-		s.headscale.GetPort("8443/tcp"))
-
-	if err := s.pool.Retry(func() error {
-		url := fmt.Sprintf("https://%s/health", hostEndpoint)
-		insecureTransport := http.DefaultTransport.(*http.Transport).Clone()
-		insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
-		client := &http.Client{Transport: insecureTransport}
-		resp, err := client.Get(url)
-		if err != nil {
-			fmt.Printf("headscale for embedded DERP tests is not ready: %s\n", err)
-			return err
-		}
-
-		if resp.StatusCode != http.StatusOK {
-			return fmt.Errorf("status code not OK")
-		}
-
-		return nil
-	}); err != nil {
-		// TODO(kradalby): If we cannot access headscale, or any other fatal error during
-		// test setup, we need to abort and tear down. However, testify does not seem to
-		// support that at the moment:
-		// https://github.com/stretchr/testify/issues/849
-		return // fmt.Errorf("Could not connect to headscale: %s", err)
-	}
-	log.Println("headscale container is ready for embedded DERP tests")
-
-	log.Printf("Creating headscale user: %s\n", userName)
-	result, _, err := ExecuteCommand(
-		&s.headscale,
-		[]string{"headscale", "users", "create", userName},
-		[]string{},
-	)
-	log.Println("headscale create user result: ", result)
-	assert.Nil(s.T(), err)
-
-	log.Printf("Creating pre auth key for %s\n", userName)
-	preAuthResult, _, err := ExecuteCommand(
-		&s.headscale,
-		[]string{
-			"headscale",
-			"--user",
-			userName,
-			"preauthkeys",
-			"create",
-			"--reusable",
-			"--expiration",
-			"24h",
-			"--output",
-			"json",
-		},
-		[]string{"LOG_LEVEL=error"},
-	)
-	assert.Nil(s.T(), err)
-
-	var preAuthKey v1.PreAuthKey
-	err = json.Unmarshal([]byte(preAuthResult), &preAuthKey)
-	assert.Nil(s.T(), err)
-	assert.True(s.T(), preAuthKey.Reusable)
-
-	headscaleEndpoint := fmt.Sprintf(
-		"https://headscale:%s",
-		s.headscale.GetPort("8443/tcp"),
-	)
-
-	log.Printf(
-		"Joining tailscale containers to headscale at %s\n",
-		headscaleEndpoint,
-	)
-	for hostname, tailscale := range s.tailscales {
-		s.joinWaitGroup.Add(1)
-		go s.Join(headscaleEndpoint, preAuthKey.Key, hostname, tailscale)
-	}
-
-	s.joinWaitGroup.Wait()
-
-	// The nodes need a bit of time to get their updated maps from headscale
-	// TODO: See if we can have a more deterministic wait here.
-	time.Sleep(60 * time.Second)
-}
-
-func (s *IntegrationDERPTestSuite) Join(
-	endpoint, key, hostname string,
-	tailscale dockertest.Resource,
-) {
-	defer s.joinWaitGroup.Done()
-
-	command := []string{
-		"tailscale",
-		"up",
-		"-login-server",
-		endpoint,
-		"--authkey",
-		key,
-		"--hostname",
-		hostname,
-	}
-
-	log.Println("Join command:", command)
-	log.Printf("Running join command for %s\n", hostname)
-	_, _, err := ExecuteCommand(
-		&tailscale,
-		command,
-		[]string{},
-	)
-	assert.Nil(s.T(), err)
-	log.Printf("%s joined\n", hostname)
-}
-
-func (s *IntegrationDERPTestSuite) tailscaleContainer(
-	identifier, version string,
-	network dockertest.Network,
-) (string, *dockertest.Resource) {
-	tailscaleBuildOptions := getDockerBuildOptions(version)
-
-	hostname := fmt.Sprintf(
-		"tailscale-%s-%s",
-		strings.Replace(version, ".", "-", -1),
-		identifier,
-	)
-	tailscaleOptions := &dockertest.RunOptions{
-		Name:     hostname,
-		Networks: []*dockertest.Network{&network},
-		Cmd: []string{
-			"tailscaled", "--tun=tsdev",
-		},
-
-		// expose the host IP address, so we can access it from inside the container
-		ExtraHosts: []string{
-			"host.docker.internal:host-gateway",
-			"headscale:host-gateway",
-		},
-	}
-
-	pts, err := s.pool.BuildAndRunWithBuildOptions(
-		tailscaleBuildOptions,
-		tailscaleOptions,
-		DockerRestartPolicy,
-		DockerAllowLocalIPv6,
-		DockerAllowNetworkAdministration,
-	)
-	if err != nil {
-		log.Fatalf("Could not start tailscale container version %s: %s", version, err)
-	}
-	log.Printf("Created %s container\n", hostname)
-
-	return hostname, pts
-}
-
-func (s *IntegrationDERPTestSuite) TearDownSuite() {
-	if !s.saveLogs {
-		for _, tailscale := range s.tailscales {
-			if err := s.pool.Purge(&tailscale); err != nil {
-				log.Printf("Could not purge resource: %s\n", err)
-			}
-		}
-
-		if err := s.pool.Purge(&s.headscale); err != nil {
-			log.Printf("Could not purge resource: %s\n", err)
-		}
-
-		for _, network := range s.containerNetworks {
-			if err := network.Close(); err != nil {
-				log.Printf("Could not close network: %s\n", err)
-			}
-		}
-	}
-}
-
-func (s *IntegrationDERPTestSuite) HandleStats(
-	suiteName string,
-	stats *suite.SuiteInformation,
-) {
-	s.stats = stats
-}
-
-func (s *IntegrationDERPTestSuite) saveLog(
-	resource *dockertest.Resource,
-	basePath string,
-) error {
-	err := os.MkdirAll(basePath, os.ModePerm)
-	if err != nil {
-		return err
-	}
-
-	var stdout bytes.Buffer
-	var stderr bytes.Buffer
-
-	err = s.pool.Client.Logs(
-		docker.LogsOptions{
-			Context:      context.TODO(),
-			Container:    resource.Container.ID,
-			OutputStream: &stdout,
-			ErrorStream:  &stderr,
-			Tail:         "all",
-			RawTerminal:  false,
-			Stdout:       true,
-			Stderr:       true,
-			Follow:       false,
-			Timestamps:   false,
-		},
-	)
-	if err != nil {
-		return err
-	}
-
-	log.Printf("Saving logs for %s to %s\n", resource.Container.Name, basePath)
-
-	err = os.WriteFile(
-		path.Join(basePath, resource.Container.Name+".stdout.log"),
-		stderr.Bytes(),
-		0o644,
-	)
-	if err != nil {
-		return err
-	}
-
-	err = os.WriteFile(
-		path.Join(basePath, resource.Container.Name+".stderr.log"),
-		stderr.Bytes(),
-		0o644,
-	)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (s *IntegrationDERPTestSuite) TestPingAllPeersByHostname() {
-	hostnames, err := getDNSNames(&s.headscale)
-	assert.Nil(s.T(), err)
-
-	log.Printf("Hostnames: %#v\n", hostnames)
-
-	for hostname, tailscale := range s.tailscales {
-		for _, peername := range hostnames {
-			if strings.Contains(peername, hostname) {
-				continue
-			}
-			s.T().Run(fmt.Sprintf("%s-%s", hostname, peername), func(t *testing.T) {
-				command := []string{
-					"tailscale", "ping",
-					"--timeout=10s",
-					"--c=5",
-					"--until-direct=false",
-					peername,
-				}
-
-				log.Printf(
-					"Pinging using hostname from %s to %s\n",
-					hostname,
-					peername,
-				)
-				log.Println(command)
-				result, _, err := ExecuteCommand(
-					&tailscale,
-					command,
-					[]string{},
-				)
-				assert.Nil(t, err)
-				log.Printf("Result for %s: %s\n", hostname, result)
-				assert.Contains(t, result, "via DERP(headscale)")
-			})
-		}
-	}
-}
-
-func (s *IntegrationDERPTestSuite) TestDERPSTUN() {
-	headscaleSTUNAddr := fmt.Sprintf("%s:%s",
-		s.headscale.GetIPInNetwork(&s.network),
-		s.headscale.GetPort("3478/udp"))
-	client := stun.NewClient()
-	client.SetVerbose(true)
-	client.SetVVerbose(true)
-	client.SetServerAddr(headscaleSTUNAddr)
-	_, _, err := client.Discover()
-	assert.Nil(s.T(), err)
-}

machine.go

@@ -4,6 +4,7 @@ import (
 	"database/sql/driver"
 	"errors"
 	"fmt"
+	"net"
 	"net/netip"
 	"sort"
 	"strconv"
@@ -172,7 +173,7 @@ func filterMachinesByACL(
 	machine *Machine,
 	machines Machines,
 	lock *sync.RWMutex,
-	aclPeerCacheMap map[string]map[string]struct{},
+	aclPeerCacheMap map[string][]string,
 ) Machines {
 	log.Trace().
 		Caller().
@@ -197,43 +198,59 @@ func filterMachinesByACL(
 
 		if dstMap, ok := aclPeerCacheMap["*"]; ok {
 			// match source and all destination
-			if _, dstOk := dstMap["*"]; dstOk {
+
+			for _, dst := range dstMap {
+				if dst == "*" {
 					peers[peer.ID] = peer
 
 					continue
 				}
+			}
 
 			// match source and all destination
 			for _, peerIP := range peerIPs {
-				if _, dstOk := dstMap[peerIP]; dstOk {
+				for _, dst := range dstMap {
+					_, cdr, _ := net.ParseCIDR(dst)
+					ip := net.ParseIP(peerIP)
+					if dst == peerIP || (cdr != nil && ip != nil && cdr.Contains(ip)) {
 						peers[peer.ID] = peer
 
 						continue
 					}
 				}
+			}
 
 			// match all sources and source
 			for _, machineIP := range machineIPs {
-				if _, dstOk := dstMap[machineIP]; dstOk {
+				for _, dst := range dstMap {
+					_, cdr, _ := net.ParseCIDR(dst)
+					ip := net.ParseIP(machineIP)
+					if dst == machineIP || (cdr != nil && ip != nil && cdr.Contains(ip)) {
 						peers[peer.ID] = peer
 
 						continue
 					}
 				}
 			}
+		}
 
 		for _, machineIP := range machineIPs {
 			if dstMap, ok := aclPeerCacheMap[machineIP]; ok {
 				// match source and all destination
-				if _, dstOk := dstMap["*"]; dstOk {
+				for _, dst := range dstMap {
+					if dst == "*" {
 						peers[peer.ID] = peer
 
 						continue
 					}
+				}
 
 				// match source and destination
 				for _, peerIP := range peerIPs {
-					if _, dstOk := dstMap[peerIP]; dstOk {
+					for _, dst := range dstMap {
+						_, cdr, _ := net.ParseCIDR(dst)
+						ip := net.ParseIP(peerIP)
+						if dst == peerIP || (cdr != nil && ip != nil && cdr.Contains(ip)) {
 							peers[peer.ID] = peer
 
 							continue
@@ -241,18 +258,25 @@ func filterMachinesByACL(
 					}
 				}
 			}
+		}
 
 		for _, peerIP := range peerIPs {
 			if dstMap, ok := aclPeerCacheMap[peerIP]; ok {
 				// match source and all destination
-				if _, dstOk := dstMap["*"]; dstOk {
+				for _, dst := range dstMap {
+					if dst == "*" {
 						peers[peer.ID] = peer
 
 						continue
 					}
+				}
+
 				// match return path
 				for _, machineIP := range machineIPs {
-					if _, dstOk := dstMap[machineIP]; dstOk {
+					for _, dst := range dstMap {
+						_, cdr, _ := net.ParseCIDR(dst)
+						ip := net.ParseIP(machineIP)
+						if dst == machineIP || (cdr != nil && ip != nil && cdr.Contains(ip)) {
 							peers[peer.ID] = peer
 
 							continue
@@ -261,6 +285,7 @@ func filterMachinesByACL(
 				}
 			}
 		}
+	}
 
 	lock.RUnlock()