From c4600346f9c29b514dc9725ac103efb9d0381f23 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Fri, 12 Dec 2025 23:01:52 +0100 Subject: [PATCH] .github/workflows: prebuilt integration test artifacts (#2954) This PR restructures the integration tests and prebuilds all common assets used in all tests: Headscale and Tailscale HEAD image hi binary that is used to run tests go cache is warmed up for compilation of the test This essentially means we spend 6-10 minutes building assets before any tests starts, when that is done, all tests can just sprint through. It looks like we are saving 3-9 minutes per test, and since we are limited to running max 20 concurrent tests across the repo, that means we had a lot of double work. There is currently 113 checks, so we have to do five runs of 20, and the saving should be quite noticeable! I think the "worst case" saving would be 20+min and "best case" probably towards an hour. --- .github/workflows/build.yml | 23 ++- .github/workflows/check-generated.yml | 4 +- .github/workflows/check-tests.yaml | 7 +- .github/workflows/docs-deploy.yml | 6 +- .github/workflows/docs-test.yml | 6 +- .github/workflows/gh-actions-updater.yaml | 4 +- .../workflows/integration-test-template.yml | 79 ++++++--- .github/workflows/lint.yml | 21 +-- .github/workflows/nix-module-test.yml | 7 +- .github/workflows/release.yml | 11 +- .github/workflows/stale.yml | 8 +- .github/workflows/test-integration.yaml | 114 ++++++++++++ .github/workflows/test.yml | 7 +- Dockerfile.integration | 30 +++- Dockerfile.integration-ci | 17 ++ Dockerfile.tailscale-HEAD | 2 +- cmd/hi/docker.go | 56 ++++-- integration/hsic/hsic.go | 165 +++++++++--------- integration/tsic/tsic.go | 159 ++++++++++------- 19 files changed, 482 insertions(+), 244 deletions(-) create mode 100644 Dockerfile.integration-ci diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cb93dae5..594829f9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest permissions: write-all steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 2 - name: Get changed files @@ -29,13 +29,12 @@ jobs: - '**/*.go' - 'integration_test/' - 'config-example.yaml' - - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31 + - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 if: steps.changed-files.outputs.files == 'true' - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3 if: steps.changed-files.outputs.files == 'true' with: - primary-key: - nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', + primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', '**/flake.lock') }} restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }} @@ -55,7 +54,7 @@ jobs: exit $BUILD_STATUS - name: Nix gosum diverging - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 if: failure() && steps.build.outcome == 'failure' with: github-token: ${{secrets.GITHUB_TOKEN}} @@ -67,7 +66,7 @@ jobs: body: 'Nix build failed with wrong gosum, please update "vendorSha256" (${{ steps.build.outputs.OLD_HASH }}) for the "headscale" package in flake.nix with the new SHA: ${{ steps.build.outputs.NEW_HASH }}' }) - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 if: steps.changed-files.outputs.files == 'true' with: name: headscale-linux @@ -82,22 +81,20 @@ jobs: - "GOARCH=arm64 GOOS=darwin" - "GOARCH=amd64 GOOS=darwin" steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3 with: - primary-key: - nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', + primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', '**/flake.lock') }} restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }} - name: Run go cross compile env: CGO_ENABLED: 0 - run: - env ${{ matrix.env }} nix develop --command -- go build -o "headscale" + run: env ${{ matrix.env }} nix develop --command -- go build -o "headscale" ./cmd/headscale - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: "headscale-${{ matrix.env }}" path: "headscale" diff --git a/.github/workflows/check-generated.yml b/.github/workflows/check-generated.yml index 17073a35..43f1d62d 100644 --- a/.github/workflows/check-generated.yml +++ b/.github/workflows/check-generated.yml @@ -16,7 +16,7 @@ jobs: check-generated: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 2 - name: Get changed files @@ -31,7 +31,7 @@ jobs: - '**/*.proto' - 'buf.gen.yaml' - 'tools/**' - - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31 + - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 if: steps.changed-files.outputs.files == 'true' - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3 if: steps.changed-files.outputs.files == 'true' diff --git a/.github/workflows/check-tests.yaml b/.github/workflows/check-tests.yaml index f75a2297..63a18141 100644 --- a/.github/workflows/check-tests.yaml +++ b/.github/workflows/check-tests.yaml @@ -10,7 +10,7 @@ jobs: check-tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 2 - name: Get changed files @@ -24,13 +24,12 @@ jobs: - '**/*.go' - 'integration_test/' - 'config-example.yaml' - - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31 + - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 if: steps.changed-files.outputs.files == 'true' - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3 if: steps.changed-files.outputs.files == 'true' with: - primary-key: - nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', + primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', '**/flake.lock') }} restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }} diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml index 7d06b6a6..0a8be5c1 100644 --- a/.github/workflows/docs-deploy.yml +++ b/.github/workflows/docs-deploy.yml @@ -21,15 +21,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 - name: Install python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: 3.x - name: Setup cache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache@a7833574556fa59680c1b7cb190c1735db73ebf0 # v5.0.0 with: key: ${{ github.ref }} path: .cache diff --git a/.github/workflows/docs-test.yml b/.github/workflows/docs-test.yml index 63c547c8..cab8f95c 100644 --- a/.github/workflows/docs-test.yml +++ b/.github/workflows/docs-test.yml @@ -11,13 +11,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: 3.x - name: Setup cache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache@a7833574556fa59680c1b7cb190c1735db73ebf0 # v5.0.0 with: key: ${{ github.ref }} path: .cache diff --git a/.github/workflows/gh-actions-updater.yaml b/.github/workflows/gh-actions-updater.yaml index 6bda3440..647e27dc 100644 --- a/.github/workflows/gh-actions-updater.yaml +++ b/.github/workflows/gh-actions-updater.yaml @@ -11,13 +11,13 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: # [Required] Access token with `workflow` scope. token: ${{ secrets.WORKFLOW_SECRET }} - name: Run GitHub Actions Version Updater - uses: saadmk11/github-actions-version-updater@64be81ba69383f81f2be476703ea6570c4c8686e # v0.8.1 + uses: saadmk11/github-actions-version-updater@d8781caf11d11168579c8e5e94f62b068038f442 # v0.9.0 with: # [Required] Access token with `workflow` scope. token: ${{ secrets.WORKFLOW_SECRET }} diff --git a/.github/workflows/integration-test-template.yml b/.github/workflows/integration-test-template.yml index 79b1ed15..24cc51e7 100644 --- a/.github/workflows/integration-test-template.yml +++ b/.github/workflows/integration-test-template.yml @@ -28,23 +28,12 @@ jobs: # that triggered the build. HAS_TAILSCALE_SECRET: ${{ secrets.TS_OAUTH_CLIENT_ID }} steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 2 - - name: Get changed files - id: changed-files - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 - with: - filters: | - files: - - '*.nix' - - 'go.*' - - '**/*.go' - - 'integration_test/' - - 'config-example.yaml' - name: Tailscale if: ${{ env.HAS_TAILSCALE_SECRET }} - uses: tailscale/github-action@6986d2c82a91fbac2949fe01f5bab95cf21b5102 # v3.2.2 + uses: tailscale/github-action@a392da0a182bba0e9613b6243ebd69529b1878aa # v4.1.0 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} @@ -52,28 +41,62 @@ jobs: - name: Setup SSH server for Actor if: ${{ env.HAS_TAILSCALE_SECRET }} uses: alexellis/setup-sshd-actor@master - - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31 - if: steps.changed-files.outputs.files == 'true' - - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3 - if: steps.changed-files.outputs.files == 'true' + - name: Download headscale image + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: - primary-key: - nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', - '**/flake.lock') }} - restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }} + name: headscale-image + path: /tmp/artifacts + - name: Download tailscale HEAD image + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: tailscale-head-image + path: /tmp/artifacts + - name: Download hi binary + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: hi-binary + path: /tmp/artifacts + - name: Download Go cache + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: go-cache + path: /tmp/artifacts + - name: Download postgres image + if: ${{ inputs.postgres_flag == '--postgres=1' }} + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: postgres-image + path: /tmp/artifacts + - name: Load Docker images, Go cache, and prepare binary + run: | + gunzip -c /tmp/artifacts/headscale-image.tar.gz | docker load + gunzip -c /tmp/artifacts/tailscale-head-image.tar.gz | docker load + if [ -f /tmp/artifacts/postgres-image.tar.gz ]; then + gunzip -c /tmp/artifacts/postgres-image.tar.gz | docker load + fi + chmod +x /tmp/artifacts/hi + docker images + # Extract Go cache to host directories for bind mounting + mkdir -p /tmp/go-cache + tar -xzf /tmp/artifacts/go-cache.tar.gz -C /tmp/go-cache + ls -la /tmp/go-cache/ /tmp/go-cache/.cache/ - name: Run Integration Test - if: always() && steps.changed-files.outputs.files == 'true' - run: - nix develop --command -- hi run --stats --ts-memory-limit=300 --hs-memory-limit=1500 "^${{ inputs.test }}$" \ + env: + HEADSCALE_INTEGRATION_HEADSCALE_IMAGE: headscale:${{ github.sha }} + HEADSCALE_INTEGRATION_TAILSCALE_IMAGE: tailscale-head:${{ github.sha }} + HEADSCALE_INTEGRATION_POSTGRES_IMAGE: ${{ inputs.postgres_flag == '--postgres=1' && format('postgres:{0}', github.sha) || '' }} + HEADSCALE_INTEGRATION_GO_CACHE: /tmp/go-cache/go + HEADSCALE_INTEGRATION_GO_BUILD_CACHE: /tmp/go-cache/.cache/go-build + run: /tmp/artifacts/hi run --stats --ts-memory-limit=300 --hs-memory-limit=1500 "^${{ inputs.test }}$" \ --timeout=120m \ ${{ inputs.postgres_flag }} - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - if: always() && steps.changed-files.outputs.files == 'true' + - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + if: always() with: name: ${{ inputs.database_name }}-${{ inputs.test }}-logs path: "control_logs/*/*.log" - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - if: always() && steps.changed-files.outputs.files == 'true' + - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + if: always() with: name: ${{ inputs.database_name }}-${{ inputs.test }}-artifacts path: control_logs/ diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2959c18a..75088b38 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,7 +10,7 @@ jobs: golangci-lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 2 - name: Get changed files @@ -24,13 +24,12 @@ jobs: - '**/*.go' - 'integration_test/' - 'config-example.yaml' - - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31 + - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 if: steps.changed-files.outputs.files == 'true' - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3 if: steps.changed-files.outputs.files == 'true' with: - primary-key: - nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', + primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', '**/flake.lock') }} restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }} @@ -46,7 +45,7 @@ jobs: prettier-lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 2 - name: Get changed files @@ -65,13 +64,12 @@ jobs: - '**/*.css' - '**/*.scss' - '**/*.html' - - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31 + - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 if: steps.changed-files.outputs.files == 'true' - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3 if: steps.changed-files.outputs.files == 'true' with: - primary-key: - nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', + primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', '**/flake.lock') }} restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }} @@ -83,12 +81,11 @@ jobs: proto-lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3 with: - primary-key: - nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', + primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', '**/flake.lock') }} restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }} diff --git a/.github/workflows/nix-module-test.yml b/.github/workflows/nix-module-test.yml index 18f40f91..68ad9545 100644 --- a/.github/workflows/nix-module-test.yml +++ b/.github/workflows/nix-module-test.yml @@ -19,7 +19,7 @@ jobs: contents: read steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 2 @@ -38,14 +38,13 @@ jobs: - 'cmd/**' - 'hscontrol/**' - - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31 + - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 if: steps.changed-files.outputs.nix == 'true' || steps.changed-files.outputs.go == 'true' - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3 if: steps.changed-files.outputs.nix == 'true' || steps.changed-files.outputs.go == 'true' with: - primary-key: - nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', + primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', '**/flake.lock') }} restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5b6fd18d..4835e255 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,28 +13,27 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 - name: Login to DockerHub - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GHCR - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31 + - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3 with: - primary-key: - nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', + primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', '**/flake.lock') }} restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 1041f1af..0915ec2c 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -12,16 +12,14 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0 + - uses: actions/stale@997185467fa4f803885201cee163a9f38240193d # v10.1.1 with: days-before-issue-stale: 90 days-before-issue-close: 7 stale-issue-label: "stale" - stale-issue-message: - "This issue is stale because it has been open for 90 days with no + stale-issue-message: "This issue is stale because it has been open for 90 days with no activity." - close-issue-message: - "This issue was closed because it has been inactive for 14 days + close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale." days-before-pr-stale: -1 days-before-pr-close: -1 diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml index a9597cc3..aaff9575 100644 --- a/.github/workflows/test-integration.yaml +++ b/.github/workflows/test-integration.yaml @@ -7,7 +7,117 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: + # build: Builds binaries and Docker images once, uploads as artifacts for reuse. + # build-postgres: Pulls postgres image separately to avoid Docker Hub rate limits. + # sqlite: Runs all integration tests with SQLite backend. + # postgres: Runs a subset of tests with PostgreSQL to verify database compatibility. + build: + runs-on: ubuntu-latest + outputs: + files-changed: ${{ steps.changed-files.outputs.files }} + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + fetch-depth: 2 + - name: Get changed files + id: changed-files + uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + with: + filters: | + files: + - '*.nix' + - 'go.*' + - '**/*.go' + - 'integration/**' + - 'config-example.yaml' + - '.github/workflows/test-integration.yaml' + - '.github/workflows/integration-test-template.yml' + - 'Dockerfile.*' + - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 + if: steps.changed-files.outputs.files == 'true' + - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3 + if: steps.changed-files.outputs.files == 'true' + with: + primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', '**/flake.lock') }} + restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }} + - name: Build binaries and warm Go cache + if: steps.changed-files.outputs.files == 'true' + run: | + # Build all Go binaries in one nix shell to maximize cache reuse + nix develop --command -- bash -c ' + go build -o hi ./cmd/hi + CGO_ENABLED=0 GOOS=linux go build -o headscale ./cmd/headscale + # Build integration test binary to warm the cache with all dependencies + go test -c ./integration -o /dev/null 2>/dev/null || true + ' + - name: Upload hi binary + if: steps.changed-files.outputs.files == 'true' + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: hi-binary + path: hi + retention-days: 10 + - name: Package Go cache + if: steps.changed-files.outputs.files == 'true' + run: | + # Package Go module cache and build cache + tar -czf go-cache.tar.gz -C ~ go .cache/go-build + - name: Upload Go cache + if: steps.changed-files.outputs.files == 'true' + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: go-cache + path: go-cache.tar.gz + retention-days: 10 + - name: Build headscale image + if: steps.changed-files.outputs.files == 'true' + run: | + docker build \ + --file Dockerfile.integration-ci \ + --tag headscale:${{ github.sha }} \ + . + docker save headscale:${{ github.sha }} | gzip > headscale-image.tar.gz + - name: Build tailscale HEAD image + if: steps.changed-files.outputs.files == 'true' + run: | + docker build \ + --file Dockerfile.tailscale-HEAD \ + --tag tailscale-head:${{ github.sha }} \ + . + docker save tailscale-head:${{ github.sha }} | gzip > tailscale-head-image.tar.gz + - name: Upload headscale image + if: steps.changed-files.outputs.files == 'true' + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: headscale-image + path: headscale-image.tar.gz + retention-days: 10 + - name: Upload tailscale HEAD image + if: steps.changed-files.outputs.files == 'true' + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: tailscale-head-image + path: tailscale-head-image.tar.gz + retention-days: 10 + build-postgres: + runs-on: ubuntu-latest + needs: build + if: needs.build.outputs.files-changed == 'true' + steps: + - name: Pull and save postgres image + run: | + docker pull postgres:latest + docker tag postgres:latest postgres:${{ github.sha }} + docker save postgres:${{ github.sha }} | gzip > postgres-image.tar.gz + - name: Upload postgres image + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: postgres-image + path: postgres-image.tar.gz + retention-days: 10 sqlite: + needs: build + if: needs.build.outputs.files-changed == 'true' strategy: fail-fast: false matrix: @@ -121,11 +231,14 @@ jobs: - TestTagsAdminAPICannotRemoveAllTags - TestTagsAdminAPICannotSetInvalidFormat uses: ./.github/workflows/integration-test-template.yml + secrets: inherit with: test: ${{ matrix.test }} postgres_flag: "--postgres=0" database_name: "sqlite" postgres: + needs: [build, build-postgres] + if: needs.build.outputs.files-changed == 'true' strategy: fail-fast: false matrix: @@ -136,6 +249,7 @@ jobs: - TestPingAllByIPManyUpDown - TestSubnetRouterMultiNetwork uses: ./.github/workflows/integration-test-template.yml + secrets: inherit with: test: ${{ matrix.test }} postgres_flag: "--postgres=1" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d43f8e83..31eb431b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 2 @@ -27,13 +27,12 @@ jobs: - 'integration_test/' - 'config-example.yaml' - - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31 + - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 if: steps.changed-files.outputs.files == 'true' - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3 if: steps.changed-files.outputs.files == 'true' with: - primary-key: - nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', + primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', '**/flake.lock') }} restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }} diff --git a/Dockerfile.integration b/Dockerfile.integration index c02223ff..3916f140 100644 --- a/Dockerfile.integration +++ b/Dockerfile.integration @@ -2,27 +2,43 @@ # and are in no way endorsed by Headscale's maintainers as an # official nor supported release or distribution. -FROM docker.io/golang:1.25-trixie +FROM docker.io/golang:1.25-trixie AS builder ARG VERSION=dev ENV GOPATH /go WORKDIR /go/src/headscale -RUN apt-get --update install --no-install-recommends --yes less jq sqlite3 dnsutils \ - && apt-get dist-clean -RUN mkdir -p /var/run/headscale - -# Install delve debugger +# Install delve debugger first - rarely changes, good cache candidate RUN go install github.com/go-delve/delve/cmd/dlv@latest +# Download dependencies - only invalidated when go.mod/go.sum change COPY go.mod go.sum /go/src/headscale/ RUN go mod download +# Copy source and build - invalidated on any source change COPY . . # Build debug binary with debug symbols for delve RUN CGO_ENABLED=0 GOOS=linux go build -gcflags="all=-N -l" -o /go/bin/headscale ./cmd/headscale +# Runtime stage +FROM debian:trixie-slim + +RUN apt-get --update install --no-install-recommends --yes \ + less jq sqlite3 dnsutils ca-certificates procps bash findutils curl traceroute python3 \ + && apt-get dist-clean + +RUN mkdir -p /var/run/headscale + +# Copy binaries from builder +COPY --from=builder /go/bin/headscale /usr/local/bin/headscale +COPY --from=builder /go/bin/dlv /usr/local/bin/dlv + +# Copy source code for delve source-level debugging +COPY --from=builder /go/src/headscale /go/src/headscale + +WORKDIR /go/src/headscale + # Need to reset the entrypoint or everything will run as a busybox script ENTRYPOINT [] EXPOSE 8080/tcp 40000/tcp -CMD ["/go/bin/dlv", "--listen=0.0.0.0:40000", "--headless=true", "--api-version=2", "--accept-multiclient", "exec", "/go/bin/headscale", "--"] +CMD ["dlv", "--listen=0.0.0.0:40000", "--headless=true", "--api-version=2", "--accept-multiclient", "exec", "/usr/local/bin/headscale", "--"] diff --git a/Dockerfile.integration-ci b/Dockerfile.integration-ci new file mode 100644 index 00000000..d0907714 --- /dev/null +++ b/Dockerfile.integration-ci @@ -0,0 +1,17 @@ +# Minimal CI image - expects pre-built headscale binary in build context +# For local development with delve debugging, use Dockerfile.integration instead + +FROM debian:trixie-slim + +RUN apt-get --update install --no-install-recommends --yes \ + less jq sqlite3 dnsutils ca-certificates procps bash findutils curl traceroute \ + && apt-get dist-clean + +RUN mkdir -p /var/run/headscale + +# Copy pre-built headscale binary from build context +COPY headscale /usr/local/bin/headscale + +ENTRYPOINT [] +EXPOSE 8080/tcp +CMD ["/usr/local/bin/headscale"] diff --git a/Dockerfile.tailscale-HEAD b/Dockerfile.tailscale-HEAD index 240d528b..2df355d6 100644 --- a/Dockerfile.tailscale-HEAD +++ b/Dockerfile.tailscale-HEAD @@ -37,7 +37,7 @@ RUN GOARCH=$TARGETARCH go install -tags="${BUILD_TAGS}" -ldflags="\ -v ./cmd/tailscale ./cmd/tailscaled ./cmd/containerboot FROM alpine:3.22 -RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables curl +RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables curl traceroute COPY --from=build-env /go/bin/* /usr/local/bin/ # For compat with the previous run.sh, although ideally you should be diff --git a/cmd/hi/docker.go b/cmd/hi/docker.go index 9df7dcc8..65250e65 100644 --- a/cmd/hi/docker.go +++ b/cmd/hi/docker.go @@ -301,6 +301,11 @@ func createGoTestContainer(ctx context.Context, cli *client.Client, config *RunC "HEADSCALE_INTEGRATION_RUN_ID=" + runID, } + // Pass through CI environment variable for CI detection + if ci := os.Getenv("CI"); ci != "" { + env = append(env, "CI="+ci) + } + // Pass through all HEADSCALE_INTEGRATION_* environment variables for _, e := range os.Environ() { if strings.HasPrefix(e, "HEADSCALE_INTEGRATION_") { @@ -313,6 +318,10 @@ func createGoTestContainer(ctx context.Context, cli *client.Client, config *RunC env = append(env, e) } } + + // Set GOCACHE to a known location (used by both bind mount and volume cases) + env = append(env, "GOCACHE=/cache/go-build") + containerConfig := &container.Config{ Image: "golang:" + config.GoVersion, Cmd: goTestCmd, @@ -332,20 +341,43 @@ func createGoTestContainer(ctx context.Context, cli *client.Client, config *RunC log.Printf("Using Docker socket: %s", dockerSocketPath) } + binds := []string{ + fmt.Sprintf("%s:%s", projectRoot, projectRoot), + dockerSocketPath + ":/var/run/docker.sock", + logsDir + ":/tmp/control", + } + + // Use bind mounts for Go cache if provided via environment variables, + // otherwise fall back to Docker volumes for local development + var mounts []mount.Mount + + goCache := os.Getenv("HEADSCALE_INTEGRATION_GO_CACHE") + goBuildCache := os.Getenv("HEADSCALE_INTEGRATION_GO_BUILD_CACHE") + + if goCache != "" { + binds = append(binds, goCache+":/go") + } else { + mounts = append(mounts, mount.Mount{ + Type: mount.TypeVolume, + Source: "hs-integration-go-cache", + Target: "/go", + }) + } + + if goBuildCache != "" { + binds = append(binds, goBuildCache+":/cache/go-build") + } else { + mounts = append(mounts, mount.Mount{ + Type: mount.TypeVolume, + Source: "hs-integration-go-build-cache", + Target: "/cache/go-build", + }) + } + hostConfig := &container.HostConfig{ AutoRemove: false, // We'll remove manually for better control - Binds: []string{ - fmt.Sprintf("%s:%s", projectRoot, projectRoot), - dockerSocketPath + ":/var/run/docker.sock", - logsDir + ":/tmp/control", - }, - Mounts: []mount.Mount{ - { - Type: mount.TypeVolume, - Source: "hs-integration-go-cache", - Target: "/go", - }, - }, + Binds: binds, + Mounts: mounts, } return cli.ContainerCreate(ctx, containerConfig, hostConfig, nil, nil, containerName) diff --git a/integration/hsic/hsic.go b/integration/hsic/hsic.go index 76910f2f..8a6b869d 100644 --- a/integration/hsic/hsic.go +++ b/integration/hsic/hsic.go @@ -33,7 +33,6 @@ import ( "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" "gopkg.in/yaml.v3" - "tailscale.com/envknob" "tailscale.com/tailcfg" "tailscale.com/util/mak" ) @@ -49,7 +48,12 @@ const ( IntegrationTestDockerFileName = "Dockerfile.integration" ) -var errHeadscaleStatusCodeNotOk = errors.New("headscale status code not ok") +var ( + errHeadscaleStatusCodeNotOk = errors.New("headscale status code not ok") + errInvalidHeadscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_HEADSCALE_IMAGE format, expected repository:tag") + errHeadscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE must be set in CI") + errInvalidPostgresImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_POSTGRES_IMAGE format, expected repository:tag") +) type fileInContainer struct { path string @@ -70,7 +74,6 @@ type HeadscaleInContainer struct { // optional config port int extraPorts []string - debugPort int caCerts [][]byte hostPortBindings map[string][]string aclPolicy *policyv2.Policy @@ -281,24 +284,9 @@ func WithDERPAsIP() Option { } } -// WithDebugPort sets the debug port for delve debugging. -func WithDebugPort(port int) Option { - return func(hsic *HeadscaleInContainer) { - hsic.debugPort = port - } -} - // buildEntrypoint builds the container entrypoint command based on configuration. func (hsic *HeadscaleInContainer) buildEntrypoint() []string { - debugCmd := fmt.Sprintf( - "/go/bin/dlv --listen=0.0.0.0:%d --headless=true --api-version=2 --accept-multiclient --allow-non-terminal-interactive=true exec /go/bin/headscale --continue -- serve", - hsic.debugPort, - ) - - entrypoint := fmt.Sprintf( - "/bin/sleep 3 ; update-ca-certificates ; %s ; /bin/sleep 30", - debugCmd, - ) + entrypoint := "/bin/sleep 3 ; update-ca-certificates ; /usr/local/bin/headscale serve ; /bin/sleep 30" return []string{"/bin/bash", "-c", entrypoint} } @@ -316,18 +304,9 @@ func New( hostname := "hs-" + hash - // Get debug port from environment or use default - debugPort := 40000 - if envDebugPort := envknob.String("HEADSCALE_DEBUG_PORT"); envDebugPort != "" { - if port, err := strconv.Atoi(envDebugPort); err == nil { - debugPort = port - } - } - hsic := &HeadscaleInContainer{ - hostname: hostname, - port: headscaleDefaultPort, - debugPort: debugPort, + hostname: hostname, + port: headscaleDefaultPort, pool: pool, networks: networks, @@ -344,7 +323,6 @@ func New( log.Println("NAME: ", hsic.hostname) portProto := fmt.Sprintf("%d/tcp", hsic.port) - debugPortProto := fmt.Sprintf("%d/tcp", hsic.debugPort) headscaleBuildOptions := &dockertest.BuildOptions{ Dockerfile: IntegrationTestDockerFileName, @@ -359,10 +337,24 @@ func New( hsic.env["HEADSCALE_DATABASE_POSTGRES_NAME"] = "headscale" delete(hsic.env, "HEADSCALE_DATABASE_SQLITE_PATH") + // Determine postgres image - use prebuilt if available, otherwise pull from registry + pgRepo := "postgres" + pgTag := "latest" + + if prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_POSTGRES_IMAGE"); prebuiltImage != "" { + repo, tag, found := strings.Cut(prebuiltImage, ":") + if !found { + return nil, errInvalidPostgresImageFormat + } + + pgRepo = repo + pgTag = tag + } + pgRunOptions := &dockertest.RunOptions{ Name: "postgres-" + hash, - Repository: "postgres", - Tag: "latest", + Repository: pgRepo, + Tag: pgTag, Networks: networks, Env: []string{ "POSTGRES_USER=headscale", @@ -409,7 +401,7 @@ func New( runOptions := &dockertest.RunOptions{ Name: hsic.hostname, - ExposedPorts: append([]string{portProto, debugPortProto, "9090/tcp"}, hsic.extraPorts...), + ExposedPorts: append([]string{portProto, "9090/tcp"}, hsic.extraPorts...), Networks: networks, // Cmd: []string{"headscale", "serve"}, // TODO(kradalby): Get rid of this hack, we currently need to give us some @@ -418,13 +410,10 @@ func New( Env: env, } - // Always bind debug port and metrics port to predictable host ports + // Bind metrics port to predictable host port if runOptions.PortBindings == nil { runOptions.PortBindings = map[docker.Port][]docker.PortBinding{} } - runOptions.PortBindings[docker.Port(debugPortProto)] = []docker.PortBinding{ - {HostPort: strconv.Itoa(hsic.debugPort)}, - } runOptions.PortBindings["9090/tcp"] = []docker.PortBinding{ {HostPort: "49090"}, } @@ -451,52 +440,80 @@ func New( // Add integration test labels if running under hi tool dockertestutil.DockerAddIntegrationLabels(runOptions, "headscale") - container, err := pool.BuildAndRunWithBuildOptions( - headscaleBuildOptions, - runOptions, - dockertestutil.DockerRestartPolicy, - dockertestutil.DockerAllowLocalIPv6, - dockertestutil.DockerAllowNetworkAdministration, - ) - if err != nil { - // Try to get more detailed build output - log.Printf("Docker build failed, attempting to get detailed output...") + var container *dockertest.Resource - buildOutput, buildErr := dockertestutil.RunDockerBuildForDiagnostics(dockerContextPath, IntegrationTestDockerFileName) + // Check if a pre-built image is available via environment variable + prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE") - // Show the last 100 lines of build output to avoid overwhelming the logs - lines := strings.Split(buildOutput, "\n") - - const maxLines = 100 - - startLine := 0 - if len(lines) > maxLines { - startLine = len(lines) - maxLines + if prebuiltImage != "" { + log.Printf("Using pre-built headscale image: %s", prebuiltImage) + // Parse image into repository and tag + repo, tag, ok := strings.Cut(prebuiltImage, ":") + if !ok { + return nil, errInvalidHeadscaleImageFormat } - relevantOutput := strings.Join(lines[startLine:], "\n") + runOptions.Repository = repo + runOptions.Tag = tag - if buildErr != nil { - // The diagnostic build also failed - this is the real error - return nil, fmt.Errorf("could not start headscale container: %w\n\nDocker build failed. Last %d lines of output:\n%s", err, maxLines, relevantOutput) + container, err = pool.RunWithOptions( + runOptions, + dockertestutil.DockerRestartPolicy, + dockertestutil.DockerAllowLocalIPv6, + dockertestutil.DockerAllowNetworkAdministration, + ) + if err != nil { + return nil, fmt.Errorf("could not run pre-built headscale container %q: %w", prebuiltImage, err) } + } else if util.IsCI() { + return nil, errHeadscaleImageRequiredInCI + } else { + container, err = pool.BuildAndRunWithBuildOptions( + headscaleBuildOptions, + runOptions, + dockertestutil.DockerRestartPolicy, + dockertestutil.DockerAllowLocalIPv6, + dockertestutil.DockerAllowNetworkAdministration, + ) + if err != nil { + // Try to get more detailed build output + log.Printf("Docker build/run failed, attempting to get detailed output...") - if buildOutput != "" { - // Build succeeded on retry but container creation still failed - return nil, fmt.Errorf("could not start headscale container: %w\n\nDocker build succeeded on retry, but container creation failed. Last %d lines of build output:\n%s", err, maxLines, relevantOutput) + buildOutput, buildErr := dockertestutil.RunDockerBuildForDiagnostics(dockerContextPath, IntegrationTestDockerFileName) + + // Show the last 100 lines of build output to avoid overwhelming the logs + lines := strings.Split(buildOutput, "\n") + + const maxLines = 100 + + startLine := 0 + if len(lines) > maxLines { + startLine = len(lines) - maxLines + } + + relevantOutput := strings.Join(lines[startLine:], "\n") + + if buildErr != nil { + // The diagnostic build also failed - this is the real error + return nil, fmt.Errorf("could not start headscale container: %w\n\nDocker build failed. Last %d lines of output:\n%s", err, maxLines, relevantOutput) + } + + if buildOutput != "" { + // Build succeeded on retry but container creation still failed + return nil, fmt.Errorf("could not start headscale container: %w\n\nDocker build succeeded on retry, but container creation failed. Last %d lines of build output:\n%s", err, maxLines, relevantOutput) + } + + // No output at all - diagnostic build command may have failed + return nil, fmt.Errorf("could not start headscale container: %w\n\nUnable to get diagnostic build output (command may have failed silently)", err) } - - // No output at all - diagnostic build command may have failed - return nil, fmt.Errorf("could not start headscale container: %w\n\nUnable to get diagnostic build output (command may have failed silently)", err) } log.Printf("Created %s container\n", hsic.hostname) hsic.container = container log.Printf( - "Debug ports for %s: delve=%s, metrics/pprof=49090\n", + "Ports for %s: metrics/pprof=49090\n", hsic.hostname, - hsic.GetHostDebugPort(), ) // Write the CA certificates to the container @@ -886,16 +903,6 @@ func (t *HeadscaleInContainer) GetPort() string { return strconv.Itoa(t.port) } -// GetDebugPort returns the debug port as a string. -func (t *HeadscaleInContainer) GetDebugPort() string { - return strconv.Itoa(t.debugPort) -} - -// GetHostDebugPort returns the host port mapped to the debug port. -func (t *HeadscaleInContainer) GetHostDebugPort() string { - return strconv.Itoa(t.debugPort) -} - // GetHealthEndpoint returns a health endpoint for the HeadscaleInContainer // instance. func (t *HeadscaleInContainer) GetHealthEndpoint() string { diff --git a/integration/tsic/tsic.go b/integration/tsic/tsic.go index f60748a8..ff2690b0 100644 --- a/integration/tsic/tsic.go +++ b/integration/tsic/tsic.go @@ -54,6 +54,8 @@ var ( errTailscaleNotConnected = errors.New("tailscale not connected") errTailscaledNotReadyForLogin = errors.New("tailscaled not ready for login") errInvalidClientConfig = errors.New("verifiably invalid client config requested") + errInvalidTailscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_TAILSCALE_IMAGE format, expected repository:tag") + errTailscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE must be set in CI for HEAD version") ) const ( @@ -299,80 +301,119 @@ func New( switch version { case VersionHead: - buildOptions := &dockertest.BuildOptions{ - Dockerfile: "Dockerfile.tailscale-HEAD", - ContextDir: dockerContextPath, - BuildArgs: []docker.BuildArg{}, + // Check if a pre-built image is available via environment variable + prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE") + + // If custom build tags are required (e.g., for websocket DERP), we cannot use + // the pre-built image as it won't have the necessary code compiled in. + hasBuildTags := len(tsic.buildConfig.tags) > 0 + if hasBuildTags && prebuiltImage != "" { + log.Printf("Ignoring pre-built image %s because custom build tags are required: %v", + prebuiltImage, tsic.buildConfig.tags) + prebuiltImage = "" } - buildTags := strings.Join(tsic.buildConfig.tags, ",") - if len(buildTags) > 0 { - buildOptions.BuildArgs = append( - buildOptions.BuildArgs, - docker.BuildArg{ - Name: "BUILD_TAGS", - Value: buildTags, - }, - ) - } + if prebuiltImage != "" { + log.Printf("Using pre-built tailscale image: %s", prebuiltImage) - container, err = pool.BuildAndRunWithBuildOptions( - buildOptions, - tailscaleOptions, - dockertestutil.DockerRestartPolicy, - dockertestutil.DockerAllowLocalIPv6, - dockertestutil.DockerAllowNetworkAdministration, - dockertestutil.DockerMemoryLimit, - ) - if err != nil { - // Try to get more detailed build output - log.Printf("Docker build failed for %s, attempting to get detailed output...", hostname) - - buildOutput, buildErr := dockertestutil.RunDockerBuildForDiagnostics(dockerContextPath, "Dockerfile.tailscale-HEAD") - - // Show the last 100 lines of build output to avoid overwhelming the logs - lines := strings.Split(buildOutput, "\n") - - const maxLines = 100 - - startLine := 0 - if len(lines) > maxLines { - startLine = len(lines) - maxLines + // Parse image into repository and tag + repo, tag, ok := strings.Cut(prebuiltImage, ":") + if !ok { + return nil, errInvalidTailscaleImageFormat } - relevantOutput := strings.Join(lines[startLine:], "\n") + tailscaleOptions.Repository = repo + tailscaleOptions.Tag = tag - if buildErr != nil { - // The diagnostic build also failed - this is the real error - return nil, fmt.Errorf( - "%s could not start tailscale container (version: %s): %w\n\nDocker build failed. Last %d lines of output:\n%s", - hostname, - version, - err, - maxLines, - relevantOutput, + container, err = pool.RunWithOptions( + tailscaleOptions, + dockertestutil.DockerRestartPolicy, + dockertestutil.DockerAllowLocalIPv6, + dockertestutil.DockerAllowNetworkAdministration, + dockertestutil.DockerMemoryLimit, + ) + if err != nil { + return nil, fmt.Errorf("could not run pre-built tailscale container %q: %w", prebuiltImage, err) + } + } else if util.IsCI() && !hasBuildTags { + // In CI, we require a pre-built image unless custom build tags are needed + return nil, errTailscaleImageRequiredInCI + } else { + buildOptions := &dockertest.BuildOptions{ + Dockerfile: "Dockerfile.tailscale-HEAD", + ContextDir: dockerContextPath, + BuildArgs: []docker.BuildArg{}, + } + + buildTags := strings.Join(tsic.buildConfig.tags, ",") + if len(buildTags) > 0 { + buildOptions.BuildArgs = append( + buildOptions.BuildArgs, + docker.BuildArg{ + Name: "BUILD_TAGS", + Value: buildTags, + }, ) } - if buildOutput != "" { - // Build succeeded on retry but container creation still failed + container, err = pool.BuildAndRunWithBuildOptions( + buildOptions, + tailscaleOptions, + dockertestutil.DockerRestartPolicy, + dockertestutil.DockerAllowLocalIPv6, + dockertestutil.DockerAllowNetworkAdministration, + dockertestutil.DockerMemoryLimit, + ) + if err != nil { + // Try to get more detailed build output + log.Printf("Docker build failed for %s, attempting to get detailed output...", hostname) + + buildOutput, buildErr := dockertestutil.RunDockerBuildForDiagnostics(dockerContextPath, "Dockerfile.tailscale-HEAD") + + // Show the last 100 lines of build output to avoid overwhelming the logs + lines := strings.Split(buildOutput, "\n") + + const maxLines = 100 + + startLine := 0 + if len(lines) > maxLines { + startLine = len(lines) - maxLines + } + + relevantOutput := strings.Join(lines[startLine:], "\n") + + if buildErr != nil { + // The diagnostic build also failed - this is the real error + return nil, fmt.Errorf( + "%s could not start tailscale container (version: %s): %w\n\nDocker build failed. Last %d lines of output:\n%s", + hostname, + version, + err, + maxLines, + relevantOutput, + ) + } + + if buildOutput != "" { + // Build succeeded on retry but container creation still failed + return nil, fmt.Errorf( + "%s could not start tailscale container (version: %s): %w\n\nDocker build succeeded on retry, but container creation failed. Last %d lines of build output:\n%s", + hostname, + version, + err, + maxLines, + relevantOutput, + ) + } + + // No output at all - diagnostic build command may have failed return nil, fmt.Errorf( - "%s could not start tailscale container (version: %s): %w\n\nDocker build succeeded on retry, but container creation failed. Last %d lines of build output:\n%s", + "%s could not start tailscale container (version: %s): %w\n\nUnable to get diagnostic build output (command may have failed silently)", hostname, version, err, - maxLines, - relevantOutput, ) } - - // No output at all - diagnostic build command may have failed - return nil, fmt.Errorf( - "%s could not start tailscale container (version: %s): %w\n\nUnable to get diagnostic build output (command may have failed silently)", - hostname, - version, - err, - ) } case "unstable": tailscaleOptions.Repository = "tailscale/tailscale"