diff --git a/.github/workflows/gh-action-integration-generator.go b/.github/workflows/gh-action-integration-generator.go index 048ad768..c0a3d6aa 100644 --- a/.github/workflows/gh-action-integration-generator.go +++ b/.github/workflows/gh-action-integration-generator.go @@ -10,6 +10,55 @@ import ( "strings" ) +// testsToSplit defines tests that should be split into multiple CI jobs. +// Key is the test function name, value is a list of subtest prefixes. +// Each prefix becomes a separate CI job as "TestName/prefix". +// +// Example: TestAutoApproveMultiNetwork has subtests like: +// - TestAutoApproveMultiNetwork/authkey-tag-advertiseduringup-false-pol-database +// - TestAutoApproveMultiNetwork/webauth-user-advertiseduringup-true-pol-file +// +// Splitting by approver type (tag, user, group) creates 6 CI jobs with 4 tests each: +// - TestAutoApproveMultiNetwork/authkey-tag.* (4 tests) +// - TestAutoApproveMultiNetwork/authkey-user.* (4 tests) +// - TestAutoApproveMultiNetwork/authkey-group.* (4 tests) +// - TestAutoApproveMultiNetwork/webauth-tag.* (4 tests) +// - TestAutoApproveMultiNetwork/webauth-user.* (4 tests) +// - TestAutoApproveMultiNetwork/webauth-group.* (4 tests) +// +// This reduces load per CI job (4 tests instead of 12) to avoid infrastructure +// flakiness when running many sequential Docker-based integration tests. +var testsToSplit = map[string][]string{ + "TestAutoApproveMultiNetwork": { + "authkey-tag", + "authkey-user", + "authkey-group", + "webauth-tag", + "webauth-user", + "webauth-group", + }, +} + +// expandTests takes a list of test names and expands any that need splitting +// into multiple subtest patterns. +func expandTests(tests []string) []string { + var expanded []string + for _, test := range tests { + if prefixes, ok := testsToSplit[test]; ok { + // This test should be split into multiple jobs. + // We append ".*" to each prefix because the CI runner wraps patterns + // with ^...$ anchors. Without ".*", a pattern like "authkey$" wouldn't + // match "authkey-tag-advertiseduringup-false-pol-database". + for _, prefix := range prefixes { + expanded = append(expanded, fmt.Sprintf("%s/%s.*", test, prefix)) + } + } else { + expanded = append(expanded, test) + } + } + return expanded +} + func findTests() []string { rgBin, err := exec.LookPath("rg") if err != nil { @@ -66,8 +115,11 @@ func updateYAML(tests []string, jobName string, testPath string) { func main() { tests := findTests() - quotedTests := make([]string, len(tests)) - for i, test := range tests { + // Expand tests that should be split into multiple jobs + expandedTests := expandTests(tests) + + quotedTests := make([]string, len(expandedTests)) + for i, test := range expandedTests { quotedTests[i] = fmt.Sprintf("\"%s\"", test) } diff --git a/.github/workflows/integration-test-template.yml b/.github/workflows/integration-test-template.yml index 24cc51e7..0a884814 100644 --- a/.github/workflows/integration-test-template.yml +++ b/.github/workflows/integration-test-template.yml @@ -90,15 +90,22 @@ jobs: run: /tmp/artifacts/hi run --stats --ts-memory-limit=300 --hs-memory-limit=1500 "^${{ inputs.test }}$" \ --timeout=120m \ ${{ inputs.postgres_flag }} + # Sanitize test name for artifact upload (replace invalid characters: " : < > | * ? \ / with -) + - name: Sanitize test name for artifacts + if: always() + id: sanitize + run: echo "name=${TEST_NAME//[\":<>|*?\\\/]/-}" >> $GITHUB_OUTPUT + env: + TEST_NAME: ${{ inputs.test }} - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 if: always() with: - name: ${{ inputs.database_name }}-${{ inputs.test }}-logs + name: ${{ inputs.database_name }}-${{ steps.sanitize.outputs.name }}-logs path: "control_logs/*/*.log" - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 if: always() with: - name: ${{ inputs.database_name }}-${{ inputs.test }}-artifacts + name: ${{ inputs.database_name }}-${{ steps.sanitize.outputs.name }}-artifacts path: control_logs/ - name: Setup a blocking tmux session if: ${{ env.HAS_TAILSCALE_SECRET }} diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml index 19caa79c..a1b61cdb 100644 --- a/.github/workflows/test-integration.yaml +++ b/.github/workflows/test-integration.yaml @@ -195,7 +195,12 @@ jobs: - TestEnablingExitRoutes - TestSubnetRouterMultiNetwork - TestSubnetRouterMultiNetworkExitNode - - TestAutoApproveMultiNetwork + - TestAutoApproveMultiNetwork/authkey-tag.* + - TestAutoApproveMultiNetwork/authkey-user.* + - TestAutoApproveMultiNetwork/authkey-group.* + - TestAutoApproveMultiNetwork/webauth-tag.* + - TestAutoApproveMultiNetwork/webauth-user.* + - TestAutoApproveMultiNetwork/webauth-group.* - TestSubnetRouteACLFiltering - TestHeadscale - TestTailscaleNodesJoiningHeadcale