From 56dd7343001803379ce6951eaa03a7d5c68e9fc1 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Thu, 27 Apr 2023 16:57:11 +0200 Subject: [PATCH] Add go profiling flag, and enable on integration tests (#1382) --- ...st-integration-v2-TestACLAllowStarDst.yaml | 6 ++ ...-integration-v2-TestACLAllowUser80Dst.yaml | 6 ++ ...st-integration-v2-TestACLAllowUserDst.yaml | 6 ++ ...t-integration-v2-TestACLDenyAllPort80.yaml | 6 ++ ...ion-v2-TestACLDevice1CanAccessDevice2.yaml | 6 ++ ...egration-v2-TestACLHostsInNetMapTable.yaml | 6 ++ ...egration-v2-TestACLNamedHostsCanReach.yaml | 6 ++ ...-v2-TestACLNamedHostsCanReachBySubnet.yaml | 6 ++ ...ration-v2-TestAuthKeyLogoutAndRelogin.yaml | 6 ++ ...-TestAuthWebFlowAuthenticationPingAll.yaml | 6 ++ ...on-v2-TestAuthWebFlowLogoutAndRelogin.yaml | 6 ++ ...st-integration-v2-TestCreateTailscale.yaml | 6 ++ ...integration-v2-TestDERPServerScenario.yaml | 6 ++ ...est-integration-v2-TestEnablingRoutes.yaml | 6 ++ .../test-integration-v2-TestEphemeral.yaml | 6 ++ .../test-integration-v2-TestExpireNode.yaml | 6 ++ .../test-integration-v2-TestHeadscale.yaml | 6 ++ ...tion-v2-TestOIDCAuthenticationPingAll.yaml | 6 ++ ...TestOIDCExpireNodesBasedOnTokenExpiry.yaml | 6 ++ ...-integration-v2-TestPingAllByHostname.yaml | 6 ++ .../test-integration-v2-TestPingAllByIP.yaml | 6 ++ ...-integration-v2-TestPreAuthKeyCommand.yaml | 6 ++ ...estPreAuthKeyCommandReusableEphemeral.yaml | 6 ++ ...v2-TestPreAuthKeyCommandWithoutExpiry.yaml | 6 ++ ...st-integration-v2-TestResolveMagicDNS.yaml | 6 ++ ...-integration-v2-TestSSHIsBlockedInACL.yaml | 6 ++ ...ation-v2-TestSSHMultipleUsersAllToAll.yaml | 6 ++ ...integration-v2-TestSSHNoSSHConfigured.yaml | 6 ++ ...integration-v2-TestSSHOneUserAllToAll.yaml | 6 ++ ...ntegration-v2-TestSSUserOnlyIsolation.yaml | 6 ++ .../test-integration-v2-TestTaildrop.yaml | 6 ++ ...-v2-TestTailscaleNodesJoiningHeadcale.yaml | 6 ++ .../test-integration-v2-TestUserCommand.yaml | 6 ++ CHANGELOG.md | 3 + app.go | 1 - cmd/gh-action-integration-generator/main.go | 6 ++ cmd/headscale/headscale.go | 14 ++++ flake.nix | 6 +- go.mod | 3 + go.sum | 8 ++ integration/control.go | 1 + integration/hsic/hsic.go | 75 ++++++++++++++++++- integration/integrationutil/util.go | 21 ++++++ integration/scenario.go | 10 +-- 44 files changed, 333 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test-integration-v2-TestACLAllowStarDst.yaml b/.github/workflows/test-integration-v2-TestACLAllowStarDst.yaml index 47a1b60d..f63a6d2a 100644 --- a/.github/workflows/test-integration-v2-TestACLAllowStarDst.yaml +++ b/.github/workflows/test-integration-v2-TestACLAllowStarDst.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestACLAllowUser80Dst.yaml b/.github/workflows/test-integration-v2-TestACLAllowUser80Dst.yaml index 2e6c3c2e..3b3dd59e 100644 --- a/.github/workflows/test-integration-v2-TestACLAllowUser80Dst.yaml +++ b/.github/workflows/test-integration-v2-TestACLAllowUser80Dst.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestACLAllowUserDst.yaml b/.github/workflows/test-integration-v2-TestACLAllowUserDst.yaml index d745e871..54b425cf 100644 --- a/.github/workflows/test-integration-v2-TestACLAllowUserDst.yaml +++ b/.github/workflows/test-integration-v2-TestACLAllowUserDst.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestACLDenyAllPort80.yaml b/.github/workflows/test-integration-v2-TestACLDenyAllPort80.yaml index 767aa213..ed01bf71 100644 --- a/.github/workflows/test-integration-v2-TestACLDenyAllPort80.yaml +++ b/.github/workflows/test-integration-v2-TestACLDenyAllPort80.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestACLDevice1CanAccessDevice2.yaml b/.github/workflows/test-integration-v2-TestACLDevice1CanAccessDevice2.yaml index d25f774f..4efbf1a5 100644 --- a/.github/workflows/test-integration-v2-TestACLDevice1CanAccessDevice2.yaml +++ b/.github/workflows/test-integration-v2-TestACLDevice1CanAccessDevice2.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestACLHostsInNetMapTable.yaml b/.github/workflows/test-integration-v2-TestACLHostsInNetMapTable.yaml index 70b7b943..7445695c 100644 --- a/.github/workflows/test-integration-v2-TestACLHostsInNetMapTable.yaml +++ b/.github/workflows/test-integration-v2-TestACLHostsInNetMapTable.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestACLNamedHostsCanReach.yaml b/.github/workflows/test-integration-v2-TestACLNamedHostsCanReach.yaml index 2d624228..f0f663ca 100644 --- a/.github/workflows/test-integration-v2-TestACLNamedHostsCanReach.yaml +++ b/.github/workflows/test-integration-v2-TestACLNamedHostsCanReach.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestACLNamedHostsCanReachBySubnet.yaml b/.github/workflows/test-integration-v2-TestACLNamedHostsCanReachBySubnet.yaml index 03951239..daeb6eee 100644 --- a/.github/workflows/test-integration-v2-TestACLNamedHostsCanReachBySubnet.yaml +++ b/.github/workflows/test-integration-v2-TestACLNamedHostsCanReachBySubnet.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestAuthKeyLogoutAndRelogin.yaml b/.github/workflows/test-integration-v2-TestAuthKeyLogoutAndRelogin.yaml index 8df6f22a..c603d4fe 100644 --- a/.github/workflows/test-integration-v2-TestAuthKeyLogoutAndRelogin.yaml +++ b/.github/workflows/test-integration-v2-TestAuthKeyLogoutAndRelogin.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestAuthWebFlowAuthenticationPingAll.yaml b/.github/workflows/test-integration-v2-TestAuthWebFlowAuthenticationPingAll.yaml index acf473d7..19d99017 100644 --- a/.github/workflows/test-integration-v2-TestAuthWebFlowAuthenticationPingAll.yaml +++ b/.github/workflows/test-integration-v2-TestAuthWebFlowAuthenticationPingAll.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestAuthWebFlowLogoutAndRelogin.yaml b/.github/workflows/test-integration-v2-TestAuthWebFlowLogoutAndRelogin.yaml index 6d70750c..a9eeb403 100644 --- a/.github/workflows/test-integration-v2-TestAuthWebFlowLogoutAndRelogin.yaml +++ b/.github/workflows/test-integration-v2-TestAuthWebFlowLogoutAndRelogin.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestCreateTailscale.yaml b/.github/workflows/test-integration-v2-TestCreateTailscale.yaml index 0ab32477..e342de61 100644 --- a/.github/workflows/test-integration-v2-TestCreateTailscale.yaml +++ b/.github/workflows/test-integration-v2-TestCreateTailscale.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestDERPServerScenario.yaml b/.github/workflows/test-integration-v2-TestDERPServerScenario.yaml index 610794e8..b3246c27 100644 --- a/.github/workflows/test-integration-v2-TestDERPServerScenario.yaml +++ b/.github/workflows/test-integration-v2-TestDERPServerScenario.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestEnablingRoutes.yaml b/.github/workflows/test-integration-v2-TestEnablingRoutes.yaml index 60637415..f6607736 100644 --- a/.github/workflows/test-integration-v2-TestEnablingRoutes.yaml +++ b/.github/workflows/test-integration-v2-TestEnablingRoutes.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestEphemeral.yaml b/.github/workflows/test-integration-v2-TestEphemeral.yaml index 068c028d..f3af06ac 100644 --- a/.github/workflows/test-integration-v2-TestEphemeral.yaml +++ b/.github/workflows/test-integration-v2-TestEphemeral.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestExpireNode.yaml b/.github/workflows/test-integration-v2-TestExpireNode.yaml index bfa4faa7..9daac7e1 100644 --- a/.github/workflows/test-integration-v2-TestExpireNode.yaml +++ b/.github/workflows/test-integration-v2-TestExpireNode.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestHeadscale.yaml b/.github/workflows/test-integration-v2-TestHeadscale.yaml index b7849ae4..d93a54c9 100644 --- a/.github/workflows/test-integration-v2-TestHeadscale.yaml +++ b/.github/workflows/test-integration-v2-TestHeadscale.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestOIDCAuthenticationPingAll.yaml b/.github/workflows/test-integration-v2-TestOIDCAuthenticationPingAll.yaml index 86dcd8f6..003319e6 100644 --- a/.github/workflows/test-integration-v2-TestOIDCAuthenticationPingAll.yaml +++ b/.github/workflows/test-integration-v2-TestOIDCAuthenticationPingAll.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestOIDCExpireNodesBasedOnTokenExpiry.yaml b/.github/workflows/test-integration-v2-TestOIDCExpireNodesBasedOnTokenExpiry.yaml index 7380c4e8..cd8d9c34 100644 --- a/.github/workflows/test-integration-v2-TestOIDCExpireNodesBasedOnTokenExpiry.yaml +++ b/.github/workflows/test-integration-v2-TestOIDCExpireNodesBasedOnTokenExpiry.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestPingAllByHostname.yaml b/.github/workflows/test-integration-v2-TestPingAllByHostname.yaml index 691b306c..12633c80 100644 --- a/.github/workflows/test-integration-v2-TestPingAllByHostname.yaml +++ b/.github/workflows/test-integration-v2-TestPingAllByHostname.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestPingAllByIP.yaml b/.github/workflows/test-integration-v2-TestPingAllByIP.yaml index 96172998..834fce94 100644 --- a/.github/workflows/test-integration-v2-TestPingAllByIP.yaml +++ b/.github/workflows/test-integration-v2-TestPingAllByIP.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestPreAuthKeyCommand.yaml b/.github/workflows/test-integration-v2-TestPreAuthKeyCommand.yaml index 32843253..6bc5ded6 100644 --- a/.github/workflows/test-integration-v2-TestPreAuthKeyCommand.yaml +++ b/.github/workflows/test-integration-v2-TestPreAuthKeyCommand.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestPreAuthKeyCommandReusableEphemeral.yaml b/.github/workflows/test-integration-v2-TestPreAuthKeyCommandReusableEphemeral.yaml index bbcb3f67..003128b6 100644 --- a/.github/workflows/test-integration-v2-TestPreAuthKeyCommandReusableEphemeral.yaml +++ b/.github/workflows/test-integration-v2-TestPreAuthKeyCommandReusableEphemeral.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestPreAuthKeyCommandWithoutExpiry.yaml b/.github/workflows/test-integration-v2-TestPreAuthKeyCommandWithoutExpiry.yaml index a3dac5c5..619b2647 100644 --- a/.github/workflows/test-integration-v2-TestPreAuthKeyCommandWithoutExpiry.yaml +++ b/.github/workflows/test-integration-v2-TestPreAuthKeyCommandWithoutExpiry.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestResolveMagicDNS.yaml b/.github/workflows/test-integration-v2-TestResolveMagicDNS.yaml index 98c0563d..ad6aba62 100644 --- a/.github/workflows/test-integration-v2-TestResolveMagicDNS.yaml +++ b/.github/workflows/test-integration-v2-TestResolveMagicDNS.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestSSHIsBlockedInACL.yaml b/.github/workflows/test-integration-v2-TestSSHIsBlockedInACL.yaml index 509e45ea..5664151b 100644 --- a/.github/workflows/test-integration-v2-TestSSHIsBlockedInACL.yaml +++ b/.github/workflows/test-integration-v2-TestSSHIsBlockedInACL.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestSSHMultipleUsersAllToAll.yaml b/.github/workflows/test-integration-v2-TestSSHMultipleUsersAllToAll.yaml index 71d5533b..286ad486 100644 --- a/.github/workflows/test-integration-v2-TestSSHMultipleUsersAllToAll.yaml +++ b/.github/workflows/test-integration-v2-TestSSHMultipleUsersAllToAll.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestSSHNoSSHConfigured.yaml b/.github/workflows/test-integration-v2-TestSSHNoSSHConfigured.yaml index 0707fdad..e4fa04a6 100644 --- a/.github/workflows/test-integration-v2-TestSSHNoSSHConfigured.yaml +++ b/.github/workflows/test-integration-v2-TestSSHNoSSHConfigured.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestSSHOneUserAllToAll.yaml b/.github/workflows/test-integration-v2-TestSSHOneUserAllToAll.yaml index 6ab4f083..3ef3e991 100644 --- a/.github/workflows/test-integration-v2-TestSSHOneUserAllToAll.yaml +++ b/.github/workflows/test-integration-v2-TestSSHOneUserAllToAll.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestSSUserOnlyIsolation.yaml b/.github/workflows/test-integration-v2-TestSSUserOnlyIsolation.yaml index b3ddf251..52fd09a4 100644 --- a/.github/workflows/test-integration-v2-TestSSUserOnlyIsolation.yaml +++ b/.github/workflows/test-integration-v2-TestSSUserOnlyIsolation.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestTaildrop.yaml b/.github/workflows/test-integration-v2-TestTaildrop.yaml index b10723da..97601239 100644 --- a/.github/workflows/test-integration-v2-TestTaildrop.yaml +++ b/.github/workflows/test-integration-v2-TestTaildrop.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestTailscaleNodesJoiningHeadcale.yaml b/.github/workflows/test-integration-v2-TestTailscaleNodesJoiningHeadcale.yaml index fc685836..ca70dd72 100644 --- a/.github/workflows/test-integration-v2-TestTailscaleNodesJoiningHeadcale.yaml +++ b/.github/workflows/test-integration-v2-TestTailscaleNodesJoiningHeadcale.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/.github/workflows/test-integration-v2-TestUserCommand.yaml b/.github/workflows/test-integration-v2-TestUserCommand.yaml index 7a4c9119..5b37b9e7 100644 --- a/.github/workflows/test-integration-v2-TestUserCommand.yaml +++ b/.github/workflows/test-integration-v2-TestUserCommand.yaml @@ -55,3 +55,9 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" diff --git a/CHANGELOG.md b/CHANGELOG.md index 0345ec90..1d256705 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ ### Changes +- Add environment flags to enable pprof (profiling) [#1382](https://github.com/juanfont/headscale/pull/1382) + - Profiles are continously generated in our integration tests. + ## 0.22.1 (2023-04-20) ### Changes diff --git a/app.go b/app.go index 480689bc..b3066916 100644 --- a/app.go +++ b/app.go @@ -820,7 +820,6 @@ func (h *Headscale) Serve() error { // And we're done: cancel() - os.Exit(0) } } } diff --git a/cmd/gh-action-integration-generator/main.go b/cmd/gh-action-integration-generator/main.go index 1ea69dae..cc0e7c92 100644 --- a/cmd/gh-action-integration-generator/main.go +++ b/cmd/gh-action-integration-generator/main.go @@ -76,6 +76,12 @@ jobs: with: name: logs path: "control_logs/*.log" + + - uses: actions/upload-artifact@v3 + if: always() && steps.changed-files.outputs.any_changed == 'true' + with: + name: pprof + path: "control_logs/*.pprof.tar" `), ) ) diff --git a/cmd/headscale/headscale.go b/cmd/headscale/headscale.go index 40772a93..26055f05 100644 --- a/cmd/headscale/headscale.go +++ b/cmd/headscale/headscale.go @@ -6,11 +6,25 @@ import ( "github.com/efekarakus/termcolor" "github.com/juanfont/headscale/cmd/headscale/cli" + "github.com/pkg/profile" "github.com/rs/zerolog" "github.com/rs/zerolog/log" ) func main() { + if _, enableProfile := os.LookupEnv("HEADSCALE_PROFILING_ENABLED"); enableProfile { + if profilePath, ok := os.LookupEnv("HEADSCALE_PROFILING_PATH"); ok { + err := os.MkdirAll(profilePath, os.ModePerm) + if err != nil { + log.Fatal().Err(err).Msg("failed to create profiling directory") + } + + defer profile.Start(profile.ProfilePath(profilePath)).Stop() + } else { + defer profile.Start().Stop() + } + } + var colors bool switch l := termcolor.SupportLevel(os.Stderr); l { case termcolor.Level16M: diff --git a/flake.nix b/flake.nix index 2986fa7d..71cb0d11 100644 --- a/flake.nix +++ b/flake.nix @@ -36,7 +36,7 @@ # When updating go.mod or go.sum, a new sha will need to be calculated, # update this if you have a mismatch after doing a change to thos files. - vendorSha256 = "sha256-lqMmTPLlnm4dGs3O+hS0LmR5LTWP5ICqMSkNnB+XXLI="; + vendorSha256 = "sha256-5a3SR77vYp8AsVFyFhBQtHn1I2kdlph+RuAkuT6hPIs="; ldflags = [ "-s" "-w" "-X github.com/juanfont/headscale/cmd/headscale/cli.Version=v${version}" ]; }; @@ -100,6 +100,10 @@ nfpm gotestsum + # 'dot' is needed for pprof graphs + # go tool pprof -http=: + graphviz + # Protobuf dependencies protobuf protoc-gen-go diff --git a/go.mod b/go.mod index e3c60072..b2c12957 100644 --- a/go.mod +++ b/go.mod @@ -64,6 +64,7 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/felixge/fgprof v0.9.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/fxamacker/cbor/v2 v2.4.0 // indirect github.com/glebarez/go-sqlite v1.20.3 // indirect @@ -75,6 +76,7 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-github v17.0.0+incompatible // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gookit/color v1.5.3 // indirect @@ -111,6 +113,7 @@ require ( github.com/opencontainers/runc v1.1.4 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pkg/profile v1.7.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect diff --git a/go.sum b/go.sum index 89115e47..26274d6e 100644 --- a/go.sum +++ b/go.sum @@ -129,6 +129,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= +github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -238,7 +240,9 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= @@ -272,6 +276,7 @@ github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -384,6 +389,8 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= +github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -669,6 +676,7 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/integration/control.go b/integration/control.go index 02d2750d..65432b3f 100644 --- a/integration/control.go +++ b/integration/control.go @@ -8,6 +8,7 @@ import ( type ControlServer interface { Shutdown() error SaveLog(string) error + SaveProfile(string) error Execute(command []string) (string, error) ConnectToNetwork(network *dockertest.Network) error GetHealthEndpoint() string diff --git a/integration/hsic/hsic.go b/integration/hsic/hsic.go index 6517b745..63034918 100644 --- a/integration/hsic/hsic.go +++ b/integration/hsic/hsic.go @@ -16,6 +16,9 @@ import ( "net" "net/http" "net/url" + "os" + "path" + "strings" "time" "github.com/davecgh/go-spew/spew" @@ -205,7 +208,10 @@ func New( ContextDir: dockerContextPath, } - env := []string{} + env := []string{ + "HEADSCALE_PROFILING_ENABLED=1", + "HEADSCALE_PROFILING_PATH=/tmp/profile", + } for key, value := range hsic.env { env = append(env, fmt.Sprintf("%s=%s", key, value)) } @@ -219,7 +225,7 @@ func New( // Cmd: []string{"headscale", "serve"}, // TODO(kradalby): Get rid of this hack, we currently need to give us some // to inject the headscale configuration further down. - Entrypoint: []string{"/bin/bash", "-c", "/bin/sleep 3 ; headscale serve"}, + Entrypoint: []string{"/bin/bash", "-c", "/bin/sleep 3 ; headscale serve ; /bin/sleep 30"}, Env: env, } @@ -305,6 +311,33 @@ func (t *HeadscaleInContainer) hasTLS() bool { // Shutdown stops and cleans up the Headscale container. func (t *HeadscaleInContainer) Shutdown() error { + err := t.SaveLog("/tmp/control") + if err != nil { + log.Printf( + "Failed to save log from control: %s", + fmt.Errorf("failed to save log from control: %w", err), + ) + } + + // Send a interrupt signal to the "headscale" process inside the container + // allowing it to shut down gracefully and flush the profile to disk. + // The container will live for a bit longer due to the sleep at the end. + err = t.SendInterrupt() + if err != nil { + log.Printf( + "Failed to send graceful interrupt to control: %s", + fmt.Errorf("failed to send graceful interrupt to control: %w", err), + ) + } + + err = t.SaveProfile("/tmp/control") + if err != nil { + log.Printf( + "Failed to save profile from control: %s", + fmt.Errorf("failed to save profile from control: %w", err), + ) + } + return t.pool.Purge(t.container) } @@ -314,6 +347,24 @@ func (t *HeadscaleInContainer) SaveLog(path string) error { return dockertestutil.SaveLog(t.pool, t.container, path) } +func (t *HeadscaleInContainer) SaveProfile(savePath string) error { + tarFile, err := t.FetchPath("/tmp/profile") + if err != nil { + return err + } + + err = os.WriteFile( + path.Join(savePath, t.hostname+".pprof.tar"), + tarFile, + os.ModePerm, + ) + if err != nil { + return err + } + + return nil +} + // Execute runs a command inside the Headscale container and returns the // result of stdout as a string. func (t *HeadscaleInContainer) Execute( @@ -498,6 +549,26 @@ func (t *HeadscaleInContainer) WriteFile(path string, data []byte) error { return integrationutil.WriteFileToContainer(t.pool, t.container, path, data) } +// FetchPath gets a path from inside the Headscale container and returns a tar +// file as byte array. +func (t *HeadscaleInContainer) FetchPath(path string) ([]byte, error) { + return integrationutil.FetchPathFromContainer(t.pool, t.container, path) +} + +func (t *HeadscaleInContainer) SendInterrupt() error { + pid, err := t.Execute([]string{"pidof", "headscale"}) + if err != nil { + return err + } + + _, err = t.Execute([]string{"kill", "-2", strings.Trim(pid, "'\n")}) + if err != nil { + return err + } + + return nil +} + // nolint func createCertificate(hostname string) ([]byte, []byte, error) { // From: diff --git a/integration/integrationutil/util.go b/integration/integrationutil/util.go index 640613b7..59eeeb17 100644 --- a/integration/integrationutil/util.go +++ b/integration/integrationutil/util.go @@ -72,3 +72,24 @@ func WriteFileToContainer( return nil } + +func FetchPathFromContainer( + pool *dockertest.Pool, + container *dockertest.Resource, + path string, +) ([]byte, error) { + buf := bytes.NewBuffer([]byte{}) + + err := pool.Client.DownloadFromContainer( + container.Container.ID, + docker.DownloadFromContainerOptions{ + OutputStream: buf, + Path: path, + }, + ) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/integration/scenario.go b/integration/scenario.go index 7efd08b0..2bb7ea66 100644 --- a/integration/scenario.go +++ b/integration/scenario.go @@ -149,15 +149,7 @@ func NewScenario() (*Scenario, error) { // environment running the tests. func (s *Scenario) Shutdown() error { s.controlServers.Range(func(_ string, control ControlServer) bool { - err := control.SaveLog("/tmp/control") - if err != nil { - log.Printf( - "Failed to save log from control: %s", - fmt.Errorf("failed to save log from control: %w", err), - ) - } - - err = control.Shutdown() + err := control.Shutdown() if err != nil { log.Printf( "Failed to shut down control: %s",