Merge branch 'main' into autoapprovers

.github/workflows/test-integration.yml

@@ -48,6 +48,15 @@ jobs:
           retry_on: error
           command: nix develop --command -- make test_integration_derp
 
+      - name: Run OIDC integration tests
+        if: steps.changed-files.outputs.any_changed == 'true'
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 240
+          max_attempts: 5
+          retry_on: error
+          command: nix develop --command -- make test_integration_oidc
+
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
         uses: nick-fields/retry@v2

CHANGELOG.md

@@ -2,11 +2,19 @@
 
 ## 0.17.0 (2022-XX-XX)
 
+### BREAKING
+
+- Log level option `log_level` was moved to a distinct `log` config section and renamed to `level` [#768](https://github.com/juanfont/headscale/pull/768)
+
+### Changes
+
 - Added support for Tailscale TS2021 protocol [#738](https://github.com/juanfont/headscale/pull/738)
 - Add ability to specify config location via env var `HEADSCALE_CONFIG` [#674](https://github.com/juanfont/headscale/issues/674)
 - Target Go 1.19 for Headscale [#778](https://github.com/juanfont/headscale/pull/778)
 - Target Tailscale v1.30.0 to build Headscale [#780](https://github.com/juanfont/headscale/pull/780)
 - Give a warning when running Headscale with reverse proxy improperly configured for WebSockets [#788](https://github.com/juanfont/headscale/pull/788)
+- Fix subnet routers with Primary Routes [#811](https://github.com/juanfont/headscale/pull/811)
+- Added support for JSON logs [#653](https://github.com/juanfont/headscale/issues/653)
 
 ## 0.16.4 (2022-08-21)
 

Makefile

@@ -24,7 +24,7 @@ dev: lint test build
 test:
 	@go test -coverprofile=coverage.out ./...
 
-test_integration: test_integration_cli test_integration_derp test_integration_general
+test_integration: test_integration_cli test_integration_derp test_integration_oidc test_integration_general
 
 test_integration_cli:
 	go test -failfast -tags integration_cli,integration -timeout 30m -count=1 ./...
@@ -35,6 +35,9 @@ test_integration_derp:
 test_integration_general:
 	go test -failfast -tags integration_general,integration -timeout 30m -count=1 ./...
 
+test_integration_oidc:
+	go test -failfast -tags integration_oidc,integration -timeout 30m -count=1 ./...
+
 coverprofile_func:
 	go tool cover -func=coverage.out
 

api_common.go

@@ -13,7 +13,7 @@ func (h *Headscale) generateMapResponse(
 		Str("func", "generateMapResponse").
 		Str("machine", mapRequest.Hostinfo.Hostname).
 		Msg("Creating Map response")
-	node, err := machine.toNode(h.cfg.BaseDomain, h.cfg.DNSConfig, true)
+	node, err := machine.toNode(h.cfg.BaseDomain, h.cfg.DNSConfig)
 	if err != nil {
 		log.Error().
 			Caller().
@@ -37,7 +37,7 @@ func (h *Headscale) generateMapResponse(
 
 	profiles := getMapResponseUserProfiles(*machine, peers)
 
-	nodePeers, err := peers.toNodes(h.cfg.BaseDomain, h.cfg.DNSConfig, true)
+	nodePeers, err := peers.toNodes(h.cfg.BaseDomain, h.cfg.DNSConfig)
 	if err != nil {
 		log.Error().
 			Caller().

cmd/headscale/cli/mockoidc.go

@@ -0,0 +1,100 @@
+package cli
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/oauth2-proxy/mockoidc"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+)
+
+const (
+	errMockOidcClientIDNotDefined     = Error("MOCKOIDC_CLIENT_ID not defined")
+	errMockOidcClientSecretNotDefined = Error("MOCKOIDC_CLIENT_SECRET not defined")
+	errMockOidcPortNotDefined         = Error("MOCKOIDC_PORT not defined")
+	accessTTL                         = 10 * time.Minute
+	refreshTTL                        = 60 * time.Minute
+)
+
+func init() {
+	rootCmd.AddCommand(mockOidcCmd)
+}
+
+var mockOidcCmd = &cobra.Command{
+	Use:   "mockoidc",
+	Short: "Runs a mock OIDC server for testing",
+	Long:  "This internal command runs a OpenID Connect for testing purposes",
+	Run: func(cmd *cobra.Command, args []string) {
+		err := mockOIDC()
+		if err != nil {
+			log.Error().Err(err).Msgf("Error running mock OIDC server")
+			os.Exit(1)
+		}
+	},
+}
+
+func mockOIDC() error {
+	clientID := os.Getenv("MOCKOIDC_CLIENT_ID")
+	if clientID == "" {
+		return errMockOidcClientIDNotDefined
+	}
+	clientSecret := os.Getenv("MOCKOIDC_CLIENT_SECRET")
+	if clientSecret == "" {
+		return errMockOidcClientSecretNotDefined
+	}
+	portStr := os.Getenv("MOCKOIDC_PORT")
+	if portStr == "" {
+		return errMockOidcPortNotDefined
+	}
+
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		return err
+	}
+
+	mock, err := getMockOIDC(clientID, clientSecret)
+	if err != nil {
+		return err
+	}
+
+	listener, err := net.Listen("tcp", fmt.Sprintf("mockoidc:%d", port))
+	if err != nil {
+		return err
+	}
+
+	err = mock.Start(listener, nil)
+	if err != nil {
+		return err
+	}
+	log.Info().Msgf("Mock OIDC server listening on %s", listener.Addr().String())
+	log.Info().Msgf("Issuer: %s", mock.Issuer())
+	c := make(chan struct{})
+	<-c
+
+	return nil
+}
+
+func getMockOIDC(clientID string, clientSecret string) (*mockoidc.MockOIDC, error) {
+	keypair, err := mockoidc.NewKeypair(nil)
+	if err != nil {
+		return nil, err
+	}
+
+	mock := mockoidc.MockOIDC{
+		ClientID:                      clientID,
+		ClientSecret:                  clientSecret,
+		AccessTTL:                     accessTTL,
+		RefreshTTL:                    refreshTTL,
+		CodeChallengeMethodsSupported: []string{"plain", "S256"},
+		Keypair:                       keypair,
+		SessionStore:                  mockoidc.NewSessionStore(),
+		UserQueue:                     &mockoidc.UserQueue{},
+		ErrorQueue:                    &mockoidc.ErrorQueue{},
+	}
+
+	return &mock, nil
+}

cmd/headscale/cli/root.go

@@ -15,6 +15,10 @@ import (
 var cfgFile string = ""
 
 func init() {
+	if len(os.Args) > 1 && os.Args[1] == "version" || os.Args[1] == "mockoidc" {
+		return
+	}
+
 	cobra.OnInitialize(initConfig)
 	rootCmd.PersistentFlags().
 		StringVarP(&cfgFile, "config", "c", "", "config file (default is /etc/headscale/config.yaml)")
@@ -47,7 +51,7 @@ func initConfig() {
 
 	machineOutput := HasMachineOutputFlag()
 
-	zerolog.SetGlobalLevel(cfg.LogLevel)
+	zerolog.SetGlobalLevel(cfg.Log.Level)
 
 	// If the user has requested a "machine" readable format,
 	// then disable login so the output remains valid.
@@ -55,6 +59,10 @@ func initConfig() {
 		zerolog.SetGlobalLevel(zerolog.Disabled)
 	}
 
+	if cfg.Log.Format == headscale.JSONLogFormat {
+		log.Logger = log.Output(os.Stdout)
+	}
+
 	if !cfg.DisableUpdateCheck && !machineOutput {
 		if (runtime.GOOS == "linux" || runtime.GOOS == "darwin") &&
 			Version != "dev" {

config-example.yaml

@@ -172,7 +172,10 @@ tls_letsencrypt_listen: ":http"
 tls_cert_path: ""
 tls_key_path: ""
 
-log_level: info
+log:
+  # Output formatting for logs: text or json
+  format: text
+  level: info
 
 # Path to a file containg ACL policies.
 # ACLs can be defined as YAML or HUJSON.

config.go

@@ -22,6 +22,9 @@ import (
 const (
 	tlsALPN01ChallengeType = "TLS-ALPN-01"
 	http01ChallengeType    = "HTTP-01"
+
+	JSONLogFormat = "json"
+	TextLogFormat = "text"
 )
 
 // Config contains the initial Headscale configuration.
@@ -37,7 +40,7 @@ type Config struct {
 	PrivateKeyPath                 string
 	NoisePrivateKeyPath            string
 	BaseDomain                     string
-	LogLevel                       zerolog.Level
+	Log                            LogConfig
 	DisableUpdateCheck             bool
 
 	DERP DERPConfig
@@ -124,6 +127,11 @@ type ACLConfig struct {
 	PolicyPath string
 }
 
+type LogConfig struct {
+	Format string
+	Level  zerolog.Level
+}
+
 func LoadConfig(path string, isFile bool) error {
 	if isFile {
 		viper.SetConfigFile(path)
@@ -147,7 +155,8 @@ func LoadConfig(path string, isFile bool) error {
 	viper.SetDefault("tls_letsencrypt_challenge_type", http01ChallengeType)
 	viper.SetDefault("tls_client_auth_mode", "relaxed")
 
-	viper.SetDefault("log_level", "info")
+	viper.SetDefault("log.level", "info")
+	viper.SetDefault("log.format", TextLogFormat)
 
 	viper.SetDefault("dns_config", nil)
 
@@ -334,6 +343,34 @@ func GetACLConfig() ACLConfig {
 	}
 }
 
+func GetLogConfig() LogConfig {
+	logLevelStr := viper.GetString("log.level")
+	logLevel, err := zerolog.ParseLevel(logLevelStr)
+	if err != nil {
+		logLevel = zerolog.DebugLevel
+	}
+
+	logFormatOpt := viper.GetString("log.format")
+	var logFormat string
+	switch logFormatOpt {
+	case "json":
+		logFormat = JSONLogFormat
+	case "text":
+		logFormat = TextLogFormat
+	case "":
+		logFormat = TextLogFormat
+	default:
+		log.Error().
+			Str("func", "GetLogConfig").
+			Msgf("Could not parse log format: %s. Valid choices are 'json' or 'text'", logFormatOpt)
+	}
+
+	return LogConfig{
+		Format: logFormat,
+		Level:  logLevel,
+	}
+}
+
 func GetDNSConfig() (*tailcfg.DNSConfig, string) {
 	if viper.IsSet("dns_config") {
 		dnsConfig := &tailcfg.DNSConfig{}
@@ -430,12 +467,6 @@ func GetHeadscaleConfig() (*Config, error) {
 	configuredPrefixes := viper.GetStringSlice("ip_prefixes")
 	parsedPrefixes := make([]netip.Prefix, 0, len(configuredPrefixes)+1)
 
-	logLevelStr := viper.GetString("log_level")
-	logLevel, err := zerolog.ParseLevel(logLevelStr)
-	if err != nil {
-		logLevel = zerolog.DebugLevel
-	}
-
 	legacyPrefixField := viper.GetString("ip_prefix")
 	if len(legacyPrefixField) > 0 {
 		log.
@@ -488,7 +519,6 @@ func GetHeadscaleConfig() (*Config, error) {
 		GRPCAddr:           viper.GetString("grpc_listen_addr"),
 		GRPCAllowInsecure:  viper.GetBool("grpc_allow_insecure"),
 		DisableUpdateCheck: viper.GetBool("disable_check_updates"),
-		LogLevel:           logLevel,
 
 		IPPrefixes: prefixes,
 		PrivateKeyPath: AbsolutePathFromConfigPath(
@@ -550,5 +580,7 @@ func GetHeadscaleConfig() (*Config, error) {
 		},
 
 		ACL: GetACLConfig(),
+
+		Log: GetLogConfig(),
 	}, nil
 }

docs/README.md

@@ -28,6 +28,7 @@ written by community members. It is _not_ verified by `headscale` developers.
 
 - [Running headscale in a container](running-headscale-container.md)
 - [Running headscale on OpenBSD](running-headscale-openbsd.md)
+- [Running headscale behind a reverse proxy](reverse-proxy.md)
 
 ## Misc
 

docs/reverse-proxy.md

@@ -0,0 +1,61 @@
+# Running headscale behind a reverse proxy
+
+Running headscale behind a reverse proxy is useful when running multiple applications on the same server, and you want to reuse the same external IP and port - usually tcp/443 for HTTPS.
+
+### WebSockets
+
+The reverse proxy MUST be configured to support WebSockets, as it is needed for clients running Tailscale v1.30+.
+
+WebSockets support is required when using the headscale embedded DERP server. In this case, you will also need to expose the UDP port used for STUN (by default, udp/3478). Please check our [config-example.yaml](https://github.com/juanfont/headscale/blob/main/config-example.yaml).
+
+### TLS
+
+Headscale can be configured not to use TLS, leaving it to the reverse proxy to handle. Add the following configuration values to your headscale config file.
+
+```yaml
+server_url: https://<YOUR_SERVER_NAME> # This should be the FQDN at which headscale will be served
+listen_addr: 0.0.0.0:8080
+metrics_listen_addr: 0.0.0.0:9090
+tls_cert_path: ""
+tls_key_path: ""
+```
+
+## nginx
+
+The following example configuration can be used in your nginx setup, substituting values as necessary. `<IP:PORT>` should be the IP address and port where headscale is running. In most cases, this will be `http://localhost:8080`.
+
+```Nginx
+map $http_upgrade $connection_upgrade {
+    default      keep-alive;
+    'websocket'  upgrade;
+    ''           close;
+}
+
+server {
+    listen 80;
+	listen [::]:80;
+
+	listen 443      ssl http2;
+	listen [::]:443 ssl http2;
+
+    server_name <YOUR_SERVER_NAME>;
+
+    ssl_certificate <PATH_TO_CERT>;
+    ssl_certificate_key <PATH_CERT_KEY>;
+    ssl_protocols TLSv1.2 TLSv1.3;
+
+    location / {
+        proxy_pass http://<IP:PORT>;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection $connection_upgrade;
+        proxy_set_header Host $server_name;
+        proxy_redirect http:// https://;
+        proxy_buffering off;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+        add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+    }
+}
+```

flake.nix

@@ -24,7 +24,7 @@
 
               # When updating go.mod or go.sum, a new sha will need to be calculated,
               # update this if you have a mismatch after doing a change to thos files.
-              vendorSha256 = "sha256-kc8EU+TkwRlsKM2+ljm/88aWe5h2QMgd/ZGPSgdd9QQ=";
+              vendorSha256 = "sha256-DosFCSiQ5FURbIrt4NcPGkExc84t2MGMqe9XLxNHdIM=";
 
               ldflags = [ "-s" "-w" "-X github.com/juanfont/headscale/cmd/headscale/cli.Version=v${version}" ];
             };

go.sum

@@ -273,8 +273,6 @@ github.com/fzipp/gocyclo v0.3.1/go.mod h1:DJHO6AUmbdqj2ET4Z9iArSuwWgYDRryYt2wASx
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
-github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
-github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
 github.com/glebarez/go-sqlite v1.17.3 h1:Rji9ROVSTTfjuWD6j5B+8DtkNvPILoUC3xRhkQzGxvk=
 github.com/glebarez/go-sqlite v1.17.3/go.mod h1:Hg+PQuhUy98XCxWEJEaWob8x7lhJzhNYF1nZbUiRGIY=
 github.com/glebarez/go-sqlite v1.18.1 h1:w0xtxKWktqYsUsXg//SQK+l1IcpKb3rGOQHmMptvL2U=

integration_cli_test.go

@@ -129,7 +129,7 @@ func (s *IntegrationCLITestSuite) HandleStats(
 }
 
 func (s *IntegrationCLITestSuite) createNamespace(name string) (*v1.Namespace, error) {
-	result, err := ExecuteCommand(
+	result, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -172,7 +172,7 @@ func (s *IntegrationCLITestSuite) TestNamespaceCommand() {
 	assert.Equal(s.T(), names[2], namespaces[2].Name)
 
 	// Test list namespaces
-	listResult, err := ExecuteCommand(
+	listResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -194,7 +194,7 @@ func (s *IntegrationCLITestSuite) TestNamespaceCommand() {
 	assert.Equal(s.T(), names[2], listedNamespaces[2].Name)
 
 	// Test rename namespace
-	renameResult, err := ExecuteCommand(
+	renameResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -216,7 +216,7 @@ func (s *IntegrationCLITestSuite) TestNamespaceCommand() {
 	assert.Equal(s.T(), renamedNamespace.Name, "newname")
 
 	// Test list after rename namespaces
-	listAfterRenameResult, err := ExecuteCommand(
+	listAfterRenameResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -247,7 +247,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommand() {
 	assert.Nil(s.T(), err)
 
 	for i := 0; i < count; i++ {
-		preAuthResult, err := ExecuteCommand(
+		preAuthResult, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{
 				"headscale",
@@ -275,7 +275,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommand() {
 	assert.Len(s.T(), keys, 5)
 
 	// Test list of keys
-	listResult, err := ExecuteCommand(
+	listResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -335,7 +335,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommand() {
 
 	// Expire three keys
 	for i := 0; i < 3; i++ {
-		_, err := ExecuteCommand(
+		_, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{
 				"headscale",
@@ -351,7 +351,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommand() {
 	}
 
 	// Test list pre auth keys after expire
-	listAfterExpireResult, err := ExecuteCommand(
+	listAfterExpireResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -396,7 +396,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommandWithoutExpiry() {
 	namespace, err := s.createNamespace("pre-auth-key-without-exp-namespace")
 	assert.Nil(s.T(), err)
 
-	preAuthResult, err := ExecuteCommand(
+	preAuthResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -417,7 +417,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommandWithoutExpiry() {
 	assert.Nil(s.T(), err)
 
 	// Test list of keys
-	listResult, err := ExecuteCommand(
+	listResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -449,7 +449,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommandReusableEphemeral() {
 	namespace, err := s.createNamespace("pre-auth-key-reus-ephm-namespace")
 	assert.Nil(s.T(), err)
 
-	preAuthReusableResult, err := ExecuteCommand(
+	preAuthReusableResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -472,7 +472,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommandReusableEphemeral() {
 	assert.True(s.T(), preAuthReusableKey.GetReusable())
 	assert.False(s.T(), preAuthReusableKey.GetEphemeral())
 
-	preAuthEphemeralResult, err := ExecuteCommand(
+	preAuthEphemeralResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -514,7 +514,7 @@ func (s *IntegrationCLITestSuite) TestPreAuthKeyCommandReusableEphemeral() {
 	// assert.NotNil(s.T(), err)
 
 	// Test list of keys
-	listResult, err := ExecuteCommand(
+	listResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -548,7 +548,7 @@ func (s *IntegrationCLITestSuite) TestNodeTagCommand() {
 	assert.Nil(s.T(), err)
 
 	for index, machineKey := range machineKeys {
-		_, err := ExecuteCommand(
+		_, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{
 				"headscale",
@@ -567,7 +567,7 @@ func (s *IntegrationCLITestSuite) TestNodeTagCommand() {
 		)
 		assert.Nil(s.T(), err)
 
-		machineResult, err := ExecuteCommand(
+		machineResult, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{
 				"headscale",
@@ -592,7 +592,7 @@ func (s *IntegrationCLITestSuite) TestNodeTagCommand() {
 	}
 	assert.Len(s.T(), machines, len(machineKeys))
 
-	addTagResult, err := ExecuteCommand(
+	addTagResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -612,7 +612,7 @@ func (s *IntegrationCLITestSuite) TestNodeTagCommand() {
 	assert.Equal(s.T(), []string{"tag:test"}, machine.ForcedTags)
 
 	// try to set a wrong tag and retrieve the error
-	wrongTagResult, err := ExecuteCommand(
+	wrongTagResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -634,7 +634,7 @@ func (s *IntegrationCLITestSuite) TestNodeTagCommand() {
 	assert.Contains(s.T(), errorOutput.Error, "tag must start with the string 'tag:'")
 
 	// Test list all nodes after added seconds
-	listAllResult, err := ExecuteCommand(
+	listAllResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -684,7 +684,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() {
 	assert.Nil(s.T(), err)
 
 	for index, machineKey := range machineKeys {
-		_, err := ExecuteCommand(
+		_, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{
 				"headscale",
@@ -703,7 +703,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() {
 		)
 		assert.Nil(s.T(), err)
 
-		machineResult, err := ExecuteCommand(
+		machineResult, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{
 				"headscale",
@@ -730,7 +730,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() {
 	assert.Len(s.T(), machines, len(machineKeys))
 
 	// Test list all nodes after added seconds
-	listAllResult, err := ExecuteCommand(
+	listAllResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -769,7 +769,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() {
 	assert.Nil(s.T(), err)
 
 	for index, machineKey := range otherNamespaceMachineKeys {
-		_, err := ExecuteCommand(
+		_, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{
 				"headscale",
@@ -788,7 +788,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() {
 		)
 		assert.Nil(s.T(), err)
 
-		machineResult, err := ExecuteCommand(
+		machineResult, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{
 				"headscale",
@@ -815,7 +815,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() {
 	assert.Len(s.T(), otherNamespaceMachines, len(otherNamespaceMachineKeys))
 
 	// Test list all nodes after added otherNamespace
-	listAllWithotherNamespaceResult, err := ExecuteCommand(
+	listAllWithotherNamespaceResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -845,7 +845,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() {
 	assert.Equal(s.T(), "otherNamespace-machine-2", listAllWithotherNamespace[6].Name)
 
 	// Test list all nodes after added otherNamespace
-	listOnlyotherNamespaceMachineNamespaceResult, err := ExecuteCommand(
+	listOnlyotherNamespaceMachineNamespaceResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -884,7 +884,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() {
 	)
 
 	// Delete a machines
-	_, err = ExecuteCommand(
+	_, _, err = ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -902,7 +902,7 @@ func (s *IntegrationCLITestSuite) TestNodeCommand() {
 	assert.Nil(s.T(), err)
 
 	// Test: list main namespace after machine is deleted
-	listOnlyMachineNamespaceAfterDeleteResult, err := ExecuteCommand(
+	listOnlyMachineNamespaceAfterDeleteResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -943,7 +943,7 @@ func (s *IntegrationCLITestSuite) TestNodeExpireCommand() {
 	assert.Nil(s.T(), err)
 
 	for index, machineKey := range machineKeys {
-		_, err := ExecuteCommand(
+		_, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{
 				"headscale",
@@ -962,7 +962,7 @@ func (s *IntegrationCLITestSuite) TestNodeExpireCommand() {
 		)
 		assert.Nil(s.T(), err)
 
-		machineResult, err := ExecuteCommand(
+		machineResult, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{
 				"headscale",
@@ -988,7 +988,7 @@ func (s *IntegrationCLITestSuite) TestNodeExpireCommand() {
 
 	assert.Len(s.T(), machines, len(machineKeys))
 
-	listAllResult, err := ExecuteCommand(
+	listAllResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1014,7 +1014,7 @@ func (s *IntegrationCLITestSuite) TestNodeExpireCommand() {
 	assert.True(s.T(), listAll[4].Expiry.AsTime().IsZero())
 
 	for i := 0; i < 3; i++ {
-		_, err := ExecuteCommand(
+		_, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{
 				"headscale",
@@ -1028,7 +1028,7 @@ func (s *IntegrationCLITestSuite) TestNodeExpireCommand() {
 		assert.Nil(s.T(), err)
 	}
 
-	listAllAfterExpiryResult, err := ExecuteCommand(
+	listAllAfterExpiryResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1070,7 +1070,7 @@ func (s *IntegrationCLITestSuite) TestNodeRenameCommand() {
 	assert.Nil(s.T(), err)
 
 	for index, machineKey := range machineKeys {
-		_, err := ExecuteCommand(
+		_, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{
 				"headscale",
@@ -1089,7 +1089,7 @@ func (s *IntegrationCLITestSuite) TestNodeRenameCommand() {
 		)
 		assert.Nil(s.T(), err)
 
-		machineResult, err := ExecuteCommand(
+		machineResult, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{
 				"headscale",
@@ -1115,7 +1115,7 @@ func (s *IntegrationCLITestSuite) TestNodeRenameCommand() {
 
 	assert.Len(s.T(), machines, len(machineKeys))
 
-	listAllResult, err := ExecuteCommand(
+	listAllResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1141,7 +1141,7 @@ func (s *IntegrationCLITestSuite) TestNodeRenameCommand() {
 	assert.Contains(s.T(), listAll[4].GetGivenName(), "machine-5")
 
 	for i := 0; i < 3; i++ {
-		_, err := ExecuteCommand(
+		_, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{
 				"headscale",
@@ -1156,7 +1156,7 @@ func (s *IntegrationCLITestSuite) TestNodeRenameCommand() {
 		assert.Nil(s.T(), err)
 	}
 
-	listAllAfterRenameResult, err := ExecuteCommand(
+	listAllAfterRenameResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1182,7 +1182,7 @@ func (s *IntegrationCLITestSuite) TestNodeRenameCommand() {
 	assert.Contains(s.T(), listAllAfterRename[4].GetGivenName(), "machine-5")
 
 	// Test failure for too long names
-	result, err := ExecuteCommand(
+	result, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1197,7 +1197,7 @@ func (s *IntegrationCLITestSuite) TestNodeRenameCommand() {
 	assert.Nil(s.T(), err)
 	assert.Contains(s.T(), result, "not be over 63 chars")
 
-	listAllAfterRenameAttemptResult, err := ExecuteCommand(
+	listAllAfterRenameAttemptResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1233,7 +1233,7 @@ func (s *IntegrationCLITestSuite) TestRouteCommand() {
 	// Randomly generated machine keys
 	machineKey := "9b2ffa7e08cc421a3d2cca9012280f6a236fd0de0b4ce005b30a98ad930306fe"
 
-	_, err = ExecuteCommand(
+	_, _, err = ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1256,7 +1256,7 @@ func (s *IntegrationCLITestSuite) TestRouteCommand() {
 	)
 	assert.Nil(s.T(), err)
 
-	machineResult, err := ExecuteCommand(
+	machineResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1280,7 +1280,7 @@ func (s *IntegrationCLITestSuite) TestRouteCommand() {
 	assert.Equal(s.T(), uint64(1), machine.Id)
 	assert.Equal(s.T(), "route-machine", machine.Name)
 
-	listAllResult, err := ExecuteCommand(
+	listAllResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1305,7 +1305,7 @@ func (s *IntegrationCLITestSuite) TestRouteCommand() {
 
 	assert.Empty(s.T(), listAll.EnabledRoutes)
 
-	enableTwoRoutesResult, err := ExecuteCommand(
+	enableTwoRoutesResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1337,7 +1337,7 @@ func (s *IntegrationCLITestSuite) TestRouteCommand() {
 	assert.Contains(s.T(), enableTwoRoutes.EnabledRoutes, "192.168.1.0/24")
 
 	// Enable only one route, effectively disabling one of the routes
-	enableOneRouteResult, err := ExecuteCommand(
+	enableOneRouteResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1366,7 +1366,7 @@ func (s *IntegrationCLITestSuite) TestRouteCommand() {
 	assert.Contains(s.T(), enableOneRoute.EnabledRoutes, "10.0.0.0/8")
 
 	// Enable only one route, effectively disabling one of the routes
-	failEnableNonAdvertisedRoute, err := ExecuteCommand(
+	failEnableNonAdvertisedRoute, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1390,7 +1390,7 @@ func (s *IntegrationCLITestSuite) TestRouteCommand() {
 	)
 
 	// Enable all routes on host
-	enableAllRouteResult, err := ExecuteCommand(
+	enableAllRouteResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1425,7 +1425,7 @@ func (s *IntegrationCLITestSuite) TestApiKeyCommand() {
 	keys := make([]string, count)
 
 	for i := 0; i < count; i++ {
-		apiResult, err := ExecuteCommand(
+		apiResult, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{
 				"headscale",
@@ -1451,7 +1451,7 @@ func (s *IntegrationCLITestSuite) TestApiKeyCommand() {
 	assert.Len(s.T(), keys, 5)
 
 	// Test list of keys
-	listResult, err := ExecuteCommand(
+	listResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1513,7 +1513,7 @@ func (s *IntegrationCLITestSuite) TestApiKeyCommand() {
 
 	// Expire three keys
 	for i := 0; i < 3; i++ {
-		_, err := ExecuteCommand(
+		_, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{
 				"headscale",
@@ -1530,7 +1530,7 @@ func (s *IntegrationCLITestSuite) TestApiKeyCommand() {
 	}
 
 	// Test list pre auth keys after expire
-	listAfterExpireResult, err := ExecuteCommand(
+	listAfterExpireResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1573,7 +1573,7 @@ func (s *IntegrationCLITestSuite) TestNodeMoveCommand() {
 	// Randomly generated machine key
 	machineKey := "688411b767663479632d44140f08a9fde87383adc7cdeb518f62ce28a17ef0aa"
 
-	_, err = ExecuteCommand(
+	_, _, err = ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1592,7 +1592,7 @@ func (s *IntegrationCLITestSuite) TestNodeMoveCommand() {
 	)
 	assert.Nil(s.T(), err)
 
-	machineResult, err := ExecuteCommand(
+	machineResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1619,7 +1619,7 @@ func (s *IntegrationCLITestSuite) TestNodeMoveCommand() {
 
 	machineId := fmt.Sprintf("%d", machine.Id)
 
-	moveToNewNSResult, err := ExecuteCommand(
+	moveToNewNSResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1641,7 +1641,7 @@ func (s *IntegrationCLITestSuite) TestNodeMoveCommand() {
 
 	assert.Equal(s.T(), machine.Namespace, newNamespace)
 
-	listAllNodesResult, err := ExecuteCommand(
+	listAllNodesResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1664,7 +1664,7 @@ func (s *IntegrationCLITestSuite) TestNodeMoveCommand() {
 	assert.Equal(s.T(), allNodes[0].Namespace, machine.Namespace)
 	assert.Equal(s.T(), allNodes[0].Namespace, newNamespace)
 
-	moveToNonExistingNSResult, err := ExecuteCommand(
+	moveToNonExistingNSResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1688,7 +1688,7 @@ func (s *IntegrationCLITestSuite) TestNodeMoveCommand() {
 	)
 	assert.Equal(s.T(), machine.Namespace, newNamespace)
 
-	moveToOldNSResult, err := ExecuteCommand(
+	moveToOldNSResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1710,7 +1710,7 @@ func (s *IntegrationCLITestSuite) TestNodeMoveCommand() {
 
 	assert.Equal(s.T(), machine.Namespace, oldNamespace)
 
-	moveToSameNSResult, err := ExecuteCommand(
+	moveToSameNSResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1742,7 +1742,7 @@ func (s *IntegrationCLITestSuite) TestLoadConfigFromCommand() {
 	altEnvConfig, err := os.ReadFile("integration_test/etc/alt-env-config.dump.gold.yaml")
 	assert.Nil(s.T(), err)
 
-	_, err = ExecuteCommand(
+	_, _, err = ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1757,7 +1757,7 @@ func (s *IntegrationCLITestSuite) TestLoadConfigFromCommand() {
 
 	assert.YAMLEq(s.T(), string(defaultConfig), string(defaultDumpConfig))
 
-	_, err = ExecuteCommand(
+	_, _, err = ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1774,7 +1774,7 @@ func (s *IntegrationCLITestSuite) TestLoadConfigFromCommand() {
 
 	assert.YAMLEq(s.T(), string(altConfig), string(altDumpConfig))
 
-	_, err = ExecuteCommand(
+	_, _, err = ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -1791,7 +1791,7 @@ func (s *IntegrationCLITestSuite) TestLoadConfigFromCommand() {
 
 	assert.YAMLEq(s.T(), string(altEnvConfig), string(altEnvDumpConfig))
 
-	_, err = ExecuteCommand(
+	_, _, err = ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",

integration_common_test.go

@@ -68,7 +68,7 @@ func ExecuteCommand(
 	cmd []string,
 	env []string,
 	options ...ExecuteCommandOption,
-) (string, error) {
+) (string, string, error) {
 	var stdout bytes.Buffer
 	var stderr bytes.Buffer
 
@@ -78,7 +78,7 @@ func ExecuteCommand(
 
 	for _, opt := range options {
 		if err := opt(&execConfig); err != nil {
-			return "", fmt.Errorf("execute-command/options: %w", err)
+			return "", "", fmt.Errorf("execute-command/options: %w", err)
 		}
 	}
 
@@ -107,7 +107,7 @@ func ExecuteCommand(
 	select {
 	case res := <-resultChan:
 		if res.err != nil {
-			return "", res.err
+			return stdout.String(), stderr.String(), res.err
 		}
 
 		if res.exitCode != 0 {
@@ -115,13 +115,13 @@ func ExecuteCommand(
 			fmt.Println("stdout: ", stdout.String())
 			fmt.Println("stderr: ", stderr.String())
 
-			return "", fmt.Errorf("command failed with: %s", stderr.String())
+			return stdout.String(), stderr.String(), fmt.Errorf("command failed with: %s", stderr.String())
 		}
 
-		return stdout.String(), nil
+		return stdout.String(), stderr.String(), nil
 	case <-time.After(execConfig.timeout):
 
-		return "", fmt.Errorf("command timed out after %s", execConfig.timeout)
+		return stdout.String(), stderr.String(), fmt.Errorf("command timed out after %s", execConfig.timeout)
 	}
 }
 
@@ -200,7 +200,7 @@ func getIPs(
 	for hostname, tailscale := range tailscales {
 		command := []string{"tailscale", "ip"}
 
-		result, err := ExecuteCommand(
+		result, _, err := ExecuteCommand(
 			&tailscale,
 			command,
 			[]string{},
@@ -228,7 +228,7 @@ func getIPs(
 func getDNSNames(
 	headscale *dockertest.Resource,
 ) ([]string, error) {
-	listAllResult, err := ExecuteCommand(
+	listAllResult, _, err := ExecuteCommand(
 		headscale,
 		[]string{
 			"headscale",
@@ -261,7 +261,7 @@ func getDNSNames(
 func getMagicFQDN(
 	headscale *dockertest.Resource,
 ) ([]string, error) {
-	listAllResult, err := ExecuteCommand(
+	listAllResult, _, err := ExecuteCommand(
 		headscale,
 		[]string{
 			"headscale",

integration_embedded_derp_test.go

@@ -187,7 +187,7 @@ func (s *IntegrationDERPTestSuite) SetupSuite() {
 	log.Println("headscale container is ready for embedded DERP tests")
 
 	log.Printf("Creating headscale namespace: %s\n", namespaceName)
-	result, err := ExecuteCommand(
+	result, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{"headscale", "namespaces", "create", namespaceName},
 		[]string{},
@@ -196,7 +196,7 @@ func (s *IntegrationDERPTestSuite) SetupSuite() {
 	assert.Nil(s.T(), err)
 
 	log.Printf("Creating pre auth key for %s\n", namespaceName)
-	preAuthResult, err := ExecuteCommand(
+	preAuthResult, _, err := ExecuteCommand(
 		&s.headscale,
 		[]string{
 			"headscale",
@@ -259,7 +259,7 @@ func (s *IntegrationDERPTestSuite) Join(
 
 	log.Println("Join command:", command)
 	log.Printf("Running join command for %s\n", hostname)
-	_, err := ExecuteCommand(
+	_, _, err := ExecuteCommand(
 		&tailscale,
 		command,
 		[]string{},
@@ -414,7 +414,7 @@ func (s *IntegrationDERPTestSuite) TestPingAllPeersByHostname() {
 					peername,
 				)
 				log.Println(command)
-				result, err := ExecuteCommand(
+				result, _, err := ExecuteCommand(
 					&tailscale,
 					command,
 					[]string{},

integration_general_test.go

@@ -163,7 +163,7 @@ func (s *IntegrationTestSuite) Join(
 
 	log.Println("Join command:", command)
 	log.Printf("Running join command for %s\n", hostname)
-	_, err := ExecuteCommand(
+	_, _, err := ExecuteCommand(
 		&tailscale,
 		command,
 		[]string{},
@@ -305,7 +305,7 @@ func (s *IntegrationTestSuite) SetupSuite() {
 
 	for namespace, scales := range s.namespaces {
 		log.Printf("Creating headscale namespace: %s\n", namespace)
-		result, err := ExecuteCommand(
+		result, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{"headscale", "namespaces", "create", namespace},
 			[]string{},
@@ -314,7 +314,7 @@ func (s *IntegrationTestSuite) SetupSuite() {
 		assert.Nil(s.T(), err)
 
 		log.Printf("Creating pre auth key for %s\n", namespace)
-		preAuthResult, err := ExecuteCommand(
+		preAuthResult, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{
 				"headscale",
@@ -386,7 +386,7 @@ func (s *IntegrationTestSuite) HandleStats(
 func (s *IntegrationTestSuite) TestListNodes() {
 	for namespace, scales := range s.namespaces {
 		log.Println("Listing nodes")
-		result, err := ExecuteCommand(
+		result, _, err := ExecuteCommand(
 			&s.headscale,
 			[]string{"headscale", "--namespace", namespace, "nodes", "list"},
 			[]string{},
@@ -518,7 +518,7 @@ func (s *IntegrationTestSuite) TestPingAllPeersByAddress() {
 								peername,
 								ip,
 							)
-							result, err := ExecuteCommand(
+							result, _, err := ExecuteCommand(
 								&tailscale,
 								command,
 								[]string{},
@@ -552,7 +552,7 @@ func (s *IntegrationTestSuite) TestTailDrop() {
 
 		for hostname, tailscale := range scales.tailscales {
 			command := []string{"touch", fmt.Sprintf("/tmp/file_from_%s", hostname)}
-			_, err := ExecuteCommand(
+			_, _, err := ExecuteCommand(
 				&tailscale,
 				command,
 				[]string{},
@@ -586,7 +586,7 @@ func (s *IntegrationTestSuite) TestTailDrop() {
 							hostname,
 							peername,
 						)
-						_, err := ExecuteCommand(
+						_, _, err := ExecuteCommand(
 							&tailscale,
 							command,
 							[]string{},
@@ -606,7 +606,7 @@ func (s *IntegrationTestSuite) TestTailDrop() {
 				"get",
 				"/tmp/",
 			}
-			_, err := ExecuteCommand(
+			_, _, err := ExecuteCommand(
 				&tailscale,
 				command,
 				[]string{},
@@ -628,7 +628,7 @@ func (s *IntegrationTestSuite) TestTailDrop() {
 						peername,
 						ip,
 					)
-					result, err := ExecuteCommand(
+					result, _, err := ExecuteCommand(
 						&tailscale,
 						command,
 						[]string{},
@@ -672,7 +672,7 @@ func (s *IntegrationTestSuite) TestPingAllPeersByHostname() {
 						hostname,
 						peername,
 					)
-					result, err := ExecuteCommand(
+					result, _, err := ExecuteCommand(
 						&tailscale,
 						command,
 						[]string{},
@@ -724,7 +724,7 @@ func (s *IntegrationTestSuite) TestMagicDNS() {
 							peername,
 							hostname,
 						)
-						result, err := ExecuteCommand(
+						result, _, err := ExecuteCommand(
 							&tailscale,
 							command,
 							[]string{},
@@ -757,7 +757,7 @@ func getAPIURLs(
 			"/run/tailscale/tailscaled.sock",
 			"http://localhost/localapi/v0/file-targets",
 		}
-		result, err := ExecuteCommand(
+		result, _, err := ExecuteCommand(
 			&tailscale,
 			command,
 			[]string{},

integration_oidc_test.go

@@ -0,0 +1,506 @@
+//go:build integration_oidc
+
+package headscale
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+)
+
+const (
+	oidcHeadscaleHostname = "headscale"
+	oidcNamespaceName     = "oidcnamespace"
+	totalOidcContainers   = 3
+)
+
+type IntegrationOIDCTestSuite struct {
+	suite.Suite
+	stats *suite.SuiteInformation
+
+	pool      dockertest.Pool
+	network   dockertest.Network
+	headscale dockertest.Resource
+	mockOidc  dockertest.Resource
+	saveLogs  bool
+
+	tailscales    map[string]dockertest.Resource
+	joinWaitGroup sync.WaitGroup
+}
+
+func TestOIDCIntegrationTestSuite(t *testing.T) {
+	saveLogs, err := GetEnvBool("HEADSCALE_INTEGRATION_SAVE_LOG")
+	if err != nil {
+		saveLogs = false
+	}
+
+	s := new(IntegrationOIDCTestSuite)
+
+	s.tailscales = make(map[string]dockertest.Resource)
+	s.saveLogs = saveLogs
+
+	suite.Run(t, s)
+
+	// HandleStats, which allows us to check if we passed and save logs
+	// is called after TearDown, so we cannot tear down containers before
+	// we have potentially saved the logs.
+	if s.saveLogs {
+		for _, tailscale := range s.tailscales {
+			if err := s.pool.Purge(&tailscale); err != nil {
+				log.Printf("Could not purge resource: %s\n", err)
+			}
+		}
+
+		if !s.stats.Passed() {
+			err := s.saveLog(&s.headscale, "test_output")
+			if err != nil {
+				log.Printf("Could not save log: %s\n", err)
+			}
+		}
+
+		if err := s.pool.Purge(&s.mockOidc); err != nil {
+			log.Printf("Could not purge resource: %s\n", err)
+		}
+
+		if err := s.pool.Purge(&s.headscale); err != nil {
+			t.Logf("Could not purge resource: %s\n", err)
+		}
+
+		if err := s.network.Close(); err != nil {
+			log.Printf("Could not close network: %s\n", err)
+		}
+	}
+}
+
+func (s *IntegrationOIDCTestSuite) SetupSuite() {
+	if ppool, err := dockertest.NewPool(""); err == nil {
+		s.pool = *ppool
+	} else {
+		s.FailNow(fmt.Sprintf("Could not connect to docker: %s", err), "")
+	}
+
+	if pnetwork, err := s.pool.CreateNetwork("headscale-test"); err == nil {
+		s.network = *pnetwork
+	} else {
+		s.FailNow(fmt.Sprintf("Could not create network: %s", err), "")
+	}
+
+	// Create does not give us an updated version of the resource, so we need to
+	// get it again.
+	networks, err := s.pool.NetworksByName("headscale-test")
+	if err != nil {
+		s.FailNow(fmt.Sprintf("Could not get network: %s", err), "")
+	}
+	s.network = networks[0]
+
+	log.Printf("Network config: %v", s.network.Network.IPAM.Config[0])
+
+	s.Suite.T().Log("Setting up mock OIDC")
+	mockOidcOptions := &dockertest.RunOptions{
+		Name:         "mockoidc",
+		Hostname:     "mockoidc",
+		Cmd:          []string{"headscale", "mockoidc"},
+		ExposedPorts: []string{"10000/tcp"},
+		Networks:     []*dockertest.Network{&s.network},
+		PortBindings: map[docker.Port][]docker.PortBinding{
+			"10000/tcp": {{HostPort: "10000"}},
+		},
+		Env: []string{
+			"MOCKOIDC_PORT=10000",
+			"MOCKOIDC_CLIENT_ID=superclient",
+			"MOCKOIDC_CLIENT_SECRET=supersecret",
+		},
+	}
+
+	headscaleBuildOptions := &dockertest.BuildOptions{
+		Dockerfile: "Dockerfile.debug",
+		ContextDir: ".",
+	}
+
+	if pmockoidc, err := s.pool.BuildAndRunWithBuildOptions(
+		headscaleBuildOptions,
+		mockOidcOptions,
+		DockerRestartPolicy); err == nil {
+		s.mockOidc = *pmockoidc
+	} else {
+		s.FailNow(fmt.Sprintf("Could not start mockOIDC container: %s", err), "")
+	}
+
+	oidcCfg := fmt.Sprintf(`
+oidc:
+  issuer: http://%s:10000/oidc
+  client_id: superclient
+  client_secret: supersecret
+  strip_email_domain: true`, s.mockOidc.GetIPInNetwork(&s.network))
+
+	currentPath, err := os.Getwd()
+	if err != nil {
+		s.FailNow(fmt.Sprintf("Could not determine current path: %s", err), "")
+	}
+
+	baseConfig, err := os.ReadFile(
+		path.Join(currentPath, "integration_test/etc_oidc/base_config.yaml"))
+	if err != nil {
+		s.FailNow(fmt.Sprintf("Could not read base config: %s", err), "")
+	}
+	config := string(baseConfig) + oidcCfg
+
+	log.Println(config)
+
+	configPath := path.Join(currentPath, "integration_test/etc_oidc/config.yaml")
+	err = os.WriteFile(configPath, []byte(config), 0644)
+	if err != nil {
+		s.FailNow(fmt.Sprintf("Could not write config: %s", err), "")
+	}
+
+	headscaleOptions := &dockertest.RunOptions{
+		Name:     oidcHeadscaleHostname,
+		Networks: []*dockertest.Network{&s.network},
+		Mounts: []string{
+			path.Join(currentPath,
+				"integration_test/etc_oidc:/etc/headscale",
+			),
+		},
+		Cmd:          []string{"headscale", "serve"},
+		ExposedPorts: []string{"8443/tcp", "3478/udp"},
+		PortBindings: map[docker.Port][]docker.PortBinding{
+			"8443/tcp": {{HostPort: "8443"}},
+			"3478/udp": {{HostPort: "3478"}},
+		},
+	}
+
+	err = s.pool.RemoveContainerByName(oidcHeadscaleHostname)
+	if err != nil {
+		s.FailNow(
+			fmt.Sprintf(
+				"Could not remove existing container before building test: %s",
+				err,
+			),
+			"",
+		)
+	}
+
+	s.Suite.T().Logf("Creating headscale container for OIDC integration tests")
+	if pheadscale, err := s.pool.BuildAndRunWithBuildOptions(headscaleBuildOptions, headscaleOptions, DockerRestartPolicy); err == nil {
+		s.headscale = *pheadscale
+	} else {
+		s.FailNow(fmt.Sprintf("Could not start headscale container: %s", err), "")
+	}
+	s.Suite.T().Logf("Created headscale container for embedded OIDC tests")
+
+	s.Suite.T().Logf("Creating tailscale containers for embedded OIDC tests")
+
+	for i := 0; i < totalOidcContainers; i++ {
+		version := tailscaleVersions[i%len(tailscaleVersions)]
+		hostname, container := s.tailscaleContainer(
+			fmt.Sprint(i),
+			version,
+		)
+		s.tailscales[hostname] = *container
+	}
+
+	s.Suite.T().Logf("Waiting for headscale to be ready for embedded OIDC tests")
+	hostEndpoint := fmt.Sprintf("localhost:%s", s.headscale.GetPort("8443/tcp"))
+
+	if err := s.pool.Retry(func() error {
+		url := fmt.Sprintf("https://%s/health", hostEndpoint)
+		insecureTransport := http.DefaultTransport.(*http.Transport).Clone()
+		insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+		client := &http.Client{Transport: insecureTransport}
+		resp, err := client.Get(url)
+		if err != nil {
+			log.Printf("headscale for embedded OIDC tests is not ready: %s\n", err)
+			return err
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			return fmt.Errorf("status code not OK")
+		}
+
+		return nil
+	}); err != nil {
+		// TODO(kradalby): If we cannot access headscale, or any other fatal error during
+		// test setup, we need to abort and tear down. However, testify does not seem to
+		// support that at the moment:
+		// https://github.com/stretchr/testify/issues/849
+		return // fmt.Errorf("Could not connect to headscale: %s", err)
+	}
+	s.Suite.T().Log("headscale container is ready for embedded OIDC tests")
+
+	s.Suite.T().Logf("Creating headscale namespace: %s\n", oidcNamespaceName)
+	result, _, err := ExecuteCommand(
+		&s.headscale,
+		[]string{"headscale", "namespaces", "create", oidcNamespaceName},
+		[]string{},
+	)
+	log.Println("headscale create namespace result: ", result)
+	assert.Nil(s.T(), err)
+
+	headscaleEndpoint := fmt.Sprintf(
+		"https://headscale:%s",
+		s.headscale.GetPort("8443/tcp"),
+	)
+
+	log.Printf(
+		"Joining tailscale containers to headscale at %s\n",
+		headscaleEndpoint,
+	)
+	for hostname, tailscale := range s.tailscales {
+		s.joinWaitGroup.Add(1)
+		go s.AuthenticateOIDC(headscaleEndpoint, hostname, tailscale)
+
+		// TODO(juan): Workaround for https://github.com/juanfont/headscale/issues/814
+		time.Sleep(1 * time.Second)
+	}
+
+	s.joinWaitGroup.Wait()
+
+	// The nodes need a bit of time to get their updated maps from headscale
+	// TODO: See if we can have a more deterministic wait here.
+	time.Sleep(60 * time.Second)
+}
+
+func (s *IntegrationOIDCTestSuite) AuthenticateOIDC(
+	endpoint, hostname string,
+	tailscale dockertest.Resource,
+) {
+	defer s.joinWaitGroup.Done()
+
+	loginURL, err := s.joinOIDC(endpoint, hostname, tailscale)
+	if err != nil {
+		s.FailNow(fmt.Sprintf("Could not join OIDC node: %s", err), "")
+	}
+
+	insecureTransport := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{Transport: insecureTransport}
+	resp, err := client.Get(loginURL.String())
+	assert.Nil(s.T(), err)
+
+	body, err := io.ReadAll(resp.Body)
+	assert.Nil(s.T(), err)
+
+	if err != nil {
+		s.FailNow(fmt.Sprintf("Could not read login page: %s", err), "")
+	}
+
+	log.Printf("Login page for %s: %s", hostname, string(body))
+}
+
+func (s *IntegrationOIDCTestSuite) joinOIDC(
+	endpoint, hostname string,
+	tailscale dockertest.Resource,
+) (*url.URL, error) {
+
+	command := []string{
+		"tailscale",
+		"up",
+		"-login-server",
+		endpoint,
+		"--hostname",
+		hostname,
+	}
+
+	log.Println("Join command:", command)
+	log.Printf("Running join command for %s\n", hostname)
+	_, stderr, _ := ExecuteCommand(
+		&tailscale,
+		command,
+		[]string{},
+	)
+
+	// This piece of code just gets the login URL out of the stderr of the tailscale client.
+	// See https://github.com/tailscale/tailscale/blob/main/cmd/tailscale/cli/up.go#L584.
+	urlStr := strings.ReplaceAll(stderr, "\nTo authenticate, visit:\n\n\t", "")
+	urlStr = strings.TrimSpace(urlStr)
+
+	// parse URL
+	loginUrl, err := url.Parse(urlStr)
+	if err != nil {
+		log.Printf("Could not parse login URL: %s", err)
+		log.Printf("Original join command result: %s", stderr)
+		return nil, err
+	}
+
+	return loginUrl, nil
+}
+
+func (s *IntegrationOIDCTestSuite) tailscaleContainer(
+	identifier, version string,
+) (string, *dockertest.Resource) {
+	tailscaleBuildOptions := getDockerBuildOptions(version)
+
+	hostname := fmt.Sprintf(
+		"tailscale-%s-%s",
+		strings.Replace(version, ".", "-", -1),
+		identifier,
+	)
+	tailscaleOptions := &dockertest.RunOptions{
+		Name:     hostname,
+		Networks: []*dockertest.Network{&s.network},
+		Cmd: []string{
+			"tailscaled", "--tun=tsdev",
+		},
+
+		// expose the host IP address, so we can access it from inside the container
+		ExtraHosts: []string{
+			"host.docker.internal:host-gateway",
+			"headscale:host-gateway",
+		},
+	}
+
+	pts, err := s.pool.BuildAndRunWithBuildOptions(
+		tailscaleBuildOptions,
+		tailscaleOptions,
+		DockerRestartPolicy,
+		DockerAllowLocalIPv6,
+		DockerAllowNetworkAdministration,
+	)
+	if err != nil {
+		log.Fatalf("Could not start tailscale container version %s: %s", version, err)
+	}
+	log.Printf("Created %s container\n", hostname)
+
+	return hostname, pts
+}
+
+func (s *IntegrationOIDCTestSuite) TearDownSuite() {
+	if !s.saveLogs {
+		for _, tailscale := range s.tailscales {
+			if err := s.pool.Purge(&tailscale); err != nil {
+				log.Printf("Could not purge resource: %s\n", err)
+			}
+		}
+
+		if err := s.pool.Purge(&s.headscale); err != nil {
+			log.Printf("Could not purge resource: %s\n", err)
+		}
+
+		if err := s.pool.Purge(&s.mockOidc); err != nil {
+			log.Printf("Could not purge resource: %s\n", err)
+		}
+
+		if err := s.network.Close(); err != nil {
+			log.Printf("Could not close network: %s\n", err)
+		}
+	}
+}
+
+func (s *IntegrationOIDCTestSuite) HandleStats(
+	suiteName string,
+	stats *suite.SuiteInformation,
+) {
+	s.stats = stats
+}
+
+func (s *IntegrationOIDCTestSuite) saveLog(
+	resource *dockertest.Resource,
+	basePath string,
+) error {
+	err := os.MkdirAll(basePath, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	err = s.pool.Client.Logs(
+		docker.LogsOptions{
+			Context:      context.TODO(),
+			Container:    resource.Container.ID,
+			OutputStream: &stdout,
+			ErrorStream:  &stderr,
+			Tail:         "all",
+			RawTerminal:  false,
+			Stdout:       true,
+			Stderr:       true,
+			Follow:       false,
+			Timestamps:   false,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	log.Printf("Saving logs for %s to %s\n", resource.Container.Name, basePath)
+
+	err = os.WriteFile(
+		path.Join(basePath, resource.Container.Name+".stdout.log"),
+		[]byte(stdout.String()),
+		0o644,
+	)
+	if err != nil {
+		return err
+	}
+
+	err = os.WriteFile(
+		path.Join(basePath, resource.Container.Name+".stderr.log"),
+		[]byte(stdout.String()),
+		0o644,
+	)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *IntegrationOIDCTestSuite) TestPingAllPeersByAddress() {
+	for hostname, tailscale := range s.tailscales {
+		ips, err := getIPs(s.tailscales)
+		assert.Nil(s.T(), err)
+		for peername, peerIPs := range ips {
+			for i, ip := range peerIPs {
+				// We currently cant ping ourselves, so skip that.
+				if peername == hostname {
+					continue
+				}
+				s.T().
+					Run(fmt.Sprintf("%s-%s-%d", hostname, peername, i), func(t *testing.T) {
+						// We are only interested in "direct ping" which means what we
+						// might need a couple of more attempts before reaching the node.
+						command := []string{
+							"tailscale", "ping",
+							"--timeout=1s",
+							"--c=10",
+							"--until-direct=true",
+							ip.String(),
+						}
+
+						log.Printf(
+							"Pinging from %s to %s (%s)\n",
+							hostname,
+							peername,
+							ip,
+						)
+						stdout, stderr, err := ExecuteCommand(
+							&tailscale,
+							command,
+							[]string{},
+						)
+						assert.Nil(t, err)
+						log.Printf("result for %s: stdout: %s, stderr: %s\n", hostname, stdout, stderr)
+						assert.Contains(t, stdout, "pong")
+					})
+			}
+		}
+	}
+}

integration_test/etc/alt-config.dump.gold.yaml

@@ -28,7 +28,9 @@ ip_prefixes:
   - fd7a:115c:a1e0::/48
   - 100.64.0.0/10
 listen_addr: 0.0.0.0:18080
-log_level: disabled
+log:
+  level: disabled
+  format: text
 logtail:
   enabled: false
 metrics_listen_addr: 127.0.0.1:19090

integration_test/etc/alt-config.yaml

@@ -1,4 +1,5 @@
-log_level: trace
+log:
+  level: trace
 acl_policy_path: ""
 db_type: sqlite3
 ephemeral_node_inactivity_timeout: 30m

integration_test/etc/alt-env-config.dump.gold.yaml

@@ -27,7 +27,9 @@ ip_prefixes:
   - fd7a:115c:a1e0::/48
   - 100.64.0.0/10
 listen_addr: 0.0.0.0:18080
-log_level: disabled
+log:
+  level: disabled
+  format: text
 logtail:
   enabled: false
 metrics_listen_addr: 127.0.0.1:19090

integration_test/etc/alt-env-config.yaml

@@ -1,4 +1,5 @@
-log_level: trace
+log:
+  level: trace
 acl_policy_path: ""
 db_type: sqlite3
 ephemeral_node_inactivity_timeout: 30m

integration_test/etc/config.dump.gold.yaml

@@ -28,7 +28,9 @@ ip_prefixes:
   - fd7a:115c:a1e0::/48
   - 100.64.0.0/10
 listen_addr: 0.0.0.0:8080
-log_level: disabled
+log:
+  format: text
+  level: disabled
 logtail:
   enabled: false
 metrics_listen_addr: 127.0.0.1:9090

integration_test/etc/config.yaml

@@ -1,4 +1,5 @@
-log_level: trace
+log:
+  level: trace
 acl_policy_path: ""
 db_type: sqlite3
 ephemeral_node_inactivity_timeout: 30m

integration_test/etc_oidc/base_config.yaml

@@ -0,0 +1,22 @@
+log_level: trace
+acl_policy_path: ""
+db_type: sqlite3
+ephemeral_node_inactivity_timeout: 30m
+node_update_check_interval: 10s
+ip_prefixes:
+  - fd7a:115c:a1e0::/48
+  - 100.64.0.0/10
+db_path: /tmp/integration_test_db.sqlite3
+private_key_path: private.key
+noise:
+  private_key_path: noise_private.key
+listen_addr: 0.0.0.0:8443
+server_url: https://localhost:8443
+tls_cert_path: "/etc/headscale/tls/server.crt"
+tls_key_path: "/etc/headscale/tls/server.key"
+tls_client_auth_mode: disabled
+derp:
+  urls:
+    - https://controlplane.tailscale.com/derpmap/default
+  auto_update_enabled: true
+  update_frequency: 1m

integration_test/etc_oidc/tls/server.crt

@@ -0,0 +1,22 @@
+
+-----BEGIN CERTIFICATE-----
+MIIC8jCCAdqgAwIBAgIULbu+UbSTMG/LtxooLLh7BgSEyqEwDQYJKoZIhvcNAQEL
+BQAwFDESMBAGA1UEAwwJaGVhZHNjYWxlMCAXDTIyMDMwNTE2NDgwM1oYDzI1MjEx
+MTA0MTY0ODAzWjAUMRIwEAYDVQQDDAloZWFkc2NhbGUwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQDqcfpToLZUF0rlNwXkkt3lbyw4Cl4TJdx36o2PKaOK
+U+tze/IjRsCWeMwrcR1o9TNZcxsD+c2J48D1WATuQJlMeg+2UJXGaTGRKkkbPMy3
+5m7AFf/Q16UEOgm2NYjZaQ8faRGIMYURG/6sXmNeETJvBixpBev9yKJuVXgqHNS4
+NpEkNwdOCuAZXrmw0HCbiusawJOay4tFvhH14rav8Uimonl8UTNVXufMzyUOuoaQ
+TGflmzYX3hIoswRnTPlIWFoqObvx2Q8H+of3uQJXy0m8I6OrIoXLNxnqYMfFls79
+9SYgVc2jPsCbh5fwyRbx2Hof7sIZ1K/mNgxJRG1E3ZiLAgMBAAGjOjA4MBQGA1Ud
+EQQNMAuCCWhlYWRzY2FsZTALBgNVHQ8EBAMCB4AwEwYDVR0lBAwwCgYIKwYBBQUH
+AwEwDQYJKoZIhvcNAQELBQADggEBANGlVN7NCsJaKz0k0nhlRGK+tcxn2p1PXN/i
+Iy+JX8ahixPC4ocRwOhrXgb390ZXLLwq08HrWYRB/Wi1VUzCp5d8dVxvrR43dJ+v
+L2EOBiIKgcu2C3pWW1qRR46/EoXUU9kSH2VNBvIhNufi32kEOidoDzxtQf6qVCoF
+guUt1JkAqrynv1UvR/2ZRM/WzM/oJ8qfECwrwDxyYhkqU5Z5jCWg0C6kPIBvNdzt
+B0eheWS+ZxVwkePTR4e17kIafwknth3lo+orxVrq/xC+OVM1bGrt2ZyD64ZvEqQl
+w6kgbzBdLScAQptWOFThwhnJsg0UbYKimZsnYmjVEuN59TJv92M=
+-----END CERTIFICATE-----
+
+(Expires on Nov  4 16:48:03 2521 GMT)
+

integration_test/etc_oidc/tls/server.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDqcfpToLZUF0rl
+NwXkkt3lbyw4Cl4TJdx36o2PKaOKU+tze/IjRsCWeMwrcR1o9TNZcxsD+c2J48D1
+WATuQJlMeg+2UJXGaTGRKkkbPMy35m7AFf/Q16UEOgm2NYjZaQ8faRGIMYURG/6s
+XmNeETJvBixpBev9yKJuVXgqHNS4NpEkNwdOCuAZXrmw0HCbiusawJOay4tFvhH1
+4rav8Uimonl8UTNVXufMzyUOuoaQTGflmzYX3hIoswRnTPlIWFoqObvx2Q8H+of3
+uQJXy0m8I6OrIoXLNxnqYMfFls799SYgVc2jPsCbh5fwyRbx2Hof7sIZ1K/mNgxJ
+RG1E3ZiLAgMBAAECggEBALu1Ni/u5Qy++YA8ZcN0s6UXNdhItLmv/q0kZuLQ+9et
+CT8VZfFInLndTdsaXenDKLHdryunviFA8SV+q7P2lMbek+Xs735EiyMnMBFWxLIZ
+FWNGOeQERGL19QCmLEOmEi2b+iWJQHlKaMWpbPXL3w11a+lKjIBNO4ALfoJ5QveZ
+cGMKsJdm/mpqBvLeNeh2eAFk3Gp6sT1g80Ge8NkgyzFBNIqnut0eerM15kPTc6Qz
+12JLaOXUuV3PrcB4PN4nOwrTDg88GDNOQtc1Pc9r4nOHyLfr8X7QEtj1wXSwmOuK
+d6ynMnAmoxVA9wEnupLbil1bzohRzpsTpkmDruYaBEECgYEA/Z09I8D6mt2NVqIE
+KyvLjBK39ijSV9r3/lvB2Ple2OOL5YQEd+yTrIFy+3zdUnDgD1zmNnXjmjvHZ9Lc
+IFf2o06AF84QLNB5gLPdDQkGNFdDqUxljBrfAfE3oANmPS/B0SijMGOOOiDO2FtO
+xl1nfRr78mswuRs9awoUWCdNRKUCgYEA7KaTYKIQW/FEjw9lshp74q5vbn6zoXF5
+7N8VkwI+bBVNvRbM9XZ8qhfgRdu9eXs5oL/N4mSYY54I8fA//pJ0Z2vpmureMm1V
+mL5WBUmSD9DIbAchoK+sRiQhVmNMBQC6cHMABA7RfXvBeGvWrm9pKCS6ZLgLjkjp
+PsmAcaXQcW8CgYEA2inAxljjOwUK6FNGsrxhxIT1qtNC3kCGxE+6WSNq67gSR8Vg
+8qiX//T7LEslOB3RIGYRwxd2St7RkgZZRZllmOWWWuPwFhzf6E7RAL2akLvggGov
+kG4tGEagSw2hjVDfsUT73ExHtMk0Jfmlsg33UC8+PDLpHtLH6qQpDAwC8+ECgYEA
+o+AqOIWhvHmT11l7O915Ip1WzvZwYADbxLsrDnVEUsZh4epTHjvh0kvcY6PqTqCV
+ZIrOANNWb811Nkz/k8NJVoD08PFp0xPBbZeIq/qpachTsfMyRzq/mobUiyUR9Hjv
+ooUQYr78NOApNsG+lWbTNBhS9wI4BlzZIECbcJe5g4MCgYEAndRoy8S+S0Hx/S8a
+O3hzXeDmivmgWqn8NVD4AKOovpkz4PaIVVQbAQkiNfAx8/DavPvjEKAbDezJ4ECV
+j7IsOWtDVI7pd6eF9fTcECwisrda8aUoiOap8AQb48153Vx+g2N4Vy3uH0xJs4cz
+TDALZPOBg8VlV+HEFDP43sp9Bf0=
+-----END PRIVATE KEY-----

machine.go

@@ -26,7 +26,9 @@ const (
 	)
 	ErrCouldNotConvertMachineInterface = Error("failed to convert machine interface")
 	ErrHostnameTooLong                 = Error("Hostname too long")
-	ErrDifferentRegisteredNamespace    = Error("machine was previously registered with a different namespace")
+	ErrDifferentRegisteredNamespace    = Error(
+		"machine was previously registered with a different namespace",
+	)
 	MachineGivenNameHashLength = 8
 	MachineGivenNameTrimSize   = 2
 )
@@ -35,6 +37,11 @@ const (
 	maxHostnameLength = 255
 )
 
+var (
+	ExitRouteV4 = netip.MustParsePrefix("0.0.0.0/0")
+	ExitRouteV6 = netip.MustParsePrefix("::/0")
+)
+
 // Machine is a Headscale client.
 type Machine struct {
 	ID          uint64 `gorm:"primary_key"`
@@ -566,12 +573,11 @@ func (machines MachinesP) String() string {
 func (machines Machines) toNodes(
 	baseDomain string,
 	dnsConfig *tailcfg.DNSConfig,
-	includeRoutes bool,
 ) ([]*tailcfg.Node, error) {
 	nodes := make([]*tailcfg.Node, len(machines))
 
 	for index, machine := range machines {
-		node, err := machine.toNode(baseDomain, dnsConfig, includeRoutes)
+		node, err := machine.toNode(baseDomain, dnsConfig)
 		if err != nil {
 			return nil, err
 		}
@@ -587,7 +593,6 @@ func (machines Machines) toNodes(
 func (machine Machine) toNode(
 	baseDomain string,
 	dnsConfig *tailcfg.DNSConfig,
-	includeRoutes bool,
 ) (*tailcfg.Node, error) {
 	var nodeKey key.NodePublic
 	err := nodeKey.UnmarshalText([]byte(NodePublicKeyEnsurePrefix(machine.NodeKey)))
@@ -633,10 +638,22 @@ func (machine Machine) toNode(
 		[]netip.Prefix{},
 		addrs...) // we append the node own IP, as it is required by the clients
 
-	// TODO(kradalby): Needs investigation, We probably dont need this condition
-	// now that we dont have shared nodes
-	if includeRoutes {
 	allowedIPs = append(allowedIPs, machine.EnabledRoutes...)
+
+	// TODO(kradalby): This is kind of a hack where we say that
+	// all the announced routes (except exit), is presented as primary
+	// routes. This might be problematic if two nodes expose the same route.
+	// This was added to address an issue where subnet routers stopped working
+	// when we only populated AllowedIPs.
+	primaryRoutes := []netip.Prefix{}
+	if len(machine.EnabledRoutes) > 0 {
+		for _, route := range machine.EnabledRoutes {
+			if route == ExitRouteV4 || route == ExitRouteV6 {
+				continue
+			}
+
+			primaryRoutes = append(primaryRoutes, route)
+		}
 	}
 
 	var derp string
@@ -691,6 +708,7 @@ func (machine Machine) toNode(
 		DiscoKey:      discoKey,
 		Addresses:     addrs,
 		AllowedIPs:    allowedIPs,
+		PrimaryRoutes: primaryRoutes,
 		Endpoints:     machine.Endpoints,
 		DERP:          derp,
 
@@ -807,7 +825,8 @@ func (h *Headscale) RegisterMachineFromAuthCallback(
 			}
 
 			// Registration of expired machine with different namespace
-			if registrationMachine.ID != 0 && registrationMachine.NamespaceID != namespace.ID {
+			if registrationMachine.ID != 0 &&
+				registrationMachine.NamespaceID != namespace.ID {
 				return nil, ErrDifferentRegisteredNamespace
 			}