diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..90c48e92 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,36 @@ +name: Build + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: "1.16.3" + + - name: Install dependencies + run: | + go version + go install golang.org/x/lint/golint@latest + sudo apt update + sudo apt install -y make + + - name: Run lint + run: make build + + - uses: actions/upload-artifact@v2 + with: + name: headscale-linux + path: headscale \ No newline at end of file diff --git a/.goreleaser.yml b/.goreleaser.yml index fd25563c..7b1ea605 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -19,7 +19,7 @@ builds: flags: - -mod=readonly ldflags: - - -s -w -X main.version={{.Version}} + - -s -w -X github.com/juanfont/headscale/cmd/headscale/cli.Version=v{{.Version}} - id: linux-armhf main: ./cmd/headscale/headscale.go mod_timestamp: '{{ .CommitTimestamp }}' @@ -39,7 +39,7 @@ builds: flags: - -mod=readonly ldflags: - - -s -w -X main.version={{.Version}} + - -s -w -X github.com/juanfont/headscale/cmd/headscale/cli.Version=v{{.Version}} - id: linux-amd64 @@ -54,6 +54,8 @@ builds: - 7 main: ./cmd/headscale/headscale.go mod_timestamp: '{{ .CommitTimestamp }}' + ldflags: + - -s -w -X github.com/juanfont/headscale/cmd/headscale/cli.Version=v{{.Version}} archives: - id: golang-cross diff --git a/README.md b/README.md index 35f4c184..5f691a6c 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,17 @@ Headscale implements this coordination server. - [x] Share nodes between ~~users~~ namespaces - [ ] MagicDNS / Smart DNS +## Client OS support + +| OS | Supports headscale | +| --- | --- | +| Linux | Yes | +| OpenBSD | Yes | +| macOS | Yes (see `/apple` on your headscale for more information) | +| Windows | Yes | +| Android | [You need to compile the client yourself](https://github.com/juanfont/headscale/issues/58#issuecomment-885255270) | +| iOS | Not yet | + ## Roadmap 🤷 Suggestions/PRs welcomed! @@ -80,7 +91,7 @@ Suggestions/PRs welcomed! ```shell touch db.sqlite - docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml -v $(pwd)/db.sqlite:/db.sqlite -p 127.0.0.1:8000:8000 headscale/headscale:x.x.x headscale namespaces create myfirstnamespace + docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml -v $(pwd)/db.sqlite:/db.sqlite -p 127.0.0.1:8080:8080 headscale/headscale:x.x.x headscale namespaces create myfirstnamespace ``` or if your server is already running in docker: @@ -100,7 +111,7 @@ Suggestions/PRs welcomed! the db.sqlite mount is only needed if you use sqlite ```shell - docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml -v $(pwd)/db.sqlite:/db.sqlite -p 127.0.0.1:8000:8000 headscale/headscale:x.x.x headscale serve + docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml -v $(pwd)/db.sqlite:/db.sqlite -p 127.0.0.1:8080:8080 headscale/headscale:x.x.x headscale serve ``` 6. If you used tailscale.com before in your nodes, make sure you clear the tailscald data folder @@ -114,22 +125,22 @@ Suggestions/PRs welcomed! 7. Add your first machine ```shell - tailscale up -login-server YOUR_HEADSCALE_URL + tailscale up --login-server YOUR_HEADSCALE_URL ``` 8. Navigate to the URL you will get with `tailscale up`, where you'll find your machine key. 9. In the server, register your machine to a namespace with the CLI ```shell - headscale -n myfirstnamespace node register YOURMACHINEKEY + headscale -n myfirstnamespace nodes register YOURMACHINEKEY ``` or docker: ```shell - docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml headscale/headscale:x.x.x headscale -n myfirstnamespace node register YOURMACHINEKEY + docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml headscale/headscale:x.x.x headscale -n myfirstnamespace nodes register YOURMACHINEKEY ``` or if your server is already running in docker: ```shell - docker exec headscale -n myfistnamespace node register YOURMACHINEKEY + docker exec headscale -n myfirstnamespace nodes register YOURMACHINEKEY ``` Alternatively, you can use Auth Keys to register your machines: @@ -154,7 +165,7 @@ Alternatively, you can use Auth Keys to register your machines: 2. Use the authkey from your machine to register it ```shell - tailscale up -login-server YOUR_HEADSCALE_URL --authkey YOURAUTHKEY + tailscale up --login-server YOUR_HEADSCALE_URL --authkey YOURAUTHKEY ``` If you create an authkey with the `--ephemeral` flag, that key will create ephemeral nodes. This implies that `--reusable` is true. diff --git a/api.go b/api.go index 02f28919..a70df5b3 100644 --- a/api.go +++ b/api.go @@ -65,6 +65,7 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) { Str("handler", "Registration"). Err(err). Msg("Cannot parse machine key") + machineRegistrations.WithLabelValues("unkown", "web", "error", "unknown").Inc() c.String(http.StatusInternalServerError, "Sad!") return } @@ -75,6 +76,7 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) { Str("handler", "Registration"). Err(err). Msg("Cannot decode message") + machineRegistrations.WithLabelValues("unkown", "web", "error", "unknown").Inc() c.String(http.StatusInternalServerError, "Very sad!") return } @@ -95,6 +97,7 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) { Str("handler", "Registration"). Err(err). Msg("Could not create row") + machineRegistrations.WithLabelValues("unkown", "web", "error", m.Namespace.Name).Inc() return } } @@ -149,9 +152,11 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) { Str("handler", "Registration"). Err(err). Msg("Cannot encode message") + machineRegistrations.WithLabelValues("update", "web", "error", m.Namespace.Name).Inc() c.String(http.StatusInternalServerError, "") return } + machineRegistrations.WithLabelValues("update", "web", "success", m.Namespace.Name).Inc() c.Data(200, "application/json; charset=utf-8", respBody) return } @@ -178,9 +183,11 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) { Str("handler", "Registration"). Err(err). Msg("Cannot encode message") + machineRegistrations.WithLabelValues("new", "web", "error", m.Namespace.Name).Inc() c.String(http.StatusInternalServerError, "") return } + machineRegistrations.WithLabelValues("new", "web", "success", m.Namespace.Name).Inc() c.Data(200, "application/json; charset=utf-8", respBody) return } @@ -264,7 +271,7 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) { c.Data(200, "application/json; charset=utf-8", respBody) } -func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Machine) (*[]byte, error) { +func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Machine) ([]byte, error) { log.Trace(). Str("func", "getMapResponse"). Str("machine", req.Hostinfo.Hostname). @@ -277,6 +284,7 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Mac Msg("Cannot convert to node") return nil, err } + peers, err := h.getPeers(m) if err != nil { log.Error(). @@ -292,11 +300,20 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Mac DisplayName: m.Namespace.Name, } + nodePeers, err := peers.toNodes(true) + if err != nil { + log.Error(). + Str("func", "getMapResponse"). + Err(err). + Msg("Failed to convert peers to Tailscale nodes") + return nil, err + } + resp := tailcfg.MapResponse{ KeepAlive: false, Node: node, - Peers: *peers, - //TODO(kradalby): As per tailscale docs, if DNSConfig is nil, + Peers: nodePeers, + // TODO(kradalby): As per tailscale docs, if DNSConfig is nil, // it means its not updated, maybe we can have some logic // to check and only pass updates when its updates. // This is probably more relevant if we try to implement @@ -311,6 +328,7 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Mac log.Trace(). Str("func", "getMapResponse"). Str("machine", req.Hostinfo.Hostname). + // Interface("payload", resp). Msgf("Generated map response: %s", tailMapResponseToString(resp)) var respBody []byte @@ -333,10 +351,10 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Mac data := make([]byte, 4) binary.LittleEndian.PutUint32(data, uint32(len(respBody))) data = append(data, respBody...) - return &data, nil + return data, nil } -func (h *Headscale) getMapKeepAliveResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Machine) (*[]byte, error) { +func (h *Headscale) getMapKeepAliveResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Machine) ([]byte, error) { resp := tailcfg.MapResponse{ KeepAlive: true, } @@ -359,7 +377,7 @@ func (h *Headscale) getMapKeepAliveResponse(mKey wgkey.Key, req tailcfg.MapReque data := make([]byte, 4) binary.LittleEndian.PutUint32(data, uint32(len(respBody))) data = append(data, respBody...) - return &data, nil + return data, nil } func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key, req tailcfg.RegisterRequest, m Machine) { @@ -379,13 +397,15 @@ func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key, Err(err). Msg("Cannot encode message") c.String(http.StatusInternalServerError, "") + machineRegistrations.WithLabelValues("new", "authkey", "error", m.Namespace.Name).Inc() return } - c.Data(200, "application/json; charset=utf-8", respBody) + c.Data(401, "application/json; charset=utf-8", respBody) log.Error(). Str("func", "handleAuthKey"). Str("machine", m.Name). Msg("Failed authentication via AuthKey") + machineRegistrations.WithLabelValues("new", "authkey", "error", m.Namespace.Name).Inc() return } @@ -399,6 +419,7 @@ func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key, Str("func", "handleAuthKey"). Str("machine", m.Name). Msg("Failed to find an available IP") + machineRegistrations.WithLabelValues("new", "authkey", "error", m.Namespace.Name).Inc() return } log.Info(). @@ -424,9 +445,11 @@ func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key, Str("machine", m.Name). Err(err). Msg("Cannot encode message") + machineRegistrations.WithLabelValues("new", "authkey", "error", m.Namespace.Name).Inc() c.String(http.StatusInternalServerError, "Extremely sad!") return } + machineRegistrations.WithLabelValues("new", "authkey", "success", m.Namespace.Name).Inc() c.Data(200, "application/json; charset=utf-8", respBody) log.Info(). Str("func", "handleAuthKey"). diff --git a/app.go b/app.go index 2ad72154..9e688fe1 100644 --- a/app.go +++ b/app.go @@ -8,6 +8,7 @@ import ( "golang.org/x/oauth2" "net/http" "os" + "sort" "strings" "sync" "time" @@ -15,6 +16,8 @@ import ( "github.com/rs/zerolog/log" "github.com/gin-gonic/gin" + "github.com/zsais/go-gin-prometheus" + "golang.org/x/crypto/acme" "golang.org/x/crypto/acme/autocert" "gorm.io/gorm" "inet.af/netaddr" @@ -47,6 +50,9 @@ type Config struct { TLSCertPath string TLSKeyPath string + ACMEURL string + ACMEEmail string + DNSConfig *tailcfg.DNSConfig OIDCIssuer string @@ -70,9 +76,6 @@ type Headscale struct { aclPolicy *ACLPolicy aclRules *[]tailcfg.FilterRule - clientsUpdateChannels sync.Map - clientsUpdateChannelMutex sync.Mutex - lastStateChange sync.Map oidcProvider *oidc.Provider @@ -161,9 +164,9 @@ func (h *Headscale) expireEphemeralNodesWorker() { if err != nil { log.Error().Err(err).Str("machine", m.Name).Msg("🤮 Cannot delete ephemeral machine from the database") } - h.notifyChangesToPeers(&m) } } + h.setLastStateChangeToNow(ns.Name) } } @@ -184,6 +187,10 @@ func (h *Headscale) watchForKVUpdatesWorker() { // Serve launches a GIN server with the Headscale API func (h *Headscale) Serve() error { r := gin.Default() + + p := ginprometheus.NewPrometheus("gin") + p.Use(r) + r.GET("/health", func(c *gin.Context) { c.JSON(200, gin.H{"healthy": "ok"}) }) r.GET("/key", h.KeyHandler) r.GET("/register", h.RegisterWebAPI) @@ -196,16 +203,18 @@ func (h *Headscale) Serve() error { var err error - timeout := 30 * time.Second - go h.watchForKVUpdates(5000) go h.expireEphemeralNodes(5000) s := &http.Server{ - Addr: h.cfg.Addr, - Handler: r, - ReadTimeout: timeout, - WriteTimeout: timeout, + Addr: h.cfg.Addr, + Handler: r, + ReadTimeout: 30 * time.Second, + // Go does not handle timeouts in HTTP very well, and there is + // no good way to handle streaming timeouts, therefore we need to + // keep this at unlimited and be careful to clean up connections + // https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/#aboutstreaming + WriteTimeout: 0, } if h.cfg.TLSLetsEncryptHostname != "" { @@ -217,14 +226,14 @@ func (h *Headscale) Serve() error { Prompt: autocert.AcceptTOS, HostPolicy: autocert.HostWhitelist(h.cfg.TLSLetsEncryptHostname), Cache: autocert.DirCache(h.cfg.TLSLetsEncryptCacheDir), + Client: &acme.Client{ + DirectoryURL: h.cfg.ACMEURL, + }, + Email: h.cfg.ACMEEmail, } - s := &http.Server{ - Addr: h.cfg.Addr, - TLSConfig: m.TLSConfig(), - Handler: r, - ReadTimeout: timeout, - WriteTimeout: timeout, - } + + s.TLSConfig = m.TLSConfig() + if h.cfg.TLSLetsEncryptChallengeType == "TLS-ALPN-01" { // Configuration via autocert with TLS-ALPN-01 (https://tools.ietf.org/html/rfc8737) // The RFC requires that the validation is done on port 443; in other words, headscale @@ -235,7 +244,6 @@ func (h *Headscale) Serve() error { // port 80 for the certificate validation in addition to the headscale // service, which can be configured to run on any other port. go func() { - log.Fatal(). Err(http.ListenAndServe(h.cfg.TLSLetsEncryptListen, m.HTTPHandler(http.HandlerFunc(h.redirect)))). Msg("failed to set up a HTTP server") @@ -260,17 +268,32 @@ func (h *Headscale) Serve() error { func (h *Headscale) setLastStateChangeToNow(namespace string) { now := time.Now().UTC() + lastStateUpdate.WithLabelValues("", "headscale").Set(float64(now.Unix())) h.lastStateChange.Store(namespace, now) } -func (h *Headscale) getLastStateChange(namespace string) time.Time { - if wrapped, ok := h.lastStateChange.Load(namespace); ok { - lastChange, _ := wrapped.(time.Time) - return lastChange +func (h *Headscale) getLastStateChange(namespaces ...string) time.Time { + times := []time.Time{} + + for _, namespace := range namespaces { + if wrapped, ok := h.lastStateChange.Load(namespace); ok { + lastChange, _ := wrapped.(time.Time) + + times = append(times, lastChange) + } } - now := time.Now().UTC() - h.lastStateChange.Store(namespace, now) - return now + sort.Slice(times, func(i, j int) bool { + return times[i].After(times[j]) + }) + + log.Trace().Msgf("Latest times %#v", times) + + if len(times) == 0 { + return time.Now().UTC() + + } else { + return times[0] + } } diff --git a/cmd/headscale/cli/utils.go b/cmd/headscale/cli/utils.go index 67017aa0..17bc37e7 100644 --- a/cmd/headscale/cli/utils.go +++ b/cmd/headscale/cli/utils.go @@ -179,6 +179,9 @@ func getHeadscaleApp() (*headscale.Headscale, error) { TLSCertPath: absPath(viper.GetString("tls_cert_path")), TLSKeyPath: absPath(viper.GetString("tls_key_path")), + ACMEEmail: viper.GetString("acme_email"), + ACMEURL: viper.GetString("acme_url"), + DNSConfig: GetDNSConfig(), OIDCIssuer: viper.GetString("oidc_issuer"), diff --git a/config.json.postgres.example b/config.json.postgres.example index aba72063..e9118204 100644 --- a/config.json.postgres.example +++ b/config.json.postgres.example @@ -10,6 +10,8 @@ "db_name": "headscale", "db_user": "foo", "db_pass": "bar", + "acme_url": "https://acme-v02.api.letsencrypt.org/directory", + "acme_email": "", "tls_letsencrypt_hostname": "", "tls_letsencrypt_listen": ":http", "tls_letsencrypt_cache_dir": ".cache", diff --git a/config.json.sqlite.example b/config.json.sqlite.example index b22e5ace..5afa450f 100644 --- a/config.json.sqlite.example +++ b/config.json.sqlite.example @@ -6,6 +6,8 @@ "ephemeral_node_inactivity_timeout": "30m", "db_type": "sqlite3", "db_path": "db.sqlite", + "acme_url": "https://acme-v02.api.letsencrypt.org/directory", + "acme_email": "", "tls_letsencrypt_hostname": "", "tls_letsencrypt_listen": ":http", "tls_letsencrypt_cache_dir": ".cache", diff --git a/go.mod b/go.mod index a770338b..7e137e19 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/docker/cli v20.10.8+incompatible // indirect github.com/docker/docker v20.10.8+incompatible // indirect github.com/efekarakus/termcolor v1.0.1 + github.com/fatih/set v0.2.1 // indirect github.com/gin-gonic/gin v1.7.4 github.com/gofrs/uuid v4.0.0+incompatible github.com/google/go-github v17.0.0+incompatible // indirect @@ -22,6 +23,7 @@ require ( github.com/opencontainers/runc v1.0.2 // indirect github.com/ory/dockertest/v3 v3.7.0 github.com/patrickmn/go-cache v2.1.0+incompatible + github.com/prometheus/client_golang v1.11.0 // indirect github.com/pterm/pterm v0.12.30 github.com/rs/zerolog v1.25.0 github.com/s12v/go-jwks v0.2.1 @@ -31,6 +33,7 @@ require ( github.com/tailscale/hujson v0.0.0-20210818175511-7360507a6e88 github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect + github.com/zsais/go-gin-prometheus v0.1.0 // indirect golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 golang.org/x/net v0.0.0-20210913180222-943fd674d43e // indirect golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 diff --git a/go.sum b/go.sum index fc498e75..0c5befb9 100644 --- a/go.sum +++ b/go.sum @@ -103,6 +103,7 @@ github.com/aws/aws-sdk-go v1.38.52/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2z github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= @@ -118,7 +119,9 @@ github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -203,6 +206,8 @@ github.com/fanliao/go-promise v0.0.0-20141029170127-1890db352a72/go.mod h1:Pjfxu github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA= +github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= @@ -520,6 +525,7 @@ github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhB github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ= github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok= @@ -533,6 +539,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -540,6 +547,7 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= @@ -618,6 +626,7 @@ github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGw github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxzIU= github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mbilski/exhaustivestruct v1.1.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc= github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7/go.mod h1:U6ZQobyTjI/tJyq2HG+i/dfSoFUt8/aZCM+GKtmFk/Y= @@ -671,6 +680,7 @@ github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8q github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= @@ -751,21 +761,32 @@ github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/pterm/pterm v0.12.27/go.mod h1:PhQ89w4i95rhgE+xedAoqous6K9X+r6aSOI2eFF7DZI= github.com/pterm/pterm v0.12.29/go.mod h1:WI3qxgvoQFFGKGjGnJR849gU0TsEOvKn5Q8LlY1U7lg= @@ -937,6 +958,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/telnet v0.0.0-20180329124119-c3b780dc415b/go.mod h1:IZpXDfkJ6tWD3PhBK5YzgQT+xJWh7OsdwiG8hA2MkO4= +github.com/zsais/go-gin-prometheus v0.1.0 h1:bkLv1XCdzqVgQ36ScgRi09MA2UC1t3tAB6nsfErsGO4= +github.com/zsais/go-gin-prometheus v0.1.0/go.mod h1:Slirjzuz8uM8Cw0jmPNqbneoqcUtY2GGjn2bEd4NRLY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= @@ -1156,6 +1179,7 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1171,6 +1195,8 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1203,6 +1229,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1467,6 +1494,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/integration_test.go b/integration_test.go index f62ca1da..a7805646 100644 --- a/integration_test.go +++ b/integration_test.go @@ -7,6 +7,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io/ioutil" "log" @@ -233,9 +234,6 @@ func (s *IntegrationTestSuite) SetupSuite() { }, Networks: []*dockertest.Network{&network}, Cmd: []string{"headscale", "serve"}, - PortBindings: map[docker.Port][]docker.PortBinding{ - "8080/tcp": {{HostPort: "8080"}}, - }, } fmt.Println("Creating headscale container") @@ -270,7 +268,11 @@ func (s *IntegrationTestSuite) SetupSuite() { } return nil }); err != nil { - log.Fatalf("Could not connect to docker: %s", err) + // TODO(kradalby): If we cannot access headscale, or any other fatal error during + // test setup, we need to abort and tear down. However, testify does not seem to + // support that at the moment: + // https://github.com/stretchr/testify/issues/849 + return // fmt.Errorf("Could not connect to headscale: %s", err) } fmt.Println("headscale container is ready") @@ -292,7 +294,7 @@ func (s *IntegrationTestSuite) SetupSuite() { ) assert.Nil(s.T(), err) - headscaleEndpoint := fmt.Sprintf("http://headscale:%s", headscale.GetPort("8080/tcp")) + headscaleEndpoint := "http://headscale:8080" fmt.Printf("Joining tailscale containers to headscale at %s\n", headscaleEndpoint) for hostname, tailscale := range scales.tailscales { @@ -353,15 +355,16 @@ func (s *IntegrationTestSuite) TestGetIpAddresses() { for hostname := range scales.tailscales { s.T().Run(hostname, func(t *testing.T) { - ip := ips[hostname] + ip, ok := ips[hostname] + + assert.True(t, ok) + assert.NotNil(t, ip) fmt.Printf("IP for %s: %s\n", hostname, ip) // c.Assert(ip.Valid(), check.IsTrue) assert.True(t, ip.Is4()) assert.True(t, ipPrefix.Contains(ip)) - - ips[hostname] = ip }) } } @@ -433,7 +436,7 @@ func (s *IntegrationTestSuite) TestPingAllPeers() { command := []string{ "tailscale", "ping", "--timeout=1s", - "--c=20", + "--c=10", "--until-direct=true", ip.String(), } @@ -692,6 +695,9 @@ func getAPIURLs(tailscales map[string]dockertest.Resource) (map[netaddr.IP]strin n := ft.Node for _, a := range n.Addresses { // just add all the addresses if _, ok := fts[a.IP()]; !ok { + if ft.PeerAPIURL == "" { + return nil, errors.New("api url is empty") + } fts[a.IP()] = ft.PeerAPIURL } } diff --git a/integration_test/etc/config.json b/integration_test/etc/config.json index 8a6fd962..5454f2f7 100644 --- a/integration_test/etc/config.json +++ b/integration_test/etc/config.json @@ -7,5 +7,5 @@ "db_type": "sqlite3", "db_path": "/tmp/integration_test_db.sqlite3", "acl_policy_path": "", - "log_level": "debug" + "log_level": "trace" } diff --git a/machine.go b/machine.go index b5c821f5..bd5caf0d 100644 --- a/machine.go +++ b/machine.go @@ -2,12 +2,13 @@ package headscale import ( "encoding/json" - "errors" "fmt" "sort" "strconv" + "strings" "time" + "github.com/fatih/set" "github.com/rs/zerolog/log" "gorm.io/datatypes" @@ -45,6 +46,11 @@ type Machine struct { DeletedAt *time.Time } +type ( + Machines []Machine + MachinesP []*Machine +) + // For the time being this method is rather naive func (m Machine) isAlreadyRegistered() bool { return m.Registered @@ -53,6 +59,238 @@ func (m Machine) isAlreadyRegistered() bool { // isExpired returns whether the machine registration has expired func (m Machine) isExpired() bool { return time.Now().UTC().After(*m.Expiry) +} + +func (h *Headscale) getDirectPeers(m *Machine) (Machines, error) { + log.Trace(). + Str("func", "getDirectPeers"). + Str("machine", m.Name). + Msg("Finding direct peers") + + machines := Machines{} + if err := h.db.Where("namespace_id = ? AND machine_key <> ? AND registered", + m.NamespaceID, m.MachineKey).Find(&machines).Error; err != nil { + log.Error().Err(err).Msg("Error accessing db") + return Machines{}, err + } + + sort.Slice(machines, func(i, j int) bool { return machines[i].ID < machines[j].ID }) + + log.Trace(). + Str("func", "getDirectmachines"). + Str("machine", m.Name). + Msgf("Found direct machines: %s", machines.String()) + return machines, nil +} + +func (h *Headscale) getShared(m *Machine) (Machines, error) { + log.Trace(). + Str("func", "getShared"). + Str("machine", m.Name). + Msg("Finding shared peers") + + // We fetch here machines that are shared to the `Namespace` of the machine we are getting peers for + sharedMachines := []SharedMachine{} + if err := h.db.Preload("Namespace").Preload("Machine").Where("namespace_id = ?", + m.NamespaceID).Find(&sharedMachines).Error; err != nil { + return Machines{}, err + } + + peers := make(Machines, 0) + for _, sharedMachine := range sharedMachines { + peers = append(peers, sharedMachine.Machine) + } + + sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID }) + + log.Trace(). + Str("func", "getShared"). + Str("machine", m.Name). + Msgf("Found shared peers: %s", peers.String()) + return peers, nil +} + +func (h *Headscale) getPeers(m *Machine) (Machines, error) { + direct, err := h.getDirectPeers(m) + if err != nil { + log.Error(). + Str("func", "getPeers"). + Err(err). + Msg("Cannot fetch peers") + return Machines{}, err + } + + shared, err := h.getShared(m) + if err != nil { + log.Error(). + Str("func", "getDirectPeers"). + Err(err). + Msg("Cannot fetch peers") + return Machines{}, err + } + + peers := append(direct, shared...) + sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID }) + + log.Trace(). + Str("func", "getShared"). + Str("machine", m.Name). + Msgf("Found total peers: %s", peers.String()) + + return peers, nil +} + +// GetMachine finds a Machine by name and namespace and returns the Machine struct +func (h *Headscale) GetMachine(namespace string, name string) (*Machine, error) { + machines, err := h.ListMachinesInNamespace(namespace) + if err != nil { + return nil, err + } + + for _, m := range *machines { + if m.Name == name { + return &m, nil + } + } + return nil, fmt.Errorf("machine not found") +} + +// GetMachineByID finds a Machine by ID and returns the Machine struct +func (h *Headscale) GetMachineByID(id uint64) (*Machine, error) { + m := Machine{} + if result := h.db.Preload("Namespace").Find(&Machine{ID: id}).First(&m); result.Error != nil { + return nil, result.Error + } + return &m, nil +} + +// GetMachineByMachineKey finds a Machine by ID and returns the Machine struct +func (h *Headscale) GetMachineByMachineKey(mKey string) (*Machine, error) { + m := Machine{} + if result := h.db.Preload("Namespace").First(&m, "machine_key = ?", mKey); result.Error != nil { + return nil, result.Error + } + return &m, nil +} + +// UpdateMachine takes a Machine struct pointer (typically already loaded from database +// and updates it with the latest data from the database. +func (h *Headscale) UpdateMachine(m *Machine) error { + if result := h.db.Find(m).First(&m); result.Error != nil { + return result.Error + } + return nil +} + +// DeleteMachine softs deletes a Machine from the database +func (h *Headscale) DeleteMachine(m *Machine) error { + m.Registered = false + namespaceID := m.NamespaceID + h.db.Save(&m) // we mark it as unregistered, just in case + if err := h.db.Delete(&m).Error; err != nil { + return err + } + + return h.RequestMapUpdates(namespaceID) +} + +// HardDeleteMachine hard deletes a Machine from the database +func (h *Headscale) HardDeleteMachine(m *Machine) error { + namespaceID := m.NamespaceID + if err := h.db.Unscoped().Delete(&m).Error; err != nil { + return err + } + return h.RequestMapUpdates(namespaceID) +} + +// GetHostInfo returns a Hostinfo struct for the machine +func (m *Machine) GetHostInfo() (*tailcfg.Hostinfo, error) { + hostinfo := tailcfg.Hostinfo{} + if len(m.HostInfo) != 0 { + hi, err := m.HostInfo.MarshalJSON() + if err != nil { + return nil, err + } + err = json.Unmarshal(hi, &hostinfo) + if err != nil { + return nil, err + } + } + return &hostinfo, nil +} + +func (h *Headscale) isOutdated(m *Machine) bool { + err := h.UpdateMachine(m) + if err != nil { + // It does not seem meaningful to propagate this error as the end result + // will have to be that the machine has to be considered outdated. + return true + } + + sharedMachines, _ := h.getShared(m) + + namespaceSet := set.New(set.ThreadSafe) + namespaceSet.Add(m.Namespace.Name) + + // Check if any of our shared namespaces has updates that we have + // not propagated. + for _, sharedMachine := range sharedMachines { + namespaceSet.Add(sharedMachine.Namespace.Name) + } + + namespaces := make([]string, namespaceSet.Size()) + for index, namespace := range namespaceSet.List() { + namespaces[index] = namespace.(string) + } + + lastChange := h.getLastStateChange(namespaces...) + log.Trace(). + Str("func", "keepAlive"). + Str("machine", m.Name). + Time("last_successful_update", *m.LastSuccessfulUpdate). + Time("last_state_change", lastChange). + Msgf("Checking if %s is missing updates", m.Name) + return m.LastSuccessfulUpdate.Before(lastChange) +} + +func (m Machine) String() string { + return m.Name +} + +func (ms Machines) String() string { + temp := make([]string, len(ms)) + + for index, machine := range ms { + temp[index] = machine.Name + } + + return fmt.Sprintf("[ %s ](%d)", strings.Join(temp, ", "), len(temp)) +} + +// TODO(kradalby): Remove when we have generics... +func (ms MachinesP) String() string { + temp := make([]string, len(ms)) + + for index, machine := range ms { + temp[index] = machine.Name + } + + return fmt.Sprintf("[ %s ](%d)", strings.Join(temp, ", "), len(temp)) +} + +func (ms Machines) toNodes(includeRoutes bool) ([]*tailcfg.Node, error) { + nodes := make([]*tailcfg.Node, len(ms)) + + for index, machine := range ms { + node, err := machine.toNode(includeRoutes) + if err != nil { + return nil, err + } + + nodes[index] = node + } + + return nodes, nil } // toNode converts a Machine into a Tailscale Node. includeRoutes is false for shared nodes @@ -176,235 +414,3 @@ func (m Machine) toNode(includeRoutes bool) (*tailcfg.Node, error) { } return &n, nil } - -func (h *Headscale) getPeers(m Machine) (*[]*tailcfg.Node, error) { - log.Trace(). - Str("func", "getPeers"). - Str("machine", m.Name). - Msg("Finding peers") - - machines := []Machine{} - if err := h.db.Where("namespace_id = ? AND machine_key <> ? AND registered", - m.NamespaceID, m.MachineKey).Find(&machines).Error; err != nil { - log.Error().Err(err).Msg("Error accessing db") - return nil, err - } - - // We fetch here machines that are shared to the `Namespace` of the machine we are getting peers for - sharedMachines := []SharedMachine{} - if err := h.db.Preload("Namespace").Preload("Machine").Where("namespace_id = ?", - m.NamespaceID).Find(&sharedMachines).Error; err != nil { - return nil, err - } - - peers := []*tailcfg.Node{} - for _, mn := range machines { - peer, err := mn.toNode(true) - if err != nil { - return nil, err - } - peers = append(peers, peer) - } - for _, sharedMachine := range sharedMachines { - peer, err := sharedMachine.Machine.toNode(false) // shared nodes do not expose their routes - if err != nil { - return nil, err - } - peers = append(peers, peer) - } - sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID }) - - log.Trace(). - Str("func", "getPeers"). - Str("machine", m.Name). - Msgf("Found peers: %s", tailNodesToString(peers)) - return &peers, nil -} - -// GetMachine finds a Machine by name and namespace and returns the Machine struct -func (h *Headscale) GetMachine(namespace string, name string) (*Machine, error) { - machines, err := h.ListMachinesInNamespace(namespace) - if err != nil { - return nil, err - } - - for _, m := range *machines { - if m.Name == name { - return &m, nil - } - } - return nil, fmt.Errorf("machine not found") -} - -// GetMachineByID finds a Machine by ID and returns the Machine struct -func (h *Headscale) GetMachineByID(id uint64) (*Machine, error) { - m := Machine{} - if result := h.db.Preload("Namespace").Find(&Machine{ID: id}).First(&m); result.Error != nil { - return nil, result.Error - } - return &m, nil -} - -// UpdateMachine takes a Machine struct pointer (typically already loaded from database -// and updates it with the latest data from the database. -func (h *Headscale) UpdateMachine(m *Machine) error { - if result := h.db.Find(m).First(&m); result.Error != nil { - return result.Error - } - return nil -} - -// DeleteMachine softs deletes a Machine from the database -func (h *Headscale) DeleteMachine(m *Machine) error { - m.Registered = false - namespaceID := m.NamespaceID - h.db.Save(&m) // we mark it as unregistered, just in case - if err := h.db.Delete(&m).Error; err != nil { - return err - } - - return h.RequestMapUpdates(namespaceID) -} - -// HardDeleteMachine hard deletes a Machine from the database -func (h *Headscale) HardDeleteMachine(m *Machine) error { - namespaceID := m.NamespaceID - if err := h.db.Unscoped().Delete(&m).Error; err != nil { - return err - } - return h.RequestMapUpdates(namespaceID) -} - -// GetHostInfo returns a Hostinfo struct for the machine -func (m *Machine) GetHostInfo() (*tailcfg.Hostinfo, error) { - hostinfo := tailcfg.Hostinfo{} - if len(m.HostInfo) != 0 { - hi, err := m.HostInfo.MarshalJSON() - if err != nil { - return nil, err - } - err = json.Unmarshal(hi, &hostinfo) - if err != nil { - return nil, err - } - } - return &hostinfo, nil -} - -func (h *Headscale) notifyChangesToPeers(m *Machine) { - peers, err := h.getPeers(*m) - if err != nil { - log.Error(). - Str("func", "notifyChangesToPeers"). - Str("machine", m.Name). - Msgf("Error getting peers: %s", err) - return - } - for _, p := range *peers { - log.Info(). - Str("func", "notifyChangesToPeers"). - Str("machine", m.Name). - Str("peer", p.Name). - Str("address", p.Addresses[0].String()). - Msgf("Notifying peer %s (%s)", p.Name, p.Addresses[0]) - err := h.sendRequestOnUpdateChannel(p) - if err != nil { - log.Info(). - Str("func", "notifyChangesToPeers"). - Str("machine", m.Name). - Str("peer", p.Name). - Msgf("Peer %s does not appear to be polling", p.Name) - } - log.Trace(). - Str("func", "notifyChangesToPeers"). - Str("machine", m.Name). - Str("peer", p.Name). - Str("address", p.Addresses[0].String()). - Msgf("Notified peer %s (%s)", p.Name, p.Addresses[0]) - } -} - -func (h *Headscale) getOrOpenUpdateChannel(m *Machine) <-chan struct{} { - var updateChan chan struct{} - if storedChan, ok := h.clientsUpdateChannels.Load(m.ID); ok { - if unwrapped, ok := storedChan.(chan struct{}); ok { - updateChan = unwrapped - } else { - log.Error(). - Str("handler", "openUpdateChannel"). - Str("machine", m.Name). - Msg("Failed to convert update channel to struct{}") - } - } else { - log.Debug(). - Str("handler", "openUpdateChannel"). - Str("machine", m.Name). - Msg("Update channel not found, creating") - - updateChan = make(chan struct{}) - h.clientsUpdateChannels.Store(m.ID, updateChan) - } - return updateChan -} - -func (h *Headscale) closeUpdateChannel(m *Machine) { - h.clientsUpdateChannelMutex.Lock() - defer h.clientsUpdateChannelMutex.Unlock() - - if storedChan, ok := h.clientsUpdateChannels.Load(m.ID); ok { - if unwrapped, ok := storedChan.(chan struct{}); ok { - close(unwrapped) - } - } - h.clientsUpdateChannels.Delete(m.ID) -} - -func (h *Headscale) sendRequestOnUpdateChannel(m *tailcfg.Node) error { - h.clientsUpdateChannelMutex.Lock() - defer h.clientsUpdateChannelMutex.Unlock() - - pUp, ok := h.clientsUpdateChannels.Load(uint64(m.ID)) - if ok { - log.Info(). - Str("func", "requestUpdate"). - Str("machine", m.Name). - Msgf("Notifying peer %s", m.Name) - - if update, ok := pUp.(chan struct{}); ok { - log.Trace(). - Str("func", "requestUpdate"). - Str("machine", m.Name). - Msgf("Update channel is %#v", update) - - update <- struct{}{} - - log.Trace(). - Str("func", "requestUpdate"). - Str("machine", m.Name). - Msgf("Notified machine %s", m.Name) - } - } else { - log.Info(). - Str("func", "requestUpdate"). - Str("machine", m.Name). - Msgf("Machine %s does not appear to be polling", m.Name) - return errors.New("machine does not seem to be polling") - } - return nil -} - -func (h *Headscale) isOutdated(m *Machine) bool { - err := h.UpdateMachine(m) - if err != nil { - return true - } - - lastChange := h.getLastStateChange(m.Namespace.Name) - log.Trace(). - Str("func", "keepAlive"). - Str("machine", m.Name). - Time("last_successful_update", *m.LastSuccessfulUpdate). - Time("last_state_change", lastChange). - Msgf("Checking if %s is missing updates", m.Name) - return m.LastSuccessfulUpdate.Before(lastChange) -} diff --git a/machine_test.go b/machine_test.go index d535be56..dfe84d33 100644 --- a/machine_test.go +++ b/machine_test.go @@ -2,6 +2,7 @@ package headscale import ( "encoding/json" + "strconv" "gopkg.in/check.v1" ) @@ -16,7 +17,7 @@ func (s *Suite) TestGetMachine(c *check.C) { _, err = h.GetMachine("test", "testmachine") c.Assert(err, check.NotNil) - m := Machine{ + m := &Machine{ ID: 0, MachineKey: "foo", NodeKey: "bar", @@ -27,7 +28,7 @@ func (s *Suite) TestGetMachine(c *check.C) { RegisterMethod: "authKey", AuthKeyID: uint(pak.ID), } - h.db.Save(&m) + h.db.Save(m) m1, err := h.GetMachine("test", "testmachine") c.Assert(err, check.IsNil) @@ -116,3 +117,43 @@ func (s *Suite) TestHardDeleteMachine(c *check.C) { _, err = h.GetMachine(n.Name, "testmachine3") c.Assert(err, check.NotNil) } + +func (s *Suite) TestGetDirectPeers(c *check.C) { + n, err := h.CreateNamespace("test") + c.Assert(err, check.IsNil) + + pak, err := h.CreatePreAuthKey(n.Name, false, false, nil) + c.Assert(err, check.IsNil) + + _, err = h.GetMachineByID(0) + c.Assert(err, check.NotNil) + + for i := 0; i <= 10; i++ { + m := Machine{ + ID: uint64(i), + MachineKey: "foo" + strconv.Itoa(i), + NodeKey: "bar" + strconv.Itoa(i), + DiscoKey: "faa" + strconv.Itoa(i), + Name: "testmachine" + strconv.Itoa(i), + NamespaceID: n.ID, + Registered: true, + RegisterMethod: "authKey", + AuthKeyID: uint(pak.ID), + } + h.db.Save(&m) + } + + m1, err := h.GetMachineByID(0) + c.Assert(err, check.IsNil) + + _, err = m1.GetHostInfo() + c.Assert(err, check.IsNil) + + peers, err := h.getDirectPeers(m1) + c.Assert(err, check.IsNil) + + c.Assert(len(peers), check.Equals, 9) + c.Assert(peers[0].Name, check.Equals, "testmachine2") + c.Assert(peers[5].Name, check.Equals, "testmachine7") + c.Assert(peers[8].Name, check.Equals, "testmachine10") +} diff --git a/metrics.go b/metrics.go new file mode 100644 index 00000000..0d3dca34 --- /dev/null +++ b/metrics.go @@ -0,0 +1,41 @@ +package headscale + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +const prometheusNamespace = "headscale" + +var ( + // This is a high cardinality metric (namespace x machines), we might want to make this + // configurable/opt-in in the future. + lastStateUpdate = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: prometheusNamespace, + Name: "last_update_seconds", + Help: "Time stamp in unix time when a machine or headscale was updated", + }, []string{"namespace", "machine"}) + + machineRegistrations = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: prometheusNamespace, + Name: "machine_registrations_total", + Help: "The total amount of registered machine attempts", + }, []string{"action", "auth", "status", "namespace"}) + + updateRequestsFromNode = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: prometheusNamespace, + Name: "update_request_from_node_total", + Help: "The number of updates requested by a node/update function", + }, []string{"namespace", "machine", "state"}) + updateRequestsSentToNode = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: prometheusNamespace, + Name: "update_request_sent_to_node_total", + Help: "The number of calls/messages issued on a specific nodes update channel", + }, []string{"namespace", "machine", "status"}) + //TODO(kradalby): This is very debugging, we might want to remove it. + updateRequestsReceivedOnChannel = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: prometheusNamespace, + Name: "update_request_received_on_channel_total", + Help: "The number of update requests received on an update channel", + }, []string{"namespace", "machine"}) +) diff --git a/namespaces.go b/namespaces.go index 8204f96d..2bf62bb3 100644 --- a/namespaces.go +++ b/namespaces.go @@ -176,23 +176,17 @@ func (h *Headscale) checkForNamespacesPendingUpdates() { return } - names := []string{} - err = json.Unmarshal([]byte(v), &names) + namespaces := []string{} + err = json.Unmarshal([]byte(v), &namespaces) if err != nil { return } - for _, name := range names { + for _, namespace := range namespaces { log.Trace(). Str("func", "RequestMapUpdates"). - Str("machine", name). - Msg("Sending updates to nodes in namespace") - machines, err := h.ListMachinesInNamespace(name) - if err != nil { - continue - } - for _, m := range *machines { - h.notifyChangesToPeers(&m) - } + Str("machine", namespace). + Msg("Sending updates to nodes in namespacespace") + h.setLastStateChangeToNow(namespace) } newV, err := h.getValue("namespaces_pending_updates") if err != nil { diff --git a/poll.go b/poll.go index 60bfa9ea..6a652805 100644 --- a/poll.go +++ b/poll.go @@ -51,13 +51,19 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) { return } - var m Machine - if result := h.db.Preload("Namespace").First(&m, "machine_key = ?", mKey.HexString()); errors.Is(result.Error, gorm.ErrRecordNotFound) { - log.Warn(). + m, err := h.GetMachineByMachineKey(mKey.HexString()) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + log.Warn(). + Str("handler", "PollNetMap"). + Msgf("Ignoring request, cannot find machine with key %s", mKey.HexString()) + c.String(http.StatusUnauthorized, "") + return + } + log.Error(). Str("handler", "PollNetMap"). - Msgf("Ignoring request, cannot find machine with key %s", mKey.HexString()) - c.String(http.StatusUnauthorized, "") - return + Msgf("Failed to fetch machine from the database with Machine key: %s", mKey.HexString()) + c.String(http.StatusInternalServerError, "") } log.Trace(). Str("handler", "PollNetMap"). @@ -117,7 +123,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) { Str("handler", "PollNetMap"). Str("machine", m.Name). Msg("Client is starting up. Probably interested in a DERP map") - c.Data(200, "application/json; charset=utf-8", *data) + c.Data(200, "application/json; charset=utf-8", data) return } @@ -134,10 +140,9 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) { Str("id", c.Param("id")). Str("machine", m.Name). Msg("Loading or creating update channel") - updateChan := h.getOrOpenUpdateChannel(&m) + updateChan := make(chan struct{}) pollDataChan := make(chan []byte) - // defer close(pollData) keepAliveChan := make(chan []byte) @@ -149,11 +154,12 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) { Str("handler", "PollNetMap"). Str("machine", m.Name). Msg("Client sent endpoint update and is ok with a response without peer list") - c.Data(200, "application/json; charset=utf-8", *data) + c.Data(200, "application/json; charset=utf-8", data) // It sounds like we should update the nodes when we have received a endpoint update // even tho the comments in the tailscale code dont explicitly say so. - go h.notifyChangesToPeers(&m) + updateRequestsFromNode.WithLabelValues(m.Name, m.Namespace.Name, "endpoint-update").Inc() + go func() { updateChan <- struct{}{} }() return } else if req.OmitPeers && req.Stream { log.Warn(). @@ -172,13 +178,14 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) { Str("handler", "PollNetMap"). Str("machine", m.Name). Msg("Sending initial map") - go func() { pollDataChan <- *data }() + go func() { pollDataChan <- data }() log.Info(). Str("handler", "PollNetMap"). Str("machine", m.Name). Msg("Notifying peers") - go h.notifyChangesToPeers(&m) + updateRequestsFromNode.WithLabelValues(m.Name, m.Namespace.Name, "full-update").Inc() + go func() { updateChan <- struct{}{} }() h.PollNetMapStream(c, m, req, mKey, pollDataChan, keepAliveChan, updateChan, cancelKeepAlive) log.Trace(). @@ -193,15 +200,15 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) { // to the connected clients. func (h *Headscale) PollNetMapStream( c *gin.Context, - m Machine, + m *Machine, req tailcfg.MapRequest, mKey wgkey.Key, pollDataChan chan []byte, keepAliveChan chan []byte, - updateChan <-chan struct{}, + updateChan chan struct{}, cancelKeepAlive chan struct{}, ) { - go h.scheduledPollWorker(cancelKeepAlive, keepAliveChan, mKey, req, m) + go h.scheduledPollWorker(cancelKeepAlive, updateChan, keepAliveChan, mKey, req, m) c.Stream(func(w io.Writer) bool { log.Trace(). @@ -230,6 +237,7 @@ func (h *Headscale) PollNetMapStream( Str("channel", "pollData"). Err(err). Msg("Cannot write data") + return false } log.Trace(). Str("handler", "PollNetMapStream"). @@ -237,10 +245,10 @@ func (h *Headscale) PollNetMapStream( Str("channel", "pollData"). Int("bytes", len(data)). Msg("Data from pollData channel written successfully") - // TODO: Abstract away all the database calls, this can cause race conditions + // TODO(kradalby): Abstract away all the database calls, this can cause race conditions // when an outdated machine object is kept alive, e.g. db is update from // command line, but then overwritten. - err = h.UpdateMachine(&m) + err = h.UpdateMachine(m) if err != nil { log.Error(). Str("handler", "PollNetMapStream"). @@ -251,14 +259,17 @@ func (h *Headscale) PollNetMapStream( } now := time.Now().UTC() m.LastSeen = &now + + lastStateUpdate.WithLabelValues(m.Namespace.Name, m.Name).Set(float64(now.Unix())) m.LastSuccessfulUpdate = &now + h.db.Save(&m) log.Trace(). Str("handler", "PollNetMapStream"). Str("machine", m.Name). Str("channel", "pollData"). Int("bytes", len(data)). - Msg("Machine updated successfully after sending pollData") + Msg("Machine entry in database updated successfully after sending pollData") return true case data := <-keepAliveChan: @@ -276,6 +287,7 @@ func (h *Headscale) PollNetMapStream( Str("channel", "keepAlive"). Err(err). Msg("Cannot write keep alive message") + return false } log.Trace(). Str("handler", "PollNetMapStream"). @@ -283,10 +295,10 @@ func (h *Headscale) PollNetMapStream( Str("channel", "keepAlive"). Int("bytes", len(data)). Msg("Keep alive sent successfully") - // TODO: Abstract away all the database calls, this can cause race conditions + // TODO(kradalby): Abstract away all the database calls, this can cause race conditions // when an outdated machine object is kept alive, e.g. db is update from // command line, but then overwritten. - err = h.UpdateMachine(&m) + err = h.UpdateMachine(m) if err != nil { log.Error(). Str("handler", "PollNetMapStream"). @@ -312,7 +324,8 @@ func (h *Headscale) PollNetMapStream( Str("machine", m.Name). Str("channel", "update"). Msg("Received a request for update") - if h.isOutdated(&m) { + updateRequestsReceivedOnChannel.WithLabelValues(m.Name, m.Namespace.Name).Inc() + if h.isOutdated(m) { log.Debug(). Str("handler", "PollNetMapStream"). Str("machine", m.Name). @@ -328,7 +341,7 @@ func (h *Headscale) PollNetMapStream( Err(err). Msg("Could not get the map update") } - _, err = w.Write(*data) + _, err = w.Write(data) if err != nil { log.Error(). Str("handler", "PollNetMapStream"). @@ -336,21 +349,24 @@ func (h *Headscale) PollNetMapStream( Str("channel", "update"). Err(err). Msg("Could not write the map response") + updateRequestsSentToNode.WithLabelValues(m.Name, m.Namespace.Name, "failed").Inc() + return false } log.Trace(). Str("handler", "PollNetMapStream"). Str("machine", m.Name). Str("channel", "update"). Msg("Updated Map has been sent") + updateRequestsSentToNode.WithLabelValues(m.Name, m.Namespace.Name, "success").Inc() - // Keep track of the last successful update, - // we sometimes end in a state were the update - // is not picked up by a client and we use this - // to determine if we should "force" an update. - // TODO: Abstract away all the database calls, this can cause race conditions - // when an outdated machine object is kept alive, e.g. db is update from - // command line, but then overwritten. - err = h.UpdateMachine(&m) + // Keep track of the last successful update, + // we sometimes end in a state were the update + // is not picked up by a client and we use this + // to determine if we should "force" an update. + // TODO(kradalby): Abstract away all the database calls, this can cause race conditions + // when an outdated machine object is kept alive, e.g. db is update from + // command line, but then overwritten. + err = h.UpdateMachine(m) if err != nil { log.Error(). Str("handler", "PollNetMapStream"). @@ -360,7 +376,10 @@ func (h *Headscale) PollNetMapStream( Msg("Cannot update machine from database") } now := time.Now().UTC() + + lastStateUpdate.WithLabelValues(m.Namespace.Name, m.Name).Set(float64(now.Unix())) m.LastSuccessfulUpdate = &now + h.db.Save(&m) } else { log.Trace(). @@ -380,7 +399,7 @@ func (h *Headscale) PollNetMapStream( // TODO: Abstract away all the database calls, this can cause race conditions // when an outdated machine object is kept alive, e.g. db is update from // command line, but then overwritten. - err := h.UpdateMachine(&m) + err := h.UpdateMachine(m) if err != nil { log.Error(). Str("handler", "PollNetMapStream"). @@ -393,12 +412,33 @@ func (h *Headscale) PollNetMapStream( m.LastSeen = &now h.db.Save(&m) + log.Trace(). + Str("handler", "PollNetMapStream"). + Str("machine", m.Name). + Str("channel", "Done"). + Msg("Cancelling keepAlive channel") cancelKeepAlive <- struct{}{} - h.closeUpdateChannel(&m) + log.Trace(). + Str("handler", "PollNetMapStream"). + Str("machine", m.Name). + Str("channel", "Done"). + Msg("Closing update channel") + //h.closeUpdateChannel(m) + close(updateChan) + log.Trace(). + Str("handler", "PollNetMapStream"). + Str("machine", m.Name). + Str("channel", "Done"). + Msg("Closing pollData channel") close(pollDataChan) + log.Trace(). + Str("handler", "PollNetMapStream"). + Str("machine", m.Name). + Str("channel", "Done"). + Msg("Closing keepAliveChan channel") close(keepAliveChan) return false @@ -408,13 +448,14 @@ func (h *Headscale) PollNetMapStream( func (h *Headscale) scheduledPollWorker( cancelChan <-chan struct{}, + updateChan chan<- struct{}, keepAliveChan chan<- []byte, mKey wgkey.Key, req tailcfg.MapRequest, - m Machine, + m *Machine, ) { keepAliveTicker := time.NewTicker(60 * time.Second) - updateCheckerTicker := time.NewTicker(30 * time.Second) + updateCheckerTicker := time.NewTicker(10 * time.Second) for { select { @@ -435,20 +476,15 @@ func (h *Headscale) scheduledPollWorker( Str("func", "keepAlive"). Str("machine", m.Name). Msg("Sending keepalive") - keepAliveChan <- *data + keepAliveChan <- data case <-updateCheckerTicker.C: - // Send an update request regardless of outdated or not, if data is sent - // to the node is determined in the updateChan consumer block - n, _ := m.toNode(true) - err := h.sendRequestOnUpdateChannel(n) - if err != nil { - log.Error(). - Str("func", "keepAlive"). - Str("machine", m.Name). - Err(err). - Msgf("Failed to send update request to %s", m.Name) - } + log.Debug(). + Str("func", "scheduledPollWorker"). + Str("machine", m.Name). + Msg("Sending update request") + updateRequestsFromNode.WithLabelValues(m.Name, m.Namespace.Name, "scheduled-update").Inc() + updateChan <- struct{}{} } } } diff --git a/sharing_test.go b/sharing_test.go index ec4951de..baa90d0c 100644 --- a/sharing_test.go +++ b/sharing_test.go @@ -2,7 +2,6 @@ package headscale import ( "gopkg.in/check.v1" - "tailscale.com/tailcfg" ) func (s *Suite) TestBasicSharedNodesInNamespace(c *check.C) { @@ -21,7 +20,7 @@ func (s *Suite) TestBasicSharedNodesInNamespace(c *check.C) { _, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1") c.Assert(err, check.NotNil) - m1 := Machine{ + m1 := &Machine{ ID: 0, MachineKey: "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66", NodeKey: "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66", @@ -33,12 +32,12 @@ func (s *Suite) TestBasicSharedNodesInNamespace(c *check.C) { IPAddress: "100.64.0.1", AuthKeyID: uint(pak1.ID), } - h.db.Save(&m1) + h.db.Save(m1) _, err = h.GetMachine(n1.Name, m1.Name) c.Assert(err, check.IsNil) - m2 := Machine{ + m2 := &Machine{ ID: 1, MachineKey: "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863", NodeKey: "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863", @@ -50,22 +49,22 @@ func (s *Suite) TestBasicSharedNodesInNamespace(c *check.C) { IPAddress: "100.64.0.2", AuthKeyID: uint(pak2.ID), } - h.db.Save(&m2) + h.db.Save(m2) _, err = h.GetMachine(n2.Name, m2.Name) c.Assert(err, check.IsNil) p1s, err := h.getPeers(m1) c.Assert(err, check.IsNil) - c.Assert(len(*p1s), check.Equals, 0) + c.Assert(len(p1s), check.Equals, 0) - err = h.AddSharedMachineToNamespace(&m2, n1) + err = h.AddSharedMachineToNamespace(m2, n1) c.Assert(err, check.IsNil) p1sAfter, err := h.getPeers(m1) c.Assert(err, check.IsNil) - c.Assert(len(*p1sAfter), check.Equals, 1) - c.Assert((*p1sAfter)[0].ID, check.Equals, tailcfg.NodeID(m2.ID)) + c.Assert(len(p1sAfter), check.Equals, 1) + c.Assert(p1sAfter[0].ID, check.Equals, m2.ID) } func (s *Suite) TestSameNamespace(c *check.C) { @@ -84,7 +83,7 @@ func (s *Suite) TestSameNamespace(c *check.C) { _, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1") c.Assert(err, check.NotNil) - m1 := Machine{ + m1 := &Machine{ ID: 0, MachineKey: "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66", NodeKey: "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66", @@ -96,12 +95,12 @@ func (s *Suite) TestSameNamespace(c *check.C) { IPAddress: "100.64.0.1", AuthKeyID: uint(pak1.ID), } - h.db.Save(&m1) + h.db.Save(m1) _, err = h.GetMachine(n1.Name, m1.Name) c.Assert(err, check.IsNil) - m2 := Machine{ + m2 := &Machine{ ID: 1, MachineKey: "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863", NodeKey: "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863", @@ -113,16 +112,16 @@ func (s *Suite) TestSameNamespace(c *check.C) { IPAddress: "100.64.0.2", AuthKeyID: uint(pak2.ID), } - h.db.Save(&m2) + h.db.Save(m2) _, err = h.GetMachine(n2.Name, m2.Name) c.Assert(err, check.IsNil) p1s, err := h.getPeers(m1) c.Assert(err, check.IsNil) - c.Assert(len(*p1s), check.Equals, 0) + c.Assert(len(p1s), check.Equals, 0) - err = h.AddSharedMachineToNamespace(&m1, n1) + err = h.AddSharedMachineToNamespace(m1, n1) c.Assert(err, check.Equals, errorSameNamespace) } @@ -142,7 +141,7 @@ func (s *Suite) TestAlreadyShared(c *check.C) { _, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1") c.Assert(err, check.NotNil) - m1 := Machine{ + m1 := &Machine{ ID: 0, MachineKey: "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66", NodeKey: "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66", @@ -154,12 +153,12 @@ func (s *Suite) TestAlreadyShared(c *check.C) { IPAddress: "100.64.0.1", AuthKeyID: uint(pak1.ID), } - h.db.Save(&m1) + h.db.Save(m1) _, err = h.GetMachine(n1.Name, m1.Name) c.Assert(err, check.IsNil) - m2 := Machine{ + m2 := &Machine{ ID: 1, MachineKey: "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863", NodeKey: "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863", @@ -171,18 +170,18 @@ func (s *Suite) TestAlreadyShared(c *check.C) { IPAddress: "100.64.0.2", AuthKeyID: uint(pak2.ID), } - h.db.Save(&m2) + h.db.Save(m2) _, err = h.GetMachine(n2.Name, m2.Name) c.Assert(err, check.IsNil) p1s, err := h.getPeers(m1) c.Assert(err, check.IsNil) - c.Assert(len(*p1s), check.Equals, 0) + c.Assert(len(p1s), check.Equals, 0) - err = h.AddSharedMachineToNamespace(&m2, n1) + err = h.AddSharedMachineToNamespace(m2, n1) c.Assert(err, check.IsNil) - err = h.AddSharedMachineToNamespace(&m2, n1) + err = h.AddSharedMachineToNamespace(m2, n1) c.Assert(err, check.Equals, errorMachineAlreadyShared) } @@ -202,7 +201,7 @@ func (s *Suite) TestDoNotIncludeRoutesOnShared(c *check.C) { _, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1") c.Assert(err, check.NotNil) - m1 := Machine{ + m1 := &Machine{ ID: 0, MachineKey: "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66", NodeKey: "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66", @@ -214,12 +213,12 @@ func (s *Suite) TestDoNotIncludeRoutesOnShared(c *check.C) { IPAddress: "100.64.0.1", AuthKeyID: uint(pak1.ID), } - h.db.Save(&m1) + h.db.Save(m1) _, err = h.GetMachine(n1.Name, m1.Name) c.Assert(err, check.IsNil) - m2 := Machine{ + m2 := &Machine{ ID: 1, MachineKey: "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863", NodeKey: "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863", @@ -231,22 +230,22 @@ func (s *Suite) TestDoNotIncludeRoutesOnShared(c *check.C) { IPAddress: "100.64.0.2", AuthKeyID: uint(pak2.ID), } - h.db.Save(&m2) + h.db.Save(m2) _, err = h.GetMachine(n2.Name, m2.Name) c.Assert(err, check.IsNil) p1s, err := h.getPeers(m1) c.Assert(err, check.IsNil) - c.Assert(len(*p1s), check.Equals, 0) + c.Assert(len(p1s), check.Equals, 0) - err = h.AddSharedMachineToNamespace(&m2, n1) + err = h.AddSharedMachineToNamespace(m2, n1) c.Assert(err, check.IsNil) p1sAfter, err := h.getPeers(m1) c.Assert(err, check.IsNil) - c.Assert(len(*p1sAfter), check.Equals, 1) - c.Assert(len((*p1sAfter)[0].AllowedIPs), check.Equals, 1) + c.Assert(len(p1sAfter), check.Equals, 1) + c.Assert(p1sAfter[0].Name, check.Equals, "test_get_shared_nodes_2") } func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) { @@ -274,7 +273,7 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) { _, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1") c.Assert(err, check.NotNil) - m1 := Machine{ + m1 := &Machine{ ID: 0, MachineKey: "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66", NodeKey: "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66", @@ -286,12 +285,12 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) { IPAddress: "100.64.0.1", AuthKeyID: uint(pak1.ID), } - h.db.Save(&m1) + h.db.Save(m1) _, err = h.GetMachine(n1.Name, m1.Name) c.Assert(err, check.IsNil) - m2 := Machine{ + m2 := &Machine{ ID: 1, MachineKey: "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863", NodeKey: "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863", @@ -303,12 +302,12 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) { IPAddress: "100.64.0.2", AuthKeyID: uint(pak2.ID), } - h.db.Save(&m2) + h.db.Save(m2) _, err = h.GetMachine(n2.Name, m2.Name) c.Assert(err, check.IsNil) - m3 := Machine{ + m3 := &Machine{ ID: 2, MachineKey: "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863", NodeKey: "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863", @@ -320,12 +319,12 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) { IPAddress: "100.64.0.3", AuthKeyID: uint(pak3.ID), } - h.db.Save(&m3) + h.db.Save(m3) _, err = h.GetMachine(n3.Name, m3.Name) c.Assert(err, check.IsNil) - m4 := Machine{ + m4 := &Machine{ ID: 3, MachineKey: "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863", NodeKey: "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863", @@ -337,23 +336,31 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) { IPAddress: "100.64.0.4", AuthKeyID: uint(pak4.ID), } - h.db.Save(&m4) + h.db.Save(m4) _, err = h.GetMachine(n1.Name, m4.Name) c.Assert(err, check.IsNil) p1s, err := h.getPeers(m1) c.Assert(err, check.IsNil) - c.Assert(len(*p1s), check.Equals, 1) // nodes 1 and 4 + c.Assert(len(p1s), check.Equals, 1) // nodes 1 and 4 + c.Assert(p1s[0].Name, check.Equals, "test_get_shared_nodes_4") - err = h.AddSharedMachineToNamespace(&m2, n1) + err = h.AddSharedMachineToNamespace(m2, n1) c.Assert(err, check.IsNil) p1sAfter, err := h.getPeers(m1) c.Assert(err, check.IsNil) - c.Assert(len(*p1sAfter), check.Equals, 2) // nodes 1, 2, 4 + c.Assert(len(p1sAfter), check.Equals, 2) // nodes 1, 2, 4 + c.Assert(p1sAfter[0].Name, check.Equals, "test_get_shared_nodes_2") + c.Assert(p1sAfter[1].Name, check.Equals, "test_get_shared_nodes_4") + + node1shared, err := h.getShared(m1) + c.Assert(err, check.IsNil) + c.Assert(len(node1shared), check.Equals, 1) // nodes 1, 2, 4 + c.Assert(node1shared[0].Name, check.Equals, "test_get_shared_nodes_2") pAlone, err := h.getPeers(m3) c.Assert(err, check.IsNil) - c.Assert(len(*pAlone), check.Equals, 0) // node 3 is alone + c.Assert(len(pAlone), check.Equals, 0) // node 3 is alone }