Merge pull request #81 from kradalby/integration-tests

Add Integration tests

.dockerignore

@@ -0,0 +1,16 @@
+// integration tests are not needed in docker
+// ignoring it let us speed up the integration test
+// development
+integration_test.go
+
+Dockerfile*
+docker-compose*
+.dockerignore
+.goreleaser.yml
+.git
+.github
+.gitignore
+README.md
+LICENSE
+.vscode
+

.gitignore

@@ -18,3 +18,4 @@
 config.json
 *.key
 /db.sqlite
+*.sqlite3

Dockerfile

@@ -1,12 +1,19 @@
 FROM golang:latest AS build
 ENV GOPATH /go
-COPY . /go/src/headscale
+
+COPY go.mod go.sum /go/src/headscale/
 WORKDIR /go/src/headscale
+RUN go mod download
+
+COPY . /go/src/headscale
+
 RUN go install -a -ldflags="-extldflags=-static" -tags netgo,sqlite_omit_load_extension ./cmd/headscale
 RUN test -e /go/bin/headscale
 
-FROM scratch
-COPY --from=build /go/bin/headscale /go/bin/headscale
+FROM ubuntu:latest
+
+COPY --from=build /go/bin/headscale /usr/local/bin/headscale
 ENV TZ UTC
+
 EXPOSE 8080/tcp
-ENTRYPOINT ["/go/bin/headscale"]
+CMD ["headscale"]

Dockerfile.tailscale

@@ -0,0 +1,9 @@
+FROM ubuntu:latest
+
+RUN apt-get update \
+    && apt-get install -y gnupg curl \
+    && curl -fsSL https://pkgs.tailscale.com/stable/ubuntu/focal.gpg | apt-key add - \
+    && curl -fsSL https://pkgs.tailscale.com/stable/ubuntu/focal.list | tee /etc/apt/sources.list.d/tailscale.list \
+    && apt-get update \
+    && apt-get install -y tailscale \
+    && rm -rf /var/lib/apt/lists/*

Makefile

@@ -9,6 +9,9 @@ dev: lint test build
 test:
 	@go test -coverprofile=coverage.out ./...
 
+test_integration:
+	go test -tags integration -timeout 30m ./...
+
 coverprofile_func:
 	go tool cover -func=coverage.out
 

README.md

@@ -119,6 +119,11 @@ Headscale's configuration file is named `config.json` or `config.yaml`. Headscal
 
 `server_url` is the external URL via which Headscale is reachable. `listen_addr` is the IP address and port the Headscale program should listen on. `ip_prefix` is the IP prefix (range) in which IP addresses for nodes will be allocated (default 100.64.0.0/10, e.g., 192.168.4.0/24, 10.0.0.0/8)
 
+```
+    "log_level": "debug"
+```
+`log_level` can be used to set the Log level for Headscale, it defaults to `debug`, and the available levels are: `trace`, `debug`, `info`, `warn` and `error`.
+
 ```
     "private_key_path": "private.key",
 ```

acls.go

@@ -4,11 +4,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"strconv"
 	"strings"
 
+	"github.com/rs/zerolog/log"
+
 	"github.com/tailscale/hujson"
 	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
@@ -66,7 +67,8 @@ func (h *Headscale) generateACLRules() (*[]tailcfg.FilterRule, error) {
 		for j, u := range a.Users {
 			srcs, err := h.generateACLPolicySrcIP(u)
 			if err != nil {
-				log.Printf("Error parsing ACL %d, User %d", i, j)
+				log.Error().
+					Msgf("Error parsing ACL %d, User %d", i, j)
 				return nil, err
 			}
 			srcIPs = append(srcIPs, *srcs...)
@@ -77,7 +79,8 @@ func (h *Headscale) generateACLRules() (*[]tailcfg.FilterRule, error) {
 		for j, d := range a.Ports {
 			dests, err := h.generateACLPolicyDestPorts(d)
 			if err != nil {
-				log.Printf("Error parsing ACL %d, Port %d", i, j)
+				log.Error().
+					Msgf("Error parsing ACL %d, Port %d", i, j)
 				return nil, err
 			}
 			destPorts = append(destPorts, *dests...)

api.go

@@ -6,10 +6,11 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"log"
 	"net/http"
 	"time"
 
+	"github.com/rs/zerolog/log"
+
 	"github.com/gin-gonic/gin"
 	"github.com/klauspost/compress/zstd"
 	"gorm.io/datatypes"
@@ -63,21 +64,27 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 	mKeyStr := c.Param("id")
 	mKey, err := wgkey.ParseHex(mKeyStr)
 	if err != nil {
-		log.Printf("Cannot parse machine key: %s", err)
+		log.Error().
+			Str("handler", "Registration").
+			Err(err).
+			Msg("Cannot parse machine key")
 		c.String(http.StatusInternalServerError, "Sad!")
 		return
 	}
 	req := tailcfg.RegisterRequest{}
 	err = decode(body, &req, &mKey, h.privateKey)
 	if err != nil {
-		log.Printf("Cannot decode message: %s", err)
+		log.Error().
+			Str("handler", "Registration").
+			Err(err).
+			Msg("Cannot decode message")
 		c.String(http.StatusInternalServerError, "Very sad!")
 		return
 	}
 
 	var m Machine
 	if result := h.db.Preload("Namespace").First(&m, "machine_key = ?", mKey.HexString()); errors.Is(result.Error, gorm.ErrRecordNotFound) {
-		log.Println("New Machine!")
+		log.Info().Str("machine", req.Hostinfo.Hostname).Msg("New machine")
 		m = Machine{
 			Expiry:     &req.Expiry,
 			MachineKey: mKey.HexString(),
@@ -85,7 +92,10 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 			NodeKey:    wgkey.Key(req.NodeKey).HexString(),
 		}
 		if err := h.db.Create(&m).Error; err != nil {
-			log.Printf("Could not create row: %s", err)
+			log.Error().
+				Str("handler", "Registration").
+				Err(err).
+				Msg("Could not create row")
 			return
 		}
 	}
@@ -100,13 +110,20 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 	// We have the updated key!
 	if m.NodeKey == wgkey.Key(req.NodeKey).HexString() {
 		if m.Registered {
-			log.Printf("[%s] Client is registered and we have the current NodeKey. All clear to /map", m.Name)
+			log.Debug().
+				Str("handler", "Registration").
+				Str("machine", m.Name).
+				Msg("Client is registered and we have the current NodeKey. All clear to /map")
+
 			resp.AuthURL = ""
 			resp.MachineAuthorized = true
 			resp.User = *m.Namespace.toUser()
 			respBody, err := encode(resp, &mKey, h.privateKey)
 			if err != nil {
-				log.Printf("Cannot encode message: %s", err)
+				log.Error().
+					Str("handler", "Registration").
+					Err(err).
+					Msg("Cannot encode message")
 				c.String(http.StatusInternalServerError, "")
 				return
 			}
@@ -114,12 +131,18 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 			return
 		}
 
-		log.Printf("[%s] Not registered and not NodeKey rotation. Sending a authurl to register", m.Name)
+		log.Debug().
+			Str("handler", "Registration").
+			Str("machine", m.Name).
+			Msg("Not registered and not NodeKey rotation. Sending a authurl to register")
 		resp.AuthURL = fmt.Sprintf("%s/register?key=%s",
 			h.cfg.ServerURL, mKey.HexString())
 		respBody, err := encode(resp, &mKey, h.privateKey)
 		if err != nil {
-			log.Printf("Cannot encode message: %s", err)
+			log.Error().
+				Str("handler", "Registration").
+				Err(err).
+				Msg("Cannot encode message")
 			c.String(http.StatusInternalServerError, "")
 			return
 		}
@@ -129,7 +152,10 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 
 	// The NodeKey we have matches OldNodeKey, which means this is a refresh after an key expiration
 	if m.NodeKey == wgkey.Key(req.OldNodeKey).HexString() {
-		log.Printf("[%s] We have the OldNodeKey in the database. This is a key refresh", m.Name)
+		log.Debug().
+			Str("handler", "Registration").
+			Str("machine", m.Name).
+			Msg("We have the OldNodeKey in the database. This is a key refresh")
 		m.NodeKey = wgkey.Key(req.NodeKey).HexString()
 		h.db.Save(&m)
 
@@ -137,7 +163,10 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 		resp.User = *m.Namespace.toUser()
 		respBody, err := encode(resp, &mKey, h.privateKey)
 		if err != nil {
-			log.Printf("Cannot encode message: %s", err)
+			log.Error().
+				Str("handler", "Registration").
+				Err(err).
+				Msg("Cannot encode message")
 			c.String(http.StatusInternalServerError, "Extremely sad!")
 			return
 		}
@@ -148,25 +177,38 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 	// We arrive here after a client is restarted without finalizing the authentication flow or
 	// when headscale is stopped in the middle of the auth process.
 	if m.Registered {
-		log.Printf("[%s] The node is sending us a new NodeKey, but machine is registered. All clear for /map", m.Name)
+		log.Debug().
+			Str("handler", "Registration").
+			Str("machine", m.Name).
+			Msg("The node is sending us a new NodeKey, but machine is registered. All clear for /map")
 		resp.AuthURL = ""
 		resp.MachineAuthorized = true
 		resp.User = *m.Namespace.toUser()
 		respBody, err := encode(resp, &mKey, h.privateKey)
 		if err != nil {
-			log.Printf("Cannot encode message: %s", err)
+			log.Error().
+				Str("handler", "Registration").
+				Err(err).
+				Msg("Cannot encode message")
 			c.String(http.StatusInternalServerError, "")
 			return
 		}
 		c.Data(200, "application/json; charset=utf-8", respBody)
 		return
 	}
-	log.Printf("[%s] The node is sending us a new NodeKey, sending auth url", m.Name)
+
+	log.Debug().
+		Str("handler", "Registration").
+		Str("machine", m.Name).
+		Msg("The node is sending us a new NodeKey, sending auth url")
 	resp.AuthURL = fmt.Sprintf("%s/register?key=%s",
 		h.cfg.ServerURL, mKey.HexString())
 	respBody, err := encode(resp, &mKey, h.privateKey)
 	if err != nil {
-		log.Printf("Cannot encode message: %s", err)
+		log.Error().
+			Str("handler", "Registration").
+			Err(err).
+			Msg("Cannot encode message")
 		c.String(http.StatusInternalServerError, "")
 		return
 	}
@@ -183,28 +225,45 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 //
 // At this moment the updates are sent in a quite horrendous way, but they kinda work.
 func (h *Headscale) PollNetMapHandler(c *gin.Context) {
+	log.Trace().
+		Str("handler", "PollNetMap").
+		Str("id", c.Param("id")).
+		Msg("PollNetMapHandler called")
 	body, _ := io.ReadAll(c.Request.Body)
 	mKeyStr := c.Param("id")
 	mKey, err := wgkey.ParseHex(mKeyStr)
 	if err != nil {
-		log.Printf("Cannot parse client key: %s", err)
+		log.Error().
+			Str("handler", "PollNetMap").
+			Err(err).
+			Msg("Cannot parse client key")
 		c.String(http.StatusBadRequest, "")
 		return
 	}
 	req := tailcfg.MapRequest{}
 	err = decode(body, &req, &mKey, h.privateKey)
 	if err != nil {
-		log.Printf("Cannot decode message: %s", err)
+		log.Error().
+			Str("handler", "PollNetMap").
+			Err(err).
+			Msg("Cannot decode message")
 		c.String(http.StatusBadRequest, "")
 		return
 	}
 
 	var m Machine
 	if result := h.db.Preload("Namespace").First(&m, "machine_key = ?", mKey.HexString()); errors.Is(result.Error, gorm.ErrRecordNotFound) {
-		log.Printf("Ignoring request, cannot find machine with key %s", mKey.HexString())
+		log.Warn().
+			Str("handler", "PollNetMap").
+			Msgf("Ignoring request, cannot find machine with key %s", mKey.HexString())
 		c.String(http.StatusUnauthorized, "")
 		return
 	}
+	log.Trace().
+		Str("handler", "PollNetMap").
+		Str("id", c.Param("id")).
+		Str("machine", m.Name).
+		Msg("Found machine in database")
 
 	hostinfo, _ := json.Marshal(req.Hostinfo)
 	m.Name = req.Hostinfo.Hostname
@@ -227,17 +286,29 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 	}
 	h.db.Save(&m)
 
-	pollData := make(chan []byte, 1)
 	update := make(chan []byte, 1)
-	cancelKeepAlive := make(chan []byte, 1)
+
+	pollData := make(chan []byte, 1)
 	defer close(pollData)
+
+	cancelKeepAlive := make(chan []byte, 1)
 	defer close(cancelKeepAlive)
-	h.pollMu.Lock()
-	h.clientsPolling[m.ID] = update
-	h.pollMu.Unlock()
+
+	log.Trace().
+		Str("handler", "PollNetMap").
+		Str("id", c.Param("id")).
+		Str("machine", m.Name).
+		Msg("Storing update channel")
+	h.clientsPolling.Store(m.ID, update)
 
 	data, err := h.getMapResponse(mKey, req, m)
 	if err != nil {
+		log.Error().
+			Str("handler", "PollNetMap").
+			Str("id", c.Param("id")).
+			Str("machine", m.Name).
+			Err(err).
+			Msg("Failed to get Map response")
 		c.String(http.StatusInternalServerError, ":(")
 		return
 	}
@@ -247,50 +318,90 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 	// empty endpoints to peers)
 
 	// Details on the protocol can be found in https://github.com/tailscale/tailscale/blob/main/tailcfg/tailcfg.go#L696
-	log.Printf("[%s] ReadOnly=%t   OmitPeers=%t    Stream=%t", m.Name, req.ReadOnly, req.OmitPeers, req.Stream)
+	log.Debug().
+		Str("handler", "PollNetMap").
+		Str("id", c.Param("id")).
+		Str("machine", m.Name).
+		Bool("readOnly", req.ReadOnly).
+		Bool("omitPeers", req.OmitPeers).
+		Bool("stream", req.Stream).
+		Msg("Client map request processed")
 
 	if req.ReadOnly {
-		log.Printf("[%s] Client is starting up. Asking for DERP map", m.Name)
+		log.Info().
+			Str("handler", "PollNetMap").
+			Str("machine", m.Name).
+			Msg("Client is starting up. Asking for DERP map")
 		c.Data(200, "application/json; charset=utf-8", *data)
 		return
 	}
 	if req.OmitPeers && !req.Stream {
-		log.Printf("[%s] Client sent endpoint update and is ok with a response without peer list", m.Name)
+		log.Info().
+			Str("handler", "PollNetMap").
+			Str("machine", m.Name).
+			Msg("Client sent endpoint update and is ok with a response without peer list")
 		c.Data(200, "application/json; charset=utf-8", *data)
 		return
 	} else if req.OmitPeers && req.Stream {
-		log.Printf("[%s] Warning, ignoring request, don't know how to handle it", m.Name)
+		log.Warn().
+			Str("handler", "PollNetMap").
+			Str("machine", m.Name).
+			Msg("Ignoring request, don't know how to handle it")
 		c.String(http.StatusBadRequest, "")
 		return
 	}
 
-	log.Printf("[%s] Client is ready to access the tailnet", m.Name)
-	log.Printf("[%s] Sending initial map", m.Name)
+	log.Info().
+		Str("handler", "PollNetMap").
+		Str("machine", m.Name).
+		Msg("Client is ready to access the tailnet")
+	log.Info().
+		Str("handler", "PollNetMap").
+		Str("machine", m.Name).
+		Msg("Sending initial map")
 	pollData <- *data
 
-	log.Printf("[%s] Notifying peers", m.Name)
+	log.Info().
+		Str("handler", "PollNetMap").
+		Str("machine", m.Name).
+		Msg("Notifying peers")
 	peers, _ := h.getPeers(m)
-	h.pollMu.Lock()
 	for _, p := range *peers {
-		pUp, ok := h.clientsPolling[uint64(p.ID)]
+		pUp, ok := h.clientsPolling.Load(uint64(p.ID))
 		if ok {
-			log.Printf("[%s] Notifying peer %s (%s)", m.Name, p.Name, p.Addresses[0])
-			pUp <- []byte{}
+			log.Info().
+				Str("handler", "PollNetMap").
+				Str("machine", m.Name).
+				Str("peer", m.Name).
+				Str("address", p.Addresses[0].String()).
+				Msgf("Notifying peer %s (%s)", p.Name, p.Addresses[0])
+			pUp.(chan []byte) <- []byte{}
 		} else {
-			log.Printf("[%s] Peer %s does not appear to be polling", m.Name, p.Name)
+			log.Info().
+				Str("handler", "PollNetMap").
+				Str("machine", m.Name).
+				Str("peer", m.Name).
+				Msgf("Peer %s does not appear to be polling", p.Name)
 		}
 	}
-	h.pollMu.Unlock()
 
 	go h.keepAlive(cancelKeepAlive, pollData, mKey, req, m)
 
 	c.Stream(func(w io.Writer) bool {
 		select {
 		case data := <-pollData:
-			log.Printf("[%s] Sending data (%d bytes)", m.Name, len(data))
+			log.Trace().
+				Str("handler", "PollNetMap").
+				Str("machine", m.Name).
+				Int("bytes", len(data)).
+				Msg("Sending data")
 			_, err := w.Write(data)
 			if err != nil {
-				log.Printf("[%s] Cannot write data: %s", m.Name, err)
+				log.Error().
+					Str("handler", "PollNetMap").
+					Str("machine", m.Name).
+					Err(err).
+					Msg("Cannot write data")
 			}
 			now := time.Now().UTC()
 			m.LastSeen = &now
@@ -298,27 +409,39 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 			return true
 
 		case <-update:
-			log.Printf("[%s] Received a request for update", m.Name)
+			log.Debug().
+				Str("handler", "PollNetMap").
+				Str("machine", m.Name).
+				Msg("Received a request for update")
 			data, err := h.getMapResponse(mKey, req, m)
 			if err != nil {
-				log.Printf("[%s] Could not get the map update: %s", m.Name, err)
+				log.Error().
+					Str("handler", "PollNetMap").
+					Str("machine", m.Name).
+					Err(err).
+					Msg("Could not get the map update")
 			}
 			_, err = w.Write(*data)
 			if err != nil {
-				log.Printf("[%s] Could not write the map response: %s", m.Name, err)
+				log.Error().
+					Str("handler", "PollNetMap").
+					Str("machine", m.Name).
+					Err(err).
+					Msg("Could not write the map response")
 			}
 			return true
 
 		case <-c.Request.Context().Done():
-			log.Printf("[%s] The client has closed the connection", m.Name)
+			log.Info().
+				Str("handler", "PollNetMap").
+				Str("machine", m.Name).
+				Msg("The client has closed the connection")
 			now := time.Now().UTC()
 			m.LastSeen = &now
 			h.db.Save(&m)
-			h.pollMu.Lock()
 			cancelKeepAlive <- []byte{}
-			delete(h.clientsPolling, m.ID)
+			h.clientsPolling.Delete(m.ID)
 			close(update)
-			h.pollMu.Unlock()
 			return false
 
 		}
@@ -332,29 +455,45 @@ func (h *Headscale) keepAlive(cancel chan []byte, pollData chan []byte, mKey wgk
 			return
 
 		default:
-			h.pollMu.Lock()
 			data, err := h.getMapKeepAliveResponse(mKey, req, m)
 			if err != nil {
-				log.Printf("Error generating the keep alive msg: %s", err)
+				log.Error().
+					Str("func", "keepAlive").
+					Err(err).
+					Msg("Error generating the keep alive msg")
 				return
 			}
-			log.Printf("[%s] Sending keepalive", m.Name)
+
+			log.Debug().
+				Str("func", "keepAlive").
+				Str("machine", m.Name).
+				Msg("Sending keepalive")
 			pollData <- *data
-			h.pollMu.Unlock()
+
 			time.Sleep(60 * time.Second)
 		}
 	}
 }
 
 func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Machine) (*[]byte, error) {
+	log.Trace().
+		Str("func", "getMapResponse").
+		Str("machine", req.Hostinfo.Hostname).
+		Msg("Creating Map response")
 	node, err := m.toNode()
 	if err != nil {
-		log.Printf("Cannot convert to node: %s", err)
+		log.Error().
+			Str("func", "getMapResponse").
+			Err(err).
+			Msg("Cannot convert to node")
 		return nil, err
 	}
 	peers, err := h.getPeers(m)
 	if err != nil {
-		log.Printf("Cannot fetch peers: %s", err)
+		log.Error().
+			Str("func", "getMapResponse").
+			Err(err).
+			Msg("Cannot fetch peers")
 		return nil, err
 	}
 
@@ -426,26 +565,49 @@ func (h *Headscale) getMapKeepAliveResponse(mKey wgkey.Key, req tailcfg.MapReque
 }
 
 func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key, req tailcfg.RegisterRequest, m Machine) {
+	log.Debug().
+		Str("func", "handleAuthKey").
+		Str("machine", req.Hostinfo.Hostname).
+		Msgf("Processing auth key for %s", req.Hostinfo.Hostname)
 	resp := tailcfg.RegisterResponse{}
 	pak, err := h.checkKeyValidity(req.Auth.AuthKey)
 	if err != nil {
 		resp.MachineAuthorized = false
 		respBody, err := encode(resp, &idKey, h.privateKey)
 		if err != nil {
-			log.Printf("Cannot encode message: %s", err)
+			log.Error().
+				Str("func", "handleAuthKey").
+				Str("machine", m.Name).
+				Err(err).
+				Msg("Cannot encode message")
 			c.String(http.StatusInternalServerError, "")
 			return
 		}
 		c.Data(200, "application/json; charset=utf-8", respBody)
-		log.Printf("[%s] Failed authentication via AuthKey", m.Name)
+		log.Error().
+			Str("func", "handleAuthKey").
+			Str("machine", m.Name).
+			Msg("Failed authentication via AuthKey")
 		return
 	}
+
+	log.Debug().
+		Str("func", "handleAuthKey").
+		Str("machine", m.Name).
+		Msg("Authentication key was valid, proceeding to acquire an IP address")
 	ip, err := h.getAvailableIP()
 	if err != nil {
-		log.Println(err)
+		log.Error().
+			Str("func", "handleAuthKey").
+			Str("machine", m.Name).
+			Msg("Failed to find an available IP")
 		return
 	}
-	log.Printf("Assigning %s to %s", ip, m.Name)
+	log.Info().
+		Str("func", "handleAuthKey").
+		Str("machine", m.Name).
+		Str("ip", ip.String()).
+		Msgf("Assining %s to %s", ip, m.Name)
 
 	m.AuthKeyID = uint(pak.ID)
 	m.IPAddress = ip.String()
@@ -459,10 +621,18 @@ func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key,
 	resp.User = *pak.Namespace.toUser()
 	respBody, err := encode(resp, &idKey, h.privateKey)
 	if err != nil {
-		log.Printf("Cannot encode message: %s", err)
+		log.Error().
+			Str("func", "handleAuthKey").
+			Str("machine", m.Name).
+			Err(err).
+			Msg("Cannot encode message")
 		c.String(http.StatusInternalServerError, "Extremely sad!")
 		return
 	}
 	c.Data(200, "application/json; charset=utf-8", respBody)
-	log.Printf("[%s] Successfully authenticated via AuthKey", m.Name)
+	log.Info().
+		Str("func", "handleAuthKey").
+		Str("machine", m.Name).
+		Str("ip", ip.String()).
+		Msg("Successfully authenticated via AuthKey")
 }

app.go

@@ -3,13 +3,14 @@ package headscale
 import (
 	"errors"
 	"fmt"
-	"log"
 	"net/http"
 	"os"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/rs/zerolog/log"
+
 	"github.com/gin-gonic/gin"
 	"golang.org/x/crypto/acme/autocert"
 	"gorm.io/gorm"
@@ -57,8 +58,7 @@ type Headscale struct {
 	aclPolicy *ACLPolicy
 	aclRules  *[]tailcfg.FilterRule
 
-	pollMu         sync.Mutex
-	clientsPolling map[uint64]chan []byte // this is by all means a hackity hack
+	clientsPolling sync.Map
 }
 
 // NewHeadscale returns the Headscale app
@@ -98,7 +98,6 @@ func NewHeadscale(cfg Config) (*Headscale, error) {
 		return nil, err
 	}
 
-	h.clientsPolling = make(map[uint64]chan []byte)
 	return &h, nil
 }
 
@@ -120,21 +119,21 @@ func (h *Headscale) ExpireEphemeralNodes(milliSeconds int64) {
 func (h *Headscale) expireEphemeralNodesWorker() {
 	namespaces, err := h.ListNamespaces()
 	if err != nil {
-		log.Printf("Error listing namespaces: %s", err)
+		log.Error().Err(err).Msg("Error listing namespaces")
 		return
 	}
 	for _, ns := range *namespaces {
 		machines, err := h.ListMachinesInNamespace(ns.Name)
 		if err != nil {
-			log.Printf("Error listing machines in namespace %s: %s", ns.Name, err)
+			log.Error().Err(err).Str("namespace", ns.Name).Msg("Error listing machines in namespace")
 			return
 		}
 		for _, m := range *machines {
 			if m.AuthKey != nil && m.LastSeen != nil && m.AuthKey.Ephemeral && time.Now().After(m.LastSeen.Add(h.cfg.EphemeralNodeInactivityTimeout)) {
-				log.Printf("[%s] Ephemeral client removed from database\n", m.Name)
+				log.Info().Str("machine", m.Name).Msg("Ephemeral client removed from database")
 				err = h.db.Unscoped().Delete(m).Error
 				if err != nil {
-					log.Printf("[%s] 🤮 Cannot delete ephemeral machine from the database: %s", m.Name, err)
+					log.Error().Err(err).Str("machine", m.Name).Msg("🤮 Cannot delete ephemeral machine from the database")
 				}
 			}
 		}
@@ -158,6 +157,7 @@ func (h *Headscale) watchForKVUpdatesWorker() {
 // Serve launches a GIN server with the Headscale API
 func (h *Headscale) Serve() error {
 	r := gin.Default()
+	r.GET("/health", func(c *gin.Context) { c.JSON(200, gin.H{"healthy": "ok"}) })
 	r.GET("/key", h.KeyHandler)
 	r.GET("/register", h.RegisterWebAPI)
 	r.POST("/machine/:id/map", h.PollNetMapHandler)
@@ -168,7 +168,7 @@ func (h *Headscale) Serve() error {
 
 	if h.cfg.TLSLetsEncryptHostname != "" {
 		if !strings.HasPrefix(h.cfg.ServerURL, "https://") {
-			log.Println("WARNING: listening with TLS but ServerURL does not start with https://")
+			log.Warn().Msg("Listening with TLS but ServerURL does not start with https://")
 		}
 
 		m := autocert.Manager{
@@ -191,7 +191,10 @@ func (h *Headscale) Serve() error {
 			// port 80 for the certificate validation in addition to the headscale
 			// service, which can be configured to run on any other port.
 			go func() {
-				log.Fatal(http.ListenAndServe(h.cfg.TLSLetsEncryptListen, m.HTTPHandler(http.HandlerFunc(h.redirect))))
+
+				log.Fatal().
+					Err(http.ListenAndServe(h.cfg.TLSLetsEncryptListen, m.HTTPHandler(http.HandlerFunc(h.redirect)))).
+					Msg("failed to set up a HTTP server")
 			}()
 			err = s.ListenAndServeTLS("", "")
 		} else {
@@ -199,12 +202,12 @@ func (h *Headscale) Serve() error {
 		}
 	} else if h.cfg.TLSCertPath == "" {
 		if !strings.HasPrefix(h.cfg.ServerURL, "http://") {
-			log.Println("WARNING: listening without TLS but ServerURL does not start with http://")
+			log.Warn().Msg("Listening without TLS but ServerURL does not start with http://")
 		}
 		err = r.Run(h.cfg.Addr)
 	} else {
 		if !strings.HasPrefix(h.cfg.ServerURL, "https://") {
-			log.Println("WARNING: listening with TLS but ServerURL does not start with https://")
+			log.Warn().Msg("Listening with TLS but ServerURL does not start with https://")
 		}
 		err = r.RunTLS(h.cfg.Addr, h.cfg.TLSCertPath, h.cfg.TLSKeyPath)
 	}

cmd/headscale/cli/preauthkeys.go

@@ -19,6 +19,7 @@ func init() {
 	}
 	preauthkeysCmd.AddCommand(listPreAuthKeys)
 	preauthkeysCmd.AddCommand(createPreAuthKeyCmd)
+	preauthkeysCmd.AddCommand(expirePreAuthKeyCmd)
 	createPreAuthKeyCmd.PersistentFlags().Bool("reusable", false, "Make the preauthkey reusable")
 	createPreAuthKeyCmd.PersistentFlags().Bool("ephemeral", false, "Preauthkey for ephemeral nodes")
 	createPreAuthKeyCmd.Flags().StringP("expiration", "e", "", "Human-readable expiration of the key (30m, 24h, 365d...)")
@@ -116,6 +117,45 @@ var createPreAuthKeyCmd = &cobra.Command{
 			fmt.Println(err)
 			return
 		}
-		fmt.Printf("Key: %s\n", k.Key)
+		fmt.Printf("%s\n", k.Key)
+	},
+}
+
+var expirePreAuthKeyCmd = &cobra.Command{
+	Use:   "expire",
+	Short: "Expire a preauthkey",
+	Args: func(cmd *cobra.Command, args []string) error {
+		if len(args) < 1 {
+			return fmt.Errorf("missing parameters")
+		}
+		return nil
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		n, err := cmd.Flags().GetString("namespace")
+		if err != nil {
+			log.Fatalf("Error getting namespace: %s", err)
+		}
+		o, _ := cmd.Flags().GetString("output")
+
+		h, err := getHeadscaleApp()
+		if err != nil {
+			log.Fatalf("Error initializing: %s", err)
+		}
+
+		k, err := h.GetPreAuthKey(n, args[0])
+		if err != nil {
+			log.Fatalf("Error getting the key: %s", err)
+		}
+
+		err = h.MarkExpirePreAuthKey(k)
+		if strings.HasPrefix(o, "json") {
+			JsonOutput(k, err, o)
+			return
+		}
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		fmt.Println("Expired")
 	},
 }

cmd/headscale/cli/root.go

@@ -2,8 +2,9 @@ package cli
 
 import (
 	"fmt"
-	"github.com/spf13/cobra"
 	"os"
+
+	"github.com/spf13/cobra"
 )
 
 func init() {
@@ -16,8 +17,7 @@ var rootCmd = &cobra.Command{
 	Long: `
 headscale is an open source implementation of the Tailscale control server
 
-Juan Font Alonso <juanfontalonso@gmail.com> - 2021
-https://gitlab.com/juanfont/headscale`,
+https://github.com/juanfont/headscale`,
 }
 
 func Execute() {

cmd/headscale/cli/utils.go

@@ -5,13 +5,13 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/juanfont/headscale"
+	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
 	"gopkg.in/yaml.v2"
 	"inet.af/netaddr"
@@ -39,6 +39,8 @@ func LoadConfig(path string) error {
 
 	viper.SetDefault("ip_prefix", "100.64.0.0/10")
 
+	viper.SetDefault("log_level", "debug")
+
 	err := viper.ReadInConfig()
 	if err != nil {
 		return fmt.Errorf("Fatal error reading config file: %s \n", err)
@@ -52,7 +54,8 @@ func LoadConfig(path string) error {
 
 	if (viper.GetString("tls_letsencrypt_hostname") != "") && (viper.GetString("tls_letsencrypt_challenge_type") == "TLS-ALPN-01") && (!strings.HasSuffix(viper.GetString("listen_addr"), ":443")) {
 		// this is only a warning because there could be something sitting in front of headscale that redirects the traffic (e.g. an iptables rule)
-		log.Println("Warning: when using tls_letsencrypt_hostname with TLS-ALPN-01 as challenge type, headscale must be reachable on port 443, i.e. listen_addr should probably end in :443")
+		log.Warn().
+			Msg("Warning: when using tls_letsencrypt_hostname with TLS-ALPN-01 as challenge type, headscale must be reachable on port 443, i.e. listen_addr should probably end in :443")
 	}
 
 	if (viper.GetString("tls_letsencrypt_challenge_type") != "HTTP-01") && (viper.GetString("tls_letsencrypt_challenge_type") != "TLS-ALPN-01") {
@@ -82,9 +85,13 @@ func absPath(path string) string {
 }
 
 func getHeadscaleApp() (*headscale.Headscale, error) {
-	derpMap, err := loadDerpMap(absPath(viper.GetString("derp_map_path")))
+	derpPath := absPath(viper.GetString("derp_map_path"))
+	derpMap, err := loadDerpMap(derpPath)
 	if err != nil {
-		log.Printf("Could not load DERP servers map file: %s", err)
+		log.Error().
+			Str("path", derpPath).
+			Err(err).
+			Msg("Could not load DERP servers map file")
 	}
 
 	// Minimum inactivity time out is keepalive timeout (60s) plus a few seconds
@@ -129,9 +136,13 @@ func getHeadscaleApp() (*headscale.Headscale, error) {
 	// We are doing this here, as in the future could be cool to have it also hot-reload
 
 	if viper.GetString("acl_policy_path") != "" {
-		err = h.LoadACLPolicy(absPath(viper.GetString("acl_policy_path")))
+		aclPath := absPath(viper.GetString("acl_policy_path"))
+		err = h.LoadACLPolicy(aclPath)
 		if err != nil {
-			log.Printf("Could not load the ACL policy: %s", err)
+			log.Error().
+				Str("path", aclPath).
+				Err(err).
+				Msg("Could not load the ACL policy")
 		}
 	}
 
@@ -161,24 +172,24 @@ func JsonOutput(result interface{}, errResult error, outputFormat string) {
 		if errResult != nil {
 			j, err = json.MarshalIndent(ErrorOutput{errResult.Error()}, "", "\t")
 			if err != nil {
-				log.Fatalln(err)
+				log.Fatal().Err(err)
 			}
 		} else {
 			j, err = json.MarshalIndent(result, "", "\t")
 			if err != nil {
-				log.Fatalln(err)
+				log.Fatal().Err(err)
 			}
 		}
 	case "json-line":
 		if errResult != nil {
 			j, err = json.Marshal(ErrorOutput{errResult.Error()})
 			if err != nil {
-				log.Fatalln(err)
+				log.Fatal().Err(err)
 			}
 		} else {
 			j, err = json.Marshal(result)
 			if err != nil {
-				log.Fatalln(err)
+				log.Fatal().Err(err)
 			}
 		}
 	}

cmd/headscale/headscale.go

@@ -1,15 +1,64 @@
 package main
 
 import (
-	"log"
+	"os"
+	"time"
 
+	"github.com/efekarakus/termcolor"
 	"github.com/juanfont/headscale/cmd/headscale/cli"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/viper"
 )
 
 func main() {
+	var colors bool
+	switch l := termcolor.SupportLevel(os.Stderr); l {
+	case termcolor.Level16M:
+		colors = true
+	case termcolor.Level256:
+		colors = true
+	case termcolor.LevelBasic:
+		colors = true
+	default:
+		// no color, return text as is.
+		log.Trace().Msg("Colors are not supported, disabling")
+		colors = false
+	}
+
+	// Adhere to no-color.org manifesto of allowing users to
+	// turn off color in cli/services
+	if _, noColorIsSet := os.LookupEnv("NO_COLOR"); noColorIsSet {
+		log.Trace().Msg("NO_COLOR is set, disabling colors")
+		colors = false
+	}
+
+	zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
+	log.Logger = log.Output(zerolog.ConsoleWriter{
+		Out:        os.Stdout,
+		TimeFormat: time.RFC3339,
+		NoColor:    !colors,
+	})
+
 	err := cli.LoadConfig("")
 	if err != nil {
-		log.Fatalf(err.Error())
+		log.Fatal().Err(err)
+	}
+
+	logLevel := viper.GetString("log_level")
+	switch logLevel {
+	case "trace":
+		zerolog.SetGlobalLevel(zerolog.TraceLevel)
+	case "debug":
+		zerolog.SetGlobalLevel(zerolog.DebugLevel)
+	case "info":
+		zerolog.SetGlobalLevel(zerolog.InfoLevel)
+	case "warn":
+		zerolog.SetGlobalLevel(zerolog.WarnLevel)
+	case "error":
+		zerolog.SetGlobalLevel(zerolog.ErrorLevel)
+	default:
+		zerolog.SetGlobalLevel(zerolog.DebugLevel)
 	}
 
 	cli.Execute()

go.mod

@@ -4,15 +4,29 @@ go 1.16
 
 require (
 	github.com/AlecAivazis/survey/v2 v2.0.5
+	github.com/Microsoft/go-winio v0.5.0 // indirect
+	github.com/cenkalti/backoff/v3 v3.0.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.1 // indirect
+	github.com/containerd/continuity v0.1.0 // indirect
+	github.com/docker/cli v20.10.8+incompatible // indirect
+	github.com/docker/docker v20.10.8+incompatible // indirect
+	github.com/efekarakus/termcolor v1.0.1 // indirect
 	github.com/gin-gonic/gin v1.7.2
 	github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b
 	github.com/klauspost/compress v1.13.1
 	github.com/lib/pq v1.10.2 // indirect
 	github.com/mattn/go-sqlite3 v1.14.7 // indirect
+	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
+	github.com/opencontainers/runc v1.0.1 // indirect
+	github.com/ory/dockertest/v3 v3.7.0 // indirect
+	github.com/rs/zerolog v1.23.0 // indirect
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/viper v1.8.1
 	github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e
+	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
+	golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
 	gopkg.in/yaml.v2 v2.4.0
 	gorm.io/datatypes v1.0.1

go.sum

@@ -1,4 +1,5 @@
 4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a/go.mod h1:wfdC5ZjKSPr7CybKEcgJhUOgeAQW1+7WcyK8OvUilfo=
+bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
@@ -40,6 +41,9 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/AlecAivazis/survey/v2 v2.0.5 h1:xpZp+Q55wi5C7Iaze+40onHnEkex1jSc34CltJjOoPM=
 github.com/AlecAivazis/survey/v2 v2.0.5/go.mod h1:WYBhg6f0y/fNYUuesWQc0PKbJcEliGcYHB9sNT3Bg74=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs=
@@ -55,8 +59,12 @@ github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuN
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
 github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
 github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
+github.com/Microsoft/go-winio v0.5.0 h1:Elr9Wn+sGKPlkaBvwu4mTrxtmOp3F3yV9qhaHbXGjwU=
+github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8 h1:xzYJEypr/85nBpB11F9br+3HUrpgb+fcm5iADzXXYEw=
 github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8/go.mod h1:oX5x61PbNXchhh0oikYAH+4Pcfw5LKv21+Jnpr6r6Pc=
+github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
+github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
@@ -79,6 +87,7 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
@@ -91,20 +100,28 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
 github.com/bombsimon/wsl/v3 v3.1.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oDpT4efm8tSYHXV5tHSdRvBet/b/QzxZ+XyyPehvm3A=
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs=
+github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
+github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ=
+github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
+github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
 github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
@@ -114,7 +131,12 @@ github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
+github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ=
+github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
+github.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8=
+github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -130,6 +152,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/daixiang0/gci v0.2.4/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD/hfR4=
 github.com/daixiang0/gci v0.2.7/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -140,12 +164,25 @@ github.com/denisenkom/go-mssqldb v0.9.0 h1:RSohk2RsiZqLZ0zCjtfn3S4Gp4exhpBWHyQ7D
 github.com/denisenkom/go-mssqldb v0.9.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/docker/cli v20.10.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v20.10.8+incompatible h1:/zO/6y9IOpcehE49yMRTV9ea0nBpb8OeqSskXLNfH1E=
+github.com/docker/cli v20.10.8+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v20.10.8+incompatible h1:RVqD337BgQicVCzYrrlhLDWhq6OAD2PJDUg2LsEUvKM=
+github.com/docker/docker v20.10.8+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
+github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
+github.com/efekarakus/termcolor v1.0.1 h1:YAKFO3bnLrqZGTWyNLcYoSIAQFKVOmbqmDnwsU/znzg=
+github.com/efekarakus/termcolor v1.0.1/go.mod h1:AitrZNrE4nPO538fRsqf+p0WgLdAsGN5pUNrHEPsEMM=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
 github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -223,6 +260,7 @@ github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFG
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
@@ -312,6 +350,8 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/rpmpack v0.0.0-20201206194719-59e495f2b7e1/go.mod h1:+y9lKiqDhR4zkLl+V9h4q0rdyrYVsWWm6LLCQP33DIk=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
@@ -326,6 +366,7 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
 github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
@@ -374,6 +415,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
+github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
@@ -500,6 +543,7 @@ github.com/kunwardeep/paralleltest v1.0.2/go.mod h1:ZPqNm1fVHPllh5LPVujzbVz1JN2G
 github.com/kyoh86/exportloopref v0.1.8/go.mod h1:1tUcJeiioIs7VWe5gcOObrux3lb66+sBqGZrRkMwPgg=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
+github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
@@ -512,6 +556,7 @@ github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQ
 github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ=
 github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.4/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
@@ -532,6 +577,7 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
@@ -577,6 +623,10 @@ github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxd
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
+github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
+github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc=
+github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -585,6 +635,7 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8qUplsoSU4k=
 github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
+github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c=
 github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
@@ -611,6 +662,16 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
+github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
+github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
+github.com/opencontainers/runc v1.0.1 h1:G18PGckGdAm3yVQRWDVQ1rLSLntiniKJ0cNRT2Tm5gs=
+github.com/opencontainers/runc v1.0.1/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0=
+github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
 github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
 github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
 github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@@ -619,6 +680,8 @@ github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxS
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
+github.com/ory/dockertest/v3 v3.7.0 h1:Bijzonc69Ont3OU0a3TWKJ1Rzlh3TsDXP1JrTAkSmsM=
+github.com/ory/dockertest/v3 v3.7.0/go.mod h1:PvCCgnP7AfBZeVrzwiUTjZx/IUXlGLC1zQlUQrLIlUE=
 github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/getopt v1.1.0/go.mod h1:FxXoW1Re00sQG/+KIkuSqRL/LwQgSkv7uyac+STFsbk=
@@ -683,6 +746,8 @@ github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
+github.com/rs/zerolog v1.23.0 h1:UskrK+saS9P9Y789yNNulYKdARjPZuS35B8gJF2x60g=
+github.com/rs/zerolog v1.23.0/go.mod h1:6c7hFfxPOy7TacJc4Fcdi24/J0NKYGzjG8FWRI916Qo=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryancurrah/gomodguard v1.1.0/go.mod h1:4O8tr7hBODaGE6VIhfJDHcwzh5GUccKSJBU0UMXJFVM=
 github.com/ryanrolds/sqlclosecheck v0.3.0/go.mod h1:1gREqxyTGR3lVtpngyFo3hZAgk0KCtEdgEkHwDbigdA=
@@ -691,6 +756,7 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0
 github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b/go.mod h1:am+Fp8Bt506lA3Rk3QCmSqmYmLMnPDhdDUcosQCAx+I=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
 github.com/securego/gosec/v2 v2.5.0/go.mod h1:L/CDXVntIff5ypVHIkqPXbtRpJiNCh6c6Amn68jXDjo=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
@@ -708,6 +774,8 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
@@ -726,6 +794,7 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
 github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
 github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
@@ -736,6 +805,7 @@ github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44=
@@ -758,6 +828,7 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3/go.mod h1:2P+hpOwd53e7JMX/L4f3VXkv1G+33ES6IWZSrkIeWNs=
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
 github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2 h1:reREUgl2FG+o7YCsrZB8XLjnuKv5hEIWtnOdAbRAXZI=
@@ -775,6 +846,7 @@ github.com/tomarrell/wrapcheck v0.0.0-20200807122107-df9e8bcb914d/go.mod h1:yiFB
 github.com/tomarrell/wrapcheck v0.0.0-20201130113247-1683564d9756/go.mod h1:yiFB6fFoV7saXirUGfuK+cPtUh4NX/Hf5y2WC2lehu0=
 github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig=
 github.com/toqueteos/webbrowser v1.2.0/go.mod h1:XWoZq4cyp9WeUeak7w7LXRUQf1F1ATJMir8RTqb4ayM=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
@@ -789,10 +861,20 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
 github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
 github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD5KW4lOuSdPKzY=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
 github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -910,6 +992,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -947,6 +1030,8 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -990,6 +1075,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1000,6 +1086,8 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1020,7 +1108,9 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1046,11 +1136,14 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 h1:siQdpVirKtzPhKl3lZWozZraCFObP8S1v6PRp0bLrtU=
+golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY=
@@ -1090,6 +1183,7 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -1328,6 +1422,7 @@ gorm.io/gorm v1.21.6/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0=
 gorm.io/gorm v1.21.9/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0=
 gorm.io/gorm v1.21.11 h1:CxkXW6Cc+VIBlL8yJEHq+Co4RYXdSLiMKNvgoZPjLK4=
 gorm.io/gorm v1.21.11/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0=
+gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

integration_test.go

@@ -0,0 +1,246 @@
+// +build integration
+
+package headscale
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+	"inet.af/netaddr"
+
+	"gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&IntegrationSuite{})
+
+type IntegrationSuite struct{}
+
+var integrationTmpDir string
+var ih Headscale
+
+var pool dockertest.Pool
+var network dockertest.Network
+var headscale dockertest.Resource
+var tailscaleCount int = 10
+var tailscales map[string]dockertest.Resource
+
+func executeCommand(resource *dockertest.Resource, cmd []string) (string, error) {
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	exitCode, err := resource.Exec(
+		cmd,
+		dockertest.ExecOptions{
+			StdOut: &stdout,
+			StdErr: &stderr,
+		},
+	)
+	if err != nil {
+		return "", err
+	}
+
+	if exitCode != 0 {
+		fmt.Println("Command: ", cmd)
+		fmt.Println("stdout: ", stdout.String())
+		fmt.Println("stderr: ", stderr.String())
+		return "", fmt.Errorf("command failed with: %s", stderr.String())
+	}
+
+	return stdout.String(), nil
+}
+
+func dockerRestartPolicy(config *docker.HostConfig) {
+	// set AutoRemove to true so that stopped container goes away by itself
+	config.AutoRemove = true
+	config.RestartPolicy = docker.RestartPolicy{
+		Name: "no",
+	}
+}
+
+func (s *IntegrationSuite) SetUpSuite(c *check.C) {
+	var err error
+	h = Headscale{
+		dbType:   "sqlite3",
+		dbString: "integration_test_db.sqlite3",
+	}
+
+	if ppool, err := dockertest.NewPool(""); err == nil {
+		pool = *ppool
+	} else {
+		log.Fatalf("Could not connect to docker: %s", err)
+	}
+
+	if pnetwork, err := pool.CreateNetwork("headscale-test"); err == nil {
+		network = *pnetwork
+	} else {
+		log.Fatalf("Could not create network: %s", err)
+	}
+
+	headscaleBuildOptions := &dockertest.BuildOptions{
+		Dockerfile: "Dockerfile",
+		ContextDir: ".",
+	}
+
+	tailscaleBuildOptions := &dockertest.BuildOptions{
+		Dockerfile: "Dockerfile.tailscale",
+		ContextDir: ".",
+	}
+
+	currentPath, err := os.Getwd()
+	if err != nil {
+		log.Fatalf("Could not determine current path: %s", err)
+	}
+
+	headscaleOptions := &dockertest.RunOptions{
+		Name: "headscale",
+		Mounts: []string{
+			fmt.Sprintf("%s/integration_test/etc:/etc/headscale", currentPath),
+			fmt.Sprintf("%s/derp.yaml:/etc/headscale/derp.yaml", currentPath),
+		},
+		Networks: []*dockertest.Network{&network},
+		// Cmd: []string{"sleep", "3600"},
+		Cmd: []string{"headscale", "serve"},
+		PortBindings: map[docker.Port][]docker.PortBinding{
+			"8080/tcp": []docker.PortBinding{{HostPort: "8080"}},
+		},
+		Env: []string{},
+	}
+
+	fmt.Println("Creating headscale container")
+	if pheadscale, err := pool.BuildAndRunWithBuildOptions(headscaleBuildOptions, headscaleOptions, dockerRestartPolicy); err == nil {
+		headscale = *pheadscale
+	} else {
+		log.Fatalf("Could not start resource: %s", err)
+	}
+	fmt.Println("Created headscale container")
+
+	fmt.Println("Creating tailscale containers")
+	tailscales = make(map[string]dockertest.Resource)
+	for i := 0; i < tailscaleCount; i++ {
+		hostname := fmt.Sprintf("tailscale%d", i)
+		tailscaleOptions := &dockertest.RunOptions{
+			Name:     hostname,
+			Networks: []*dockertest.Network{&network},
+			// Make the container run until killed
+			// Cmd: []string{"sleep", "3600"},
+			Cmd: []string{"tailscaled", "--tun=userspace-networking", "--socks5-server=localhost:1055"},
+			Env: []string{},
+		}
+
+		if pts, err := pool.BuildAndRunWithBuildOptions(tailscaleBuildOptions, tailscaleOptions, dockerRestartPolicy); err == nil {
+			tailscales[hostname] = *pts
+		} else {
+			log.Fatalf("Could not start resource: %s", err)
+		}
+		fmt.Printf("Created %s container\n", hostname)
+	}
+
+	fmt.Println("Waiting for headscale to be ready")
+	hostEndpoint := fmt.Sprintf("localhost:%s", headscale.GetPort("8080/tcp"))
+
+	if err := pool.Retry(func() error {
+		url := fmt.Sprintf("http://%s/health", hostEndpoint)
+		resp, err := http.Get(url)
+		if err != nil {
+			return err
+		}
+		if resp.StatusCode != http.StatusOK {
+			return fmt.Errorf("status code not OK")
+		}
+		return nil
+	}); err != nil {
+		log.Fatalf("Could not connect to docker: %s", err)
+	}
+	fmt.Println("headscale container is ready")
+
+	fmt.Println("Creating headscale namespace")
+	result, err := executeCommand(
+		&headscale,
+		[]string{"headscale", "namespaces", "create", "test"},
+	)
+	c.Assert(err, check.IsNil)
+
+	fmt.Println("Creating pre auth key")
+	authKey, err := executeCommand(
+		&headscale,
+		[]string{"headscale", "-n", "test", "preauthkeys", "create", "--reusable", "--expiration", "24h"},
+	)
+	c.Assert(err, check.IsNil)
+
+	headscaleEndpoint := fmt.Sprintf("http://headscale:%s", headscale.GetPort("8080/tcp"))
+
+	fmt.Printf("Joining tailscale containers to headscale at %s\n", headscaleEndpoint)
+	for hostname, tailscale := range tailscales {
+		command := []string{"tailscale", "up", "-login-server", headscaleEndpoint, "--authkey", strings.TrimSuffix(authKey, "\n"), "--hostname", hostname}
+
+		fmt.Println("Join command:", command)
+		fmt.Printf("Running join command for %s\n", hostname)
+		result, err = executeCommand(
+			&tailscale,
+			command,
+		)
+		fmt.Println("tailscale result: ", result)
+		c.Assert(err, check.IsNil)
+		fmt.Printf("%s joined\n", hostname)
+	}
+}
+
+func (s *IntegrationSuite) TearDownSuite(c *check.C) {
+	if err := pool.Purge(&headscale); err != nil {
+		log.Printf("Could not purge resource: %s\n", err)
+	}
+
+	for _, tailscale := range tailscales {
+		if err := pool.Purge(&tailscale); err != nil {
+			log.Printf("Could not purge resource: %s\n", err)
+		}
+	}
+
+	if err := network.Close(); err != nil {
+		log.Printf("Could not close network: %s\n", err)
+	}
+}
+
+func (s *IntegrationSuite) TestListNodes(c *check.C) {
+	fmt.Println("Listing nodes")
+	result, err := executeCommand(
+		&headscale,
+		[]string{"headscale", "-n", "test", "nodes", "list"},
+	)
+	c.Assert(err, check.IsNil)
+
+	for hostname, _ := range tailscales {
+		c.Assert(strings.Contains(result, hostname), check.Equals, true)
+	}
+}
+
+func (s *IntegrationSuite) TestGetIpAddresses(c *check.C) {
+	ipPrefix := netaddr.MustParseIPPrefix("100.64.0.0/10")
+	ips := make(map[string]netaddr.IP)
+	for hostname, tailscale := range tailscales {
+		command := []string{"tailscale", "ip"}
+
+		result, err := executeCommand(
+			&tailscale,
+			command,
+		)
+		c.Assert(err, check.IsNil)
+
+		ip, err := netaddr.ParseIP(strings.TrimSuffix(result, "\n"))
+		c.Assert(err, check.IsNil)
+
+		fmt.Printf("IP for %s: %s", hostname, result)
+
+		// c.Assert(ip.Valid(), check.IsTrue)
+		c.Assert(ip.Is4(), check.Equals, true)
+		c.Assert(ipPrefix.Contains(ip), check.Equals, true)
+
+		ips[hostname] = ip
+	}
+}

integration_test/.gitignore

@@ -0,0 +1,3 @@
+derp.yaml
+*.sqlite
+*.sqlite3

integration_test/etc/config.json

@@ -0,0 +1,11 @@
+{
+  "server_url": "http://headscale:8080",
+  "listen_addr": "0.0.0.0:8080",
+  "private_key_path": "private.key",
+  "derp_map_path": "derp.yaml",
+  "ephemeral_node_inactivity_timeout": "30m",
+  "db_type": "sqlite3",
+  "db_path": "/tmp/integration_test_db.sqlite3",
+  "acl_policy_path": "",
+  "log_level": "trace"
+}

integration_test/etc/private.key

@@ -0,0 +1 @@
+SEmQwCu+tGywQWEUsf93TpTRUvlB7WhnCdHgWrSXjEA=

k8s/README.md

@@ -65,7 +65,6 @@ tasks like creating namespaces, authkeys, etc.
 
 headscale is an open source implementation of the Tailscale control server
 
-Juan Font Alonso <juanfontalonso@gmail.com> - 2021
 https://gitlab.com/juanfont/headscale
 
 Usage:

machine.go

@@ -3,11 +3,12 @@ package headscale
 import (
 	"encoding/json"
 	"fmt"
-	"log"
 	"sort"
 	"strconv"
 	"time"
 
+	"github.com/rs/zerolog/log"
+
 	"gorm.io/datatypes"
 	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
@@ -71,6 +72,10 @@ func (m Machine) toNode() (*tailcfg.Node, error) {
 	addrs := []netaddr.IPPrefix{}
 	ip, err := netaddr.ParseIPPrefix(fmt.Sprintf("%s/32", m.IPAddress))
 	if err != nil {
+		log.Trace().
+			Str("func", "toNode").
+			Str("ip", m.IPAddress).
+			Msgf("Failed to parse IP Prefix from IP: %s", m.IPAddress)
 		return nil, err
 	}
 	addrs = append(addrs, ip) // missing the ipv6 ?
@@ -157,7 +162,7 @@ func (h *Headscale) getPeers(m Machine) (*[]*tailcfg.Node, error) {
 	machines := []Machine{}
 	if err := h.db.Where("namespace_id = ? AND machine_key <> ? AND registered",
 		m.NamespaceID, m.MachineKey).Find(&machines).Error; err != nil {
-		log.Printf("Error accessing db: %s", err)
+		log.Error().Err(err).Msg("Error accessing db")
 		return nil, err
 	}
 

namespaces.go

@@ -4,9 +4,9 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"log"
 	"time"
 
+	"github.com/rs/zerolog/log"
 	"gorm.io/gorm"
 	"tailscale.com/tailcfg"
 )
@@ -33,7 +33,10 @@ func (h *Headscale) CreateNamespace(name string) (*Namespace, error) {
 	}
 	n.Name = name
 	if err := h.db.Create(&n).Error; err != nil {
-		log.Printf("Could not create row: %s", err)
+		log.Error().
+			Str("func", "CreateNamespace").
+			Err(err).
+			Msg("Could not create row")
 		return nil, err
 	}
 	return &n, nil
@@ -133,7 +136,10 @@ func (h *Headscale) RequestMapUpdates(namespaceID uint) error {
 	names = append(names, namespace.Name)
 	data, err := json.Marshal(names)
 	if err != nil {
-		log.Printf("Could not marshal namespaces_pending_updates: %s", err)
+		log.Error().
+			Str("func", "RequestMapUpdates").
+			Err(err).
+			Msg("Could not marshal namespaces_pending_updates")
 		return err
 	}
 	return h.setValue("namespaces_pending_updates", string(data))
@@ -154,24 +160,34 @@ func (h *Headscale) checkForNamespacesPendingUpdates() {
 		return
 	}
 	for _, name := range names {
-		log.Printf("Sending updates to nodes in namespace %s", name)
+		log.Trace().
+			Str("func", "RequestMapUpdates").
+			Str("machine", name).
+			Msg("Sending updates to nodes in namespace")
 		machines, err := h.ListMachinesInNamespace(name)
 		if err != nil {
 			continue
 		}
 		for _, m := range *machines {
 			peers, _ := h.getPeers(m)
-			h.pollMu.Lock()
 			for _, p := range *peers {
-				pUp, ok := h.clientsPolling[uint64(p.ID)]
+				pUp, ok := h.clientsPolling.Load(uint64(p.ID))
 				if ok {
-					log.Printf("[%s] Notifying peer %s (%s)", m.Name, p.Name, p.Addresses[0])
-					pUp <- []byte{}
+					log.Info().
+						Str("func", "checkForNamespacesPendingUpdates").
+						Str("machine", m.Name).
+						Str("peer", m.Name).
+						Str("address", p.Addresses[0].String()).
+						Msgf("Notifying peer %s (%s)", p.Name, p.Addresses[0])
+					pUp.(chan []byte) <- []byte{}
 				} else {
-					log.Printf("[%s] Peer %s does not appear to be polling", m.Name, p.Name)
+					log.Info().
+						Str("func", "checkForNamespacesPendingUpdates").
+						Str("machine", m.Name).
+						Str("peer", m.Name).
+						Msgf("Peer %s does not appear to be polling", p.Name)
 				}
 			}
-			h.pollMu.Unlock()
 		}
 	}
 	newV, err := h.getValue("namespaces_pending_updates")
@@ -181,7 +197,10 @@ func (h *Headscale) checkForNamespacesPendingUpdates() {
 	if v == newV { // only clear when no changes, so we notified everybody
 		err = h.setValue("namespaces_pending_updates", "")
 		if err != nil {
-			log.Printf("Could not save to KV: %s", err)
+			log.Error().
+				Str("func", "checkForNamespacesPendingUpdates").
+				Err(err).
+				Msg("Could not save to KV")
 			return
 		}
 	}

preauth_keys.go

@@ -67,6 +67,28 @@ func (h *Headscale) GetPreAuthKeys(namespaceName string) (*[]PreAuthKey, error)
 	return &keys, nil
 }
 
+// GetPreAuthKey returns a PreAuthKey for a given key
+func (h *Headscale) GetPreAuthKey(namespace string, key string) (*PreAuthKey, error) {
+	pak, err := h.checkKeyValidity(key)
+	if err != nil {
+		return nil, err
+	}
+
+	if pak.Namespace.Name != namespace {
+		return nil, errors.New("Namespace mismatch")
+	}
+
+	return pak, nil
+}
+
+// MarkExpirePreAuthKey marks a PreAuthKey as expired
+func (h *Headscale) MarkExpirePreAuthKey(k *PreAuthKey) error {
+	if err := h.db.Model(&k).Update("Expiration", time.Now()).Error; err != nil {
+		return err
+	}
+	return nil
+}
+
 // checkKeyValidity does the heavy lifting for validation of the PreAuthKey coming from a node
 // If returns no error and a PreAuthKey, it can be used
 func (h *Headscale) checkKeyValidity(k string) (*PreAuthKey, error) {

preauth_keys_test.go

@@ -163,3 +163,20 @@ func (*Suite) TestEphemeralKey(c *check.C) {
 	_, err = h.GetMachine("test7", "testest")
 	c.Assert(err, check.NotNil)
 }
+
+func (*Suite) TestExpirePreauthKey(c *check.C) {
+	n, err := h.CreateNamespace("test3")
+	c.Assert(err, check.IsNil)
+
+	pak, err := h.CreatePreAuthKey(n.Name, true, false, nil)
+	c.Assert(err, check.IsNil)
+	c.Assert(pak.Expiration, check.IsNil)
+
+	err = h.MarkExpirePreAuthKey(pak)
+	c.Assert(err, check.IsNil)
+	c.Assert(pak.Expiration, check.NotNil)
+
+	p, err := h.checkKeyValidity(pak.Key)
+	c.Assert(err, check.Equals, errorAuthKeyExpired)
+	c.Assert(p, check.IsNil)
+}

routes.go

@@ -47,16 +47,14 @@ func (h *Headscale) EnableNodeRoute(namespace string, nodeName string, routeStr
 
 			// THIS IS COMPLETELY USELESS.
 			// The peers map is stored in memory in the server process.
-			// Definetely not accessible from the CLI tool.
+			// Definitely not accessible from the CLI tool.
 			// We need RPC to the server - or some kind of 'needsUpdate' field in the DB
 			peers, _ := h.getPeers(*m)
-			h.pollMu.Lock()
 			for _, p := range *peers {
-				if pUp, ok := h.clientsPolling[uint64(p.ID)]; ok {
-					pUp <- []byte{}
+				if pUp, ok := h.clientsPolling.Load(uint64(p.ID)); ok {
+					pUp.(chan []byte) <- []byte{}
 				}
 			}
-			h.pollMu.Unlock()
 			return &rIP, nil
 		}
 	}