Merge pull request #171 from juanfont/force-flag

Added --force flag on node delete

.github/workflows/build.yml

@@ -0,0 +1,36 @@
+name: Build
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: "1.16.3"
+
+      - name: Install dependencies
+        run: |
+          go version
+          go install golang.org/x/lint/golint@latest
+          sudo apt update
+          sudo apt install -y make
+
+      - name: Run lint
+        run: make build
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: headscale-linux
+          path: headscale

.goreleaser.yml

@@ -19,7 +19,8 @@ builds:
     flags:
       - -mod=readonly
     ldflags:
-      - -s -w -X main.version={{.Version}}
+      - -s -w -X github.com/juanfont/headscale/cmd/headscale/cli.Version=v{{.Version}}
+
   - id: linux-armhf
     main: ./cmd/headscale/headscale.go
     mod_timestamp: '{{ .CommitTimestamp }}'
@@ -39,7 +40,7 @@ builds:
     flags:
       - -mod=readonly
     ldflags:
-      - -s -w -X main.version={{.Version}}
+      - -s -w -X github.com/juanfont/headscale/cmd/headscale/cli.Version=v{{.Version}}
 
 
   - id: linux-amd64
@@ -49,11 +50,20 @@ builds:
       - linux
     goarch:
       - amd64
-    goarm:
-      - 6
-      - 7
     main: ./cmd/headscale/headscale.go
     mod_timestamp: '{{ .CommitTimestamp }}'
+    ldflags:
+      - -s -w -X github.com/juanfont/headscale/cmd/headscale/cli.Version=v{{.Version}}
+
+  - id: linux-arm64
+    goos:
+      - linux
+    goarch:
+      - arm64
+    main: ./cmd/headscale/headscale.go
+    mod_timestamp: '{{ .CommitTimestamp }}'
+    ldflags:
+      - -s -w -X github.com/juanfont/headscale/cmd/headscale/cli.Version=v{{.Version}}
 
 archives:
   - id: golang-cross
@@ -61,6 +71,7 @@ archives:
       - darwin-amd64
       - linux-armhf
       - linux-amd64
+      - linux-arm64
     name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
     format: binary
 

Makefile

@@ -2,7 +2,7 @@
 version = $(shell ./scripts/version-at-commit.sh)
 
 build:
-	go build -ldflags "-s -w -X github.com/juanfont/headscale/cmd/headscale/cli.version=$(version)" cmd/headscale/headscale.go
+	go build -ldflags "-s -w -X github.com/juanfont/headscale/cmd/headscale/cli.Version=$(version)" cmd/headscale/headscale.go
 
 dev: lint test build
 

README.md

@@ -1,9 +1,11 @@
-# Headscale
+# headscale
 
-[![Join the chat at https://gitter.im/headscale-dev/community](https://badges.gitter.im/headscale-dev/community.svg)](https://gitter.im/headscale-dev/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) ![ci](https://github.com/juanfont/headscale/actions/workflows/test.yml/badge.svg)
+![ci](https://github.com/juanfont/headscale/actions/workflows/test.yml/badge.svg)
 
 An open source, self-hosted implementation of the Tailscale coordination server.
 
+Join our [Discord](https://discord.gg/XcQxk2VHjx) server for a chat.
+
 ## Overview
 
 Tailscale is [a modern VPN](https://tailscale.com/) built on top of [Wireguard](https://www.wireguard.com/). It [works like an overlay network](https://tailscale.com/blog/how-tailscale-works/) between the computers of your networks - using all kinds of [NAT traversal sorcery](https://tailscale.com/blog/how-nat-traversal-works/).
@@ -22,21 +24,29 @@ Headscale implements this coordination server.
 - [x] Namespace support (~equivalent to multi-user in Tailscale.com)
 - [x] Routing (advertise & accept, including exit nodes)
 - [x] Node registration via pre-auth keys (including reusable keys, and ephemeral node support)
-- [X] JSON-formatted output
-- [X] ACLs
-- [X] Taildrop (File Sharing)
-- [X] Support for alternative IP ranges in the tailnets (default Tailscale's 100.64.0.0/10)
-- [X] DNS (passing DNS servers to nodes)
-- [X] Share nodes between ~~users~~ namespaces
-- [ ] MagicDNS / Smart DNS
+- [x] JSON-formatted output
+- [x] ACLs
+- [x] Taildrop (File Sharing)
+- [x] Support for alternative IP ranges in the tailnets (default Tailscale's 100.64.0.0/10)
+- [x] DNS (passing DNS servers to nodes)
+- [x] Share nodes between ~~users~~ namespaces
+- [x] MagicDNS (see `docs/`)
 
+## Client OS support
+
+| OS      | Supports headscale                                                                                                |
+| ------- | ----------------------------------------------------------------------------------------------------------------- |
+| Linux   | Yes                                                                                                               |
+| OpenBSD | Yes                                                                                                               |
+| macOS   | Yes (see `/apple` on your headscale for more information)                                                         |
+| Windows | Yes                                                                                                               |
+| Android | [You need to compile the client yourself](https://github.com/juanfont/headscale/issues/58#issuecomment-885255270) |
+| iOS     | Not yet                                                                                                           |
 
 ## Roadmap 🤷
 
 Suggestions/PRs welcomed!
 
-
-
 ## Running it
 
 1. Download the Headscale binary https://github.com/juanfont/headscale/releases, and place it somewhere in your PATH or use the docker container
@@ -44,109 +54,125 @@ Suggestions/PRs welcomed!
    ```shell
    docker pull headscale/headscale:x.x.x
    ```
-  <!--
-   or
-   ```shell
-   docker pull ghrc.io/juanfont/headscale:x.x.x
-   ``` -->
+
+   <!--
+    or
+    ```shell
+    docker pull ghrc.io/juanfont/headscale:x.x.x
+    ``` -->
 
 2. (Optional, you can also use SQLite) Get yourself a PostgreSQL DB running
 
-    ```shell
-    docker run --name headscale -e POSTGRES_DB=headscale -e \
-      POSTGRES_USER=foo -e POSTGRES_PASSWORD=bar -p 5432:5432 -d postgres
-    ```
+   ```shell
+   docker run --name headscale -e POSTGRES_DB=headscale -e \
+     POSTGRES_USER=foo -e POSTGRES_PASSWORD=bar -p 5432:5432 -d postgres
+   ```
 
 3. Set some stuff up (headscale Wireguard keys & the config.json file)
-    ```shell
-    wg genkey > private.key
-    wg pubkey < private.key > public.key  # not needed
 
-    # Postgres
-    cp config.json.postgres.example config.json
-    # or
-    # SQLite
-    cp config.json.sqlite.example config.json
-    ```
+   ```shell
+   wg genkey > private.key
+   wg pubkey < private.key > public.key  # not needed
+
+   # Postgres
+   cp config.json.postgres.example config.json
+   # or
+   # SQLite
+   cp config.json.sqlite.example config.json
+   ```
 
 4. Create a namespace (a namespace is a 'tailnet', a group of Tailscale nodes that can talk to each other)
-    ```shell
-    headscale namespaces create myfirstnamespace
-    ```
-    or docker:
 
-    the db.sqlite mount is only needed if you use sqlite
-    ```shell
-    touch db.sqlite
-    docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml -v $(pwd)/db.sqlite:/db.sqlite -p 127.0.0.1:8000:8000 headscale/headscale:x.x.x headscale namespaces create myfirstnamespace
-    ```
-    or if your server is already running in docker:
-    ```shell
-    docker exec <container_name> headscale create myfirstnamespace
-    ```
+   ```shell
+   headscale namespaces create myfirstnamespace
+   ```
+
+   or docker:
+
+   the db.sqlite mount is only needed if you use sqlite
+
+   ```shell
+   touch db.sqlite
+   docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml -v $(pwd)/db.sqlite:/db.sqlite -p 127.0.0.1:8080:8080 headscale/headscale:x.x.x headscale namespaces create myfirstnamespace
+   ```
+
+   or if your server is already running in docker:
+
+   ```shell
+   docker exec <container_name> headscale create myfirstnamespace
+   ```
 
 5. Run the server
-    ```shell
-    headscale serve
-    ```
-    or docker:
 
-    the db.sqlite mount is only needed if you use sqlite
-    ```shell
-    docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml -v $(pwd)/db.sqlite:/db.sqlite -p 127.0.0.1:8000:8000 headscale/headscale:x.x.x headscale serve
-    ```
+   ```shell
+   headscale serve
+   ```
+
+   or docker:
+
+   the db.sqlite mount is only needed if you use sqlite
+
+   ```shell
+   docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml -v $(pwd)/db.sqlite:/db.sqlite -p 127.0.0.1:8080:8080 headscale/headscale:x.x.x headscale serve
+   ```
 
 6. If you used tailscale.com before in your nodes, make sure you clear the tailscald data folder
-    ```shell
-    systemctl stop tailscaled
-    rm -fr /var/lib/tailscale
-    systemctl start tailscaled
-    ```
+
+   ```shell
+   systemctl stop tailscaled
+   rm -fr /var/lib/tailscale
+   systemctl start tailscaled
+   ```
 
 7. Add your first machine
-    ```shell
-    tailscale up -login-server YOUR_HEADSCALE_URL
-    ```
+
+   ```shell
+   tailscale up --login-server YOUR_HEADSCALE_URL
+   ```
 
 8. Navigate to the URL you will get with `tailscale up`, where you'll find your machine key.
 
 9. In the server, register your machine to a namespace with the CLI
-    ```shell
-    headscale -n myfirstnamespace node register YOURMACHINEKEY
-    ```
-    or docker:
-    ```shell
-    docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml headscale/headscale:x.x.x headscale -n myfirstnamespace node register YOURMACHINEKEY
-    ```
-    or if your server is already running in docker:
-    ```shell
-    docker exec <container_name> headscale -n myfistnamespace node register YOURMACHINEKEY
-    ```
+   ```shell
+   headscale -n myfirstnamespace nodes register YOURMACHINEKEY
+   ```
+   or docker:
+   ```shell
+   docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v $(pwd)/derp.yaml:/derp.yaml headscale/headscale:x.x.x headscale -n myfirstnamespace nodes register YOURMACHINEKEY
+   ```
+   or if your server is already running in docker:
+   ```shell
+   docker exec <container_name> headscale -n myfirstnamespace nodes register YOURMACHINEKEY
+   ```
 
 Alternatively, you can use Auth Keys to register your machines:
 
 1. Create an authkey
-    ```shell
-    headscale -n myfirstnamespace preauthkeys create --reusable --expiration 24h
-    ```
-    or docker:
-    ```shell
-    docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v$(pwd)/derp.yaml:/derp.yaml -v $(pwd)/db.sqlite:/db.sqlite headscale/headscale:x.x.x headscale -n myfirstnamespace preauthkeys create --reusable --expiration 24h
-    ```
-    or if your server is already running in docker:
-    ```shell
-    docker exec <container_name> headscale -n myfirstnamespace preauthkeys create --reusable --expiration 24h
-    ```
+
+   ```shell
+   headscale -n myfirstnamespace preauthkeys create --reusable --expiration 24h
+   ```
+
+   or docker:
+
+   ```shell
+   docker run -v $(pwd)/private.key:/private.key -v $(pwd)/config.json:/config.json -v$(pwd)/derp.yaml:/derp.yaml -v $(pwd)/db.sqlite:/db.sqlite headscale/headscale:x.x.x headscale -n myfirstnamespace preauthkeys create --reusable --expiration 24h
+   ```
+
+   or if your server is already running in docker:
+
+   ```shell
+   docker exec <container_name> headscale -n myfirstnamespace preauthkeys create --reusable --expiration 24h
+   ```
 
 2. Use the authkey from your machine to register it
-    ```shell
-    tailscale up -login-server YOUR_HEADSCALE_URL --authkey YOURAUTHKEY
-    ```
+   ```shell
+   tailscale up --login-server YOUR_HEADSCALE_URL --authkey YOURAUTHKEY
+   ```
 
 If you create an authkey with the `--ephemeral` flag, that key will create ephemeral nodes. This implies that `--reusable` is true.
 
-Please bear in mind that all the commands from headscale support adding `-o json` or `-o json-line`  to get a nicely JSON-formatted output.
-
+Please bear in mind that all the commands from headscale support adding `-o json` or `-o json-line` to get a nicely JSON-formatted output.
 
 ## Configuration reference
 
@@ -163,6 +189,7 @@ Headscale's configuration file is named `config.json` or `config.yaml`. Headscal
 ```
     "log_level": "debug"
 ```
+
 `log_level` can be used to set the Log level for Headscale, it defaults to `debug`, and the available levels are: `trace`, `debug`, `info`, `warn` and `error`.
 
 ```
@@ -193,7 +220,6 @@ Headscale's configuration file is named `config.json` or `config.yaml`. Headscal
 
 The fields starting with `db_` are used for the PostgreSQL connection information.
 
-
 ### Running the service via TLS (optional)
 
 ```
@@ -227,21 +253,21 @@ Alternatively, `tls_letsencrypt_challenge_type` can be set to `TLS-ALPN-01`. In
 Headscale implements the same policy ACLs as Tailscale.com, adapted to the self-hosted environment.
 
 For instance, instead of referring to users when defining groups you must
- use namespaces (which are the equivalent to user/logins in Tailscale.com).
+use namespaces (which are the equivalent to user/logins in Tailscale.com).
 
 Please check https://tailscale.com/kb/1018/acls/, and `./tests/acls/` in this repo for working examples.
 
+### Apple devices
+
+An endpoint with information on how to connect your Apple devices (currently macOS only) is available at `/apple` on your running instance.
 
 ## Disclaimer
 
 1. We have nothing to do with Tailscale, or Tailscale Inc.
 2. The purpose of writing this was to learn how Tailscale works.
 
-
-
 ## More on Tailscale
 
 - https://tailscale.com/blog/how-tailscale-works/
 - https://tailscale.com/blog/tailscale-key-management/
 - https://tailscale.com/blog/an-unlikely-database-migration/
-

api.go

@@ -64,6 +64,7 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 			Str("handler", "Registration").
 			Err(err).
 			Msg("Cannot parse machine key")
+		machineRegistrations.WithLabelValues("unkown", "web", "error", "unknown").Inc()
 		c.String(http.StatusInternalServerError, "Sad!")
 		return
 	}
@@ -74,6 +75,7 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 			Str("handler", "Registration").
 			Err(err).
 			Msg("Cannot decode message")
+		machineRegistrations.WithLabelValues("unkown", "web", "error", "unknown").Inc()
 		c.String(http.StatusInternalServerError, "Very sad!")
 		return
 	}
@@ -94,6 +96,7 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 				Str("handler", "Registration").
 				Err(err).
 				Msg("Could not create row")
+			machineRegistrations.WithLabelValues("unkown", "web", "error", m.Namespace.Name).Inc()
 			return
 		}
 	}
@@ -122,9 +125,11 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 					Str("handler", "Registration").
 					Err(err).
 					Msg("Cannot encode message")
+				machineRegistrations.WithLabelValues("update", "web", "error", m.Namespace.Name).Inc()
 				c.String(http.StatusInternalServerError, "")
 				return
 			}
+			machineRegistrations.WithLabelValues("update", "web", "success", m.Namespace.Name).Inc()
 			c.Data(200, "application/json; charset=utf-8", respBody)
 			return
 		}
@@ -141,9 +146,11 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 				Str("handler", "Registration").
 				Err(err).
 				Msg("Cannot encode message")
+			machineRegistrations.WithLabelValues("new", "web", "error", m.Namespace.Name).Inc()
 			c.String(http.StatusInternalServerError, "")
 			return
 		}
+		machineRegistrations.WithLabelValues("new", "web", "success", m.Namespace.Name).Inc()
 		c.Data(200, "application/json; charset=utf-8", respBody)
 		return
 	}
@@ -213,12 +220,12 @@ func (h *Headscale) RegistrationHandler(c *gin.Context) {
 	c.Data(200, "application/json; charset=utf-8", respBody)
 }
 
-func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Machine) (*[]byte, error) {
+func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Machine) ([]byte, error) {
 	log.Trace().
 		Str("func", "getMapResponse").
 		Str("machine", req.Hostinfo.Hostname).
 		Msg("Creating Map response")
-	node, err := m.toNode(true)
+	node, err := m.toNode(h.cfg.BaseDomain, h.cfg.DNSConfig, true)
 	if err != nil {
 		log.Error().
 			Str("func", "getMapResponse").
@@ -226,6 +233,7 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Mac
 			Msg("Cannot convert to node")
 		return nil, err
 	}
+
 	peers, err := h.getPeers(m)
 	if err != nil {
 		log.Error().
@@ -241,25 +249,40 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Mac
 		DisplayName: m.Namespace.Name,
 	}
 
+	nodePeers, err := peers.toNodes(h.cfg.BaseDomain, h.cfg.DNSConfig, true)
+	if err != nil {
+		log.Error().
+			Str("func", "getMapResponse").
+			Err(err).
+			Msg("Failed to convert peers to Tailscale nodes")
+		return nil, err
+	}
+
+	var dnsConfig *tailcfg.DNSConfig
+	if h.cfg.DNSConfig != nil && h.cfg.DNSConfig.Proxied { // if MagicDNS is enabled
+		// Only inject the Search Domain of the current namespace - shared nodes should use their full FQDN
+		dnsConfig = h.cfg.DNSConfig.Clone()
+		dnsConfig.Domains = append(dnsConfig.Domains, fmt.Sprintf("%s.%s", m.Namespace.Name, h.cfg.BaseDomain))
+	} else {
+		dnsConfig = h.cfg.DNSConfig
+	}
+
 	resp := tailcfg.MapResponse{
-		KeepAlive: false,
-		Node:      node,
-		Peers:     *peers,
-		//TODO(kradalby): As per tailscale docs, if DNSConfig is nil,
-		// it means its not updated, maybe we can have some logic
-		// to check and only pass updates when its updates.
-		// This is probably more relevant if we try to implement
-		// "MagicDNS"
-		DNSConfig:    h.cfg.DNSConfig,
-		SearchPaths:  []string{},
-		Domain:       "headscale.net",
+		KeepAlive:    false,
+		Node:         node,
+		Peers:        nodePeers,
+		DNSConfig:    dnsConfig,
+		Domain:       h.cfg.BaseDomain,
 		PacketFilter: *h.aclRules,
 		DERPMap:      h.cfg.DerpMap,
+
+		// TODO(juanfont): We should send the profiles of all the peers (this own namespace + those from the shared peers)
 		UserProfiles: []tailcfg.UserProfile{profile},
 	}
 	log.Trace().
 		Str("func", "getMapResponse").
 		Str("machine", req.Hostinfo.Hostname).
+		// Interface("payload", resp).
 		Msgf("Generated map response: %s", tailMapResponseToString(resp))
 
 	var respBody []byte
@@ -282,10 +305,10 @@ func (h *Headscale) getMapResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Mac
 	data := make([]byte, 4)
 	binary.LittleEndian.PutUint32(data, uint32(len(respBody)))
 	data = append(data, respBody...)
-	return &data, nil
+	return data, nil
 }
 
-func (h *Headscale) getMapKeepAliveResponse(mKey wgkey.Key, req tailcfg.MapRequest, m Machine) (*[]byte, error) {
+func (h *Headscale) getMapKeepAliveResponse(mKey wgkey.Key, req tailcfg.MapRequest, m *Machine) ([]byte, error) {
 	resp := tailcfg.MapResponse{
 		KeepAlive: true,
 	}
@@ -308,7 +331,7 @@ func (h *Headscale) getMapKeepAliveResponse(mKey wgkey.Key, req tailcfg.MapReque
 	data := make([]byte, 4)
 	binary.LittleEndian.PutUint32(data, uint32(len(respBody)))
 	data = append(data, respBody...)
-	return &data, nil
+	return data, nil
 }
 
 func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key, req tailcfg.RegisterRequest, m Machine) {
@@ -319,6 +342,11 @@ func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key,
 	resp := tailcfg.RegisterResponse{}
 	pak, err := h.checkKeyValidity(req.Auth.AuthKey)
 	if err != nil {
+		log.Error().
+			Str("func", "handleAuthKey").
+			Str("machine", m.Name).
+			Err(err).
+			Msg("Failed authentication via AuthKey")
 		resp.MachineAuthorized = false
 		respBody, err := encode(resp, &idKey, h.privateKey)
 		if err != nil {
@@ -328,13 +356,15 @@ func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key,
 				Err(err).
 				Msg("Cannot encode message")
 			c.String(http.StatusInternalServerError, "")
+			machineRegistrations.WithLabelValues("new", "authkey", "error", m.Namespace.Name).Inc()
 			return
 		}
-		c.Data(200, "application/json; charset=utf-8", respBody)
+		c.Data(401, "application/json; charset=utf-8", respBody)
 		log.Error().
 			Str("func", "handleAuthKey").
 			Str("machine", m.Name).
 			Msg("Failed authentication via AuthKey")
+		machineRegistrations.WithLabelValues("new", "authkey", "error", m.Namespace.Name).Inc()
 		return
 	}
 
@@ -348,6 +378,7 @@ func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key,
 			Str("func", "handleAuthKey").
 			Str("machine", m.Name).
 			Msg("Failed to find an available IP")
+		machineRegistrations.WithLabelValues("new", "authkey", "error", m.Namespace.Name).Inc()
 		return
 	}
 	log.Info().
@@ -364,6 +395,9 @@ func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key,
 	m.RegisterMethod = "authKey"
 	db.Save(&m)
 
+	pak.Used = true
+	db.Save(&pak)
+
 	resp.MachineAuthorized = true
 	resp.User = *pak.Namespace.toUser()
 	respBody, err := encode(resp, &idKey, h.privateKey)
@@ -373,9 +407,11 @@ func (h *Headscale) handleAuthKey(c *gin.Context, db *gorm.DB, idKey wgkey.Key,
 			Str("machine", m.Name).
 			Err(err).
 			Msg("Cannot encode message")
+		machineRegistrations.WithLabelValues("new", "authkey", "error", m.Namespace.Name).Inc()
 		c.String(http.StatusInternalServerError, "Extremely sad!")
 		return
 	}
+	machineRegistrations.WithLabelValues("new", "authkey", "success", m.Namespace.Name).Inc()
 	c.Data(200, "application/json; charset=utf-8", respBody)
 	log.Info().
 		Str("func", "handleAuthKey").

app.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -12,10 +13,13 @@ import (
 	"github.com/rs/zerolog/log"
 
 	"github.com/gin-gonic/gin"
+	ginprometheus "github.com/zsais/go-gin-prometheus"
+	"golang.org/x/crypto/acme"
 	"golang.org/x/crypto/acme/autocert"
 	"gorm.io/gorm"
 	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/dnstype"
 	"tailscale.com/types/wgkey"
 )
 
@@ -27,6 +31,7 @@ type Config struct {
 	DerpMap                        *tailcfg.DERPMap
 	EphemeralNodeInactivityTimeout time.Duration
 	IPPrefix                       netaddr.IPPrefix
+	BaseDomain                     string
 
 	DBtype string
 	DBpath string
@@ -44,6 +49,9 @@ type Config struct {
 	TLSCertPath string
 	TLSKeyPath  string
 
+	ACMEURL   string
+	ACMEEmail string
+
 	DNSConfig *tailcfg.DNSConfig
 }
 
@@ -60,9 +68,6 @@ type Headscale struct {
 	aclPolicy *ACLPolicy
 	aclRules  *[]tailcfg.FilterRule
 
-	clientsUpdateChannels     sync.Map
-	clientsUpdateChannelMutex sync.Mutex
-
 	lastStateChange sync.Map
 }
 
@@ -103,6 +108,17 @@ func NewHeadscale(cfg Config) (*Headscale, error) {
 		return nil, err
 	}
 
+	if h.cfg.DNSConfig != nil && h.cfg.DNSConfig.Proxied { // if MagicDNS
+		magicDNSDomains, err := generateMagicDNSRootDomains(h.cfg.IPPrefix, h.cfg.BaseDomain)
+		if err != nil {
+			return nil, err
+		}
+		h.cfg.DNSConfig.Routes = make(map[string][]dnstype.Resolver)
+		for _, d := range magicDNSDomains {
+			h.cfg.DNSConfig.Routes[d.WithoutTrailingDot()] = nil
+		}
+	}
+
 	return &h, nil
 }
 
@@ -140,9 +156,9 @@ func (h *Headscale) expireEphemeralNodesWorker() {
 				if err != nil {
 					log.Error().Err(err).Str("machine", m.Name).Msg("🤮 Cannot delete ephemeral machine from the database")
 				}
-				h.notifyChangesToPeers(&m)
 			}
 		}
+		h.setLastStateChangeToNow(ns.Name)
 	}
 }
 
@@ -163,23 +179,31 @@ func (h *Headscale) watchForKVUpdatesWorker() {
 // Serve launches a GIN server with the Headscale API
 func (h *Headscale) Serve() error {
 	r := gin.Default()
+
+	p := ginprometheus.NewPrometheus("gin")
+	p.Use(r)
+
 	r.GET("/health", func(c *gin.Context) { c.JSON(200, gin.H{"healthy": "ok"}) })
 	r.GET("/key", h.KeyHandler)
 	r.GET("/register", h.RegisterWebAPI)
 	r.POST("/machine/:id/map", h.PollNetMapHandler)
 	r.POST("/machine/:id", h.RegistrationHandler)
+	r.GET("/apple", h.AppleMobileConfig)
+	r.GET("/apple/:platform", h.ApplePlatformConfig)
 	var err error
 
-	timeout := 30 * time.Second
-
 	go h.watchForKVUpdates(5000)
 	go h.expireEphemeralNodes(5000)
 
 	s := &http.Server{
-		Addr:         h.cfg.Addr,
-		Handler:      r,
-		ReadTimeout:  timeout,
-		WriteTimeout: timeout,
+		Addr:        h.cfg.Addr,
+		Handler:     r,
+		ReadTimeout: 30 * time.Second,
+		// Go does not handle timeouts in HTTP very well, and there is
+		// no good way to handle streaming timeouts, therefore we need to
+		// keep this at unlimited and be careful to clean up connections
+		// https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/#aboutstreaming
+		WriteTimeout: 0,
 	}
 
 	if h.cfg.TLSLetsEncryptHostname != "" {
@@ -191,14 +215,14 @@ func (h *Headscale) Serve() error {
 			Prompt:     autocert.AcceptTOS,
 			HostPolicy: autocert.HostWhitelist(h.cfg.TLSLetsEncryptHostname),
 			Cache:      autocert.DirCache(h.cfg.TLSLetsEncryptCacheDir),
+			Client: &acme.Client{
+				DirectoryURL: h.cfg.ACMEURL,
+			},
+			Email: h.cfg.ACMEEmail,
 		}
-		s := &http.Server{
-			Addr:         h.cfg.Addr,
-			TLSConfig:    m.TLSConfig(),
-			Handler:      r,
-			ReadTimeout:  timeout,
-			WriteTimeout: timeout,
-		}
+
+		s.TLSConfig = m.TLSConfig()
+
 		if h.cfg.TLSLetsEncryptChallengeType == "TLS-ALPN-01" {
 			// Configuration via autocert with TLS-ALPN-01 (https://tools.ietf.org/html/rfc8737)
 			// The RFC requires that the validation is done on port 443; in other words, headscale
@@ -209,7 +233,6 @@ func (h *Headscale) Serve() error {
 			// port 80 for the certificate validation in addition to the headscale
 			// service, which can be configured to run on any other port.
 			go func() {
-
 				log.Fatal().
 					Err(http.ListenAndServe(h.cfg.TLSLetsEncryptListen, m.HTTPHandler(http.HandlerFunc(h.redirect)))).
 					Msg("failed to set up a HTTP server")
@@ -234,17 +257,32 @@ func (h *Headscale) Serve() error {
 
 func (h *Headscale) setLastStateChangeToNow(namespace string) {
 	now := time.Now().UTC()
+	lastStateUpdate.WithLabelValues("", "headscale").Set(float64(now.Unix()))
 	h.lastStateChange.Store(namespace, now)
 }
 
-func (h *Headscale) getLastStateChange(namespace string) time.Time {
-	if wrapped, ok := h.lastStateChange.Load(namespace); ok {
-		lastChange, _ := wrapped.(time.Time)
-		return lastChange
+func (h *Headscale) getLastStateChange(namespaces ...string) time.Time {
+	times := []time.Time{}
+
+	for _, namespace := range namespaces {
+		if wrapped, ok := h.lastStateChange.Load(namespace); ok {
+			lastChange, _ := wrapped.(time.Time)
+
+			times = append(times, lastChange)
+		}
 
 	}
 
-	now := time.Now().UTC()
-	h.lastStateChange.Store(namespace, now)
-	return now
+	sort.Slice(times, func(i, j int) bool {
+		return times[i].After(times[j])
+	})
+
+	log.Trace().Msgf("Latest times %#v", times)
+
+	if len(times) == 0 {
+		return time.Now().UTC()
+
+	} else {
+		return times[0]
+	}
 }

apple_mobileconfig.go

@@ -0,0 +1,226 @@
+package headscale
+
+import (
+	"bytes"
+	"net/http"
+	"text/template"
+
+	"github.com/rs/zerolog/log"
+
+	"github.com/gin-gonic/gin"
+	"github.com/gofrs/uuid"
+)
+
+// AppleMobileConfig shows a simple message in the browser to point to the CLI
+// Listens in /register
+func (h *Headscale) AppleMobileConfig(c *gin.Context) {
+	t := template.Must(template.New("apple").Parse(`
+<html>
+	<body>
+		<h1>Apple configuration profiles</h1>
+		<p>
+		    This page provides <a href="https://support.apple.com/guide/mdm/mdm-overview-mdmbf9e668/web">configuration profiles</a> for the official Tailscale clients for <a href="https://apps.apple.com/us/app/tailscale/id1470499037?ls=1">iOS</a> and <a href="https://apps.apple.com/ca/app/tailscale/id1475387142?mt=12">macOS</a>.
+		</p>
+		<p>
+		    The profiles will configure Tailscale.app to use {{.Url}} as its control server.
+		</p>
+
+		<h3>Caution</h3>
+		<p>You should always inspect the profile before installing it:</p>
+		<!--
+		<p><code>curl {{.Url}}/apple/ios</code></p>
+		-->
+		<p><code>curl {{.Url}}/apple/macos</code></p>
+		
+		<h2>Profiles</h2>
+
+		<!--
+		<h3>iOS</h3>
+		<p>
+		    <a href="/apple/ios" download="headscale_ios.mobileconfig">iOS profile</a>
+		</p>
+		-->
+		
+		<h3>macOS</h3>
+		<p>Headscale can be set to the default server by installing a Headscale configuration profile:</p>
+		<p>
+		    <a href="/apple/macos" download="headscale_macos.mobileconfig">macOS profile</a>
+		</p>
+
+		<ol>
+		<li>Download the profile, then open it. When it has been opened, there should be a notification that a profile can be installed</li>
+		<li>Open System Preferences and go to "Profiles"</li>
+		<li>Find and install the Headscale profile</li>
+		<li>Restart Tailscale.app and log in</li>
+		</ol>
+
+		<p>Or</p>
+		<p>Use your terminal to configure the default setting for Tailscale by issuing:</p>
+		<code>defaults write io.tailscale.ipn.macos ControlURL {{.Url}}</code>
+
+		<p>Restart Tailscale.app and log in.</p>
+	
+	</body>
+</html>`))
+
+	config := map[string]interface{}{
+		"Url": h.cfg.ServerURL,
+	}
+
+	var payload bytes.Buffer
+	if err := t.Execute(&payload, config); err != nil {
+		log.Error().
+			Str("handler", "AppleMobileConfig").
+			Err(err).
+			Msg("Could not render Apple index template")
+		c.Data(http.StatusInternalServerError, "text/html; charset=utf-8", []byte("Could not render Apple index template"))
+		return
+	}
+
+	c.Data(http.StatusOK, "text/html; charset=utf-8", payload.Bytes())
+}
+
+func (h *Headscale) ApplePlatformConfig(c *gin.Context) {
+	platform := c.Param("platform")
+
+	id, err := uuid.NewV4()
+	if err != nil {
+		log.Error().
+			Str("handler", "ApplePlatformConfig").
+			Err(err).
+			Msg("Failed not create UUID")
+		c.Data(http.StatusInternalServerError, "text/html; charset=utf-8", []byte("Failed to create UUID"))
+		return
+	}
+
+	contentId, err := uuid.NewV4()
+	if err != nil {
+		log.Error().
+			Str("handler", "ApplePlatformConfig").
+			Err(err).
+			Msg("Failed not create UUID")
+		c.Data(http.StatusInternalServerError, "text/html; charset=utf-8", []byte("Failed to create UUID"))
+		return
+	}
+
+	platformConfig := AppleMobilePlatformConfig{
+		UUID: contentId,
+		Url:  h.cfg.ServerURL,
+	}
+
+	var payload bytes.Buffer
+
+	switch platform {
+	case "macos":
+		if err := macosTemplate.Execute(&payload, platformConfig); err != nil {
+			log.Error().
+				Str("handler", "ApplePlatformConfig").
+				Err(err).
+				Msg("Could not render Apple macOS template")
+			c.Data(http.StatusInternalServerError, "text/html; charset=utf-8", []byte("Could not render Apple macOS template"))
+			return
+		}
+	case "ios":
+		if err := iosTemplate.Execute(&payload, platformConfig); err != nil {
+			log.Error().
+				Str("handler", "ApplePlatformConfig").
+				Err(err).
+				Msg("Could not render Apple iOS template")
+			c.Data(http.StatusInternalServerError, "text/html; charset=utf-8", []byte("Could not render Apple iOS template"))
+			return
+		}
+	default:
+		c.Data(http.StatusOK, "text/html; charset=utf-8", []byte("Invalid platform, only ios and macos is supported"))
+		return
+	}
+
+	config := AppleMobileConfig{
+		UUID:    id,
+		Url:     h.cfg.ServerURL,
+		Payload: payload.String(),
+	}
+
+	var content bytes.Buffer
+	if err := commonTemplate.Execute(&content, config); err != nil {
+		log.Error().
+			Str("handler", "ApplePlatformConfig").
+			Err(err).
+			Msg("Could not render Apple platform template")
+		c.Data(http.StatusInternalServerError, "text/html; charset=utf-8", []byte("Could not render Apple platform template"))
+		return
+	}
+
+	c.Data(http.StatusOK, "application/x-apple-aspen-config; charset=utf-8", content.Bytes())
+}
+
+type AppleMobileConfig struct {
+	UUID    uuid.UUID
+	Url     string
+	Payload string
+}
+
+type AppleMobilePlatformConfig struct {
+	UUID uuid.UUID
+	Url  string
+}
+
+var commonTemplate = template.Must(template.New("mobileconfig").Parse(`<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+  <dict>
+    <key>PayloadUUID</key>
+    <string>{{.UUID}}</string>
+    <key>PayloadDisplayName</key>
+    <string>Headscale</string>
+    <key>PayloadDescription</key>
+    <string>Configure Tailscale login server to: {{.Url}}</string>
+    <key>PayloadIdentifier</key>
+    <string>com.github.juanfont.headscale</string>
+    <key>PayloadRemovalDisallowed</key>
+    <false/>
+    <key>PayloadType</key>
+    <string>Configuration</string>
+    <key>PayloadVersion</key>
+    <integer>1</integer>
+    <key>PayloadContent</key>
+    <array>
+    {{.Payload}}
+    </array>
+  </dict>
+</plist>`))
+
+var iosTemplate = template.Must(template.New("iosTemplate").Parse(`
+    <dict>
+        <key>PayloadType</key>
+        <string>io.tailscale.ipn.ios</string>
+        <key>PayloadUUID</key>
+        <string>{{.UUID}}</string>
+        <key>PayloadIdentifier</key>
+        <string>com.github.juanfont.headscale</string>
+        <key>PayloadVersion</key>
+        <integer>1</integer>
+        <key>PayloadEnabled</key>
+        <true/>
+
+        <key>ControlURL</key>
+        <string>{{.Url}}</string>
+    </dict>
+`))
+
+var macosTemplate = template.Must(template.New("macosTemplate").Parse(`
+    <dict>
+        <key>PayloadType</key>
+        <string>io.tailscale.ipn.macos</string>
+        <key>PayloadUUID</key>
+        <string>{{.UUID}}</string>
+        <key>PayloadIdentifier</key>
+        <string>com.github.juanfont.headscale</string>
+        <key>PayloadVersion</key>
+        <integer>1</integer>
+        <key>PayloadEnabled</key>
+        <true/>
+
+        <key>ControlURL</key>
+        <string>{{.Url}}</string>
+    </dict>
+`))

cmd/headscale/cli/nodes.go

@@ -129,6 +129,7 @@ var deleteNodeCmd = &cobra.Command{
 		return nil
 	},
 	Run: func(cmd *cobra.Command, args []string) {
+		output, _ := cmd.Flags().GetString("output")
 		h, err := getHeadscaleApp()
 		if err != nil {
 			log.Fatalf("Error initializing: %s", err)
@@ -143,21 +144,32 @@ var deleteNodeCmd = &cobra.Command{
 		}
 
 		confirm := false
-		prompt := &survey.Confirm{
-			Message: fmt.Sprintf("Do you want to remove the node %s?", m.Name),
-		}
-		err = survey.AskOne(prompt, &confirm)
-		if err != nil {
-			return
+		force, _ := cmd.Flags().GetBool("force")
+		if !force {
+			prompt := &survey.Confirm{
+				Message: fmt.Sprintf("Do you want to remove the node %s?", m.Name),
+			}
+			err = survey.AskOne(prompt, &confirm)
+			if err != nil {
+				return
+			}
 		}
 
-		if confirm {
+		if confirm || force {
 			err = h.DeleteMachine(m)
+			if strings.HasPrefix(output, "json") {
+				JsonOutput(map[string]string{"Result": "Node deleted"}, err, output)
+				return
+			}
 			if err != nil {
 				log.Fatalf("Error deleting node: %s", err)
 			}
 			fmt.Printf("Node deleted\n")
 		} else {
+			if strings.HasPrefix(output, "json") {
+				JsonOutput(map[string]string{"Result": "Node not deleted"}, err, output)
+				return
+			}
 			fmt.Printf("Node not deleted\n")
 		}
 	},

cmd/headscale/cli/preauthkeys.go

@@ -57,7 +57,7 @@ var listPreAuthKeys = &cobra.Command{
 			return
 		}
 
-		d := pterm.TableData{{"ID", "Key", "Reusable", "Ephemeral", "Expiration", "Created"}}
+		d := pterm.TableData{{"ID", "Key", "Reusable", "Ephemeral", "Used", "Expiration", "Created"}}
 		for _, k := range *keys {
 			expiration := "-"
 			if k.Expiration != nil {
@@ -76,6 +76,7 @@ var listPreAuthKeys = &cobra.Command{
 				k.Key,
 				reusable,
 				strconv.FormatBool(k.Ephemeral),
+				fmt.Sprintf("%v", k.Used),
 				expiration,
 				k.CreatedAt.Format("2006-01-02 15:04:05"),
 			})
@@ -130,7 +131,7 @@ var createPreAuthKeyCmd = &cobra.Command{
 }
 
 var expirePreAuthKeyCmd = &cobra.Command{
-	Use:   "expire",
+	Use:   "expire KEY",
 	Short: "Expire a preauthkey",
 	Args: func(cmd *cobra.Command, args []string) error {
 		if len(args) < 1 {
@@ -152,6 +153,10 @@ var expirePreAuthKeyCmd = &cobra.Command{
 
 		k, err := h.GetPreAuthKey(n, args[0])
 		if err != nil {
+			if strings.HasPrefix(o, "json") {
+				JsonOutput(k, err, o)
+				return
+			}
 			log.Fatalf("Error getting the key: %s", err)
 		}
 

cmd/headscale/cli/root.go

@@ -9,6 +9,7 @@ import (
 
 func init() {
 	rootCmd.PersistentFlags().StringP("output", "o", "", "Output format. Empty for human-readable, 'json' or 'json-line'")
+	rootCmd.PersistentFlags().Bool("force", false, "Disable prompts and forces the execution")
 }
 
 var rootCmd = &cobra.Command{

cmd/headscale/cli/utils.go

@@ -76,7 +76,7 @@ func LoadConfig(path string) error {
 
 }
 
-func GetDNSConfig() *tailcfg.DNSConfig {
+func GetDNSConfig() (*tailcfg.DNSConfig, string) {
 	if viper.IsSet("dns_config") {
 		dnsConfig := &tailcfg.DNSConfig{}
 
@@ -108,10 +108,27 @@ func GetDNSConfig() *tailcfg.DNSConfig {
 			dnsConfig.Domains = viper.GetStringSlice("dns_config.domains")
 		}
 
-		return dnsConfig
+		if viper.IsSet("dns_config.magic_dns") {
+			magicDNS := viper.GetBool("dns_config.magic_dns")
+			if len(dnsConfig.Nameservers) > 0 {
+				dnsConfig.Proxied = magicDNS
+			} else if magicDNS {
+				log.Warn().
+					Msg("Warning: dns_config.magic_dns is set, but no nameservers are configured. Ignoring magic_dns.")
+			}
+		}
+
+		var baseDomain string
+		if viper.IsSet("dns_config.base_domain") {
+			baseDomain = viper.GetString("dns_config.base_domain")
+		} else {
+			baseDomain = "headscale.net" // does not really matter when MagicDNS is not enabled
+		}
+
+		return dnsConfig, baseDomain
 	}
 
-	return nil
+	return nil, ""
 }
 
 func absPath(path string) string {
@@ -144,12 +161,15 @@ func getHeadscaleApp() (*headscale.Headscale, error) {
 		return nil, err
 	}
 
+	dnsConfig, baseDomain := GetDNSConfig()
+
 	cfg := headscale.Config{
 		ServerURL:      viper.GetString("server_url"),
 		Addr:           viper.GetString("listen_addr"),
 		PrivateKeyPath: absPath(viper.GetString("private_key_path")),
 		DerpMap:        derpMap,
 		IPPrefix:       netaddr.MustParseIPPrefix(viper.GetString("ip_prefix")),
+		BaseDomain:     baseDomain,
 
 		EphemeralNodeInactivityTimeout: viper.GetDuration("ephemeral_node_inactivity_timeout"),
 
@@ -169,7 +189,10 @@ func getHeadscaleApp() (*headscale.Headscale, error) {
 		TLSCertPath: absPath(viper.GetString("tls_cert_path")),
 		TLSKeyPath:  absPath(viper.GetString("tls_key_path")),
 
-		DNSConfig: GetDNSConfig(),
+		DNSConfig: dnsConfig,
+
+		ACMEEmail: viper.GetString("acme_email"),
+		ACMEURL:   viper.GetString("acme_url"),
 	}
 
 	h, err := headscale.NewHeadscale(cfg)
@@ -239,3 +262,12 @@ func JsonOutput(result interface{}, errResult error, outputFormat string) {
 	}
 	fmt.Println(string(j))
 }
+
+func HasJsonOutputFlag() bool {
+	for _, arg := range os.Args {
+		if arg == "json" || arg == "json-line" {
+			return true
+		}
+	}
+	return false
+}

cmd/headscale/cli/version.go

@@ -2,11 +2,12 @@ package cli
 
 import (
 	"fmt"
-	"github.com/spf13/cobra"
 	"strings"
+
+	"github.com/spf13/cobra"
 )
 
-var version = "dev"
+var Version = "dev"
 
 func init() {
 	rootCmd.AddCommand(versionCmd)
@@ -19,9 +20,9 @@ var versionCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		o, _ := cmd.Flags().GetString("output")
 		if strings.HasPrefix(o, "json") {
-			JsonOutput(map[string]string{"version": version}, nil, o)
+			JsonOutput(map[string]string{"version": Version}, nil, o)
 			return
 		}
-		fmt.Println(version)
+		fmt.Println(Version)
 	},
 }

cmd/headscale/headscale.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"fmt"
 	"os"
+	"runtime"
 	"time"
 
 	"github.com/efekarakus/termcolor"
@@ -9,6 +11,7 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
+	"github.com/tcnksm/go-latest"
 )
 
 func main() {
@@ -59,5 +62,20 @@ func main() {
 		zerolog.SetGlobalLevel(zerolog.DebugLevel)
 	}
 
+	jsonOutput := cli.HasJsonOutputFlag()
+	if !viper.GetBool("disable_check_updates") && !jsonOutput {
+		if (runtime.GOOS == "linux" || runtime.GOOS == "darwin") && cli.Version != "dev" {
+			githubTag := &latest.GithubTag{
+				Owner:      "juanfont",
+				Repository: "headscale",
+			}
+			res, err := latest.Check(githubTag, cli.Version)
+			if err == nil && res.Outdated {
+				fmt.Printf("An updated version of Headscale has been found (%s vs. your current %s). Check it out https://github.com/juanfont/headscale/releases\n",
+					res.Current, cli.Version)
+			}
+		}
+	}
+
 	cli.Execute()
 }

cmd/headscale/headscale_test.go

@@ -117,12 +117,12 @@ func (*Suite) TestDNSConfigLoading(c *check.C) {
 	err = cli.LoadConfig(tmpDir)
 	c.Assert(err, check.IsNil)
 
-	dnsConfig := cli.GetDNSConfig()
-	fmt.Println(dnsConfig)
+	dnsConfig, baseDomain := cli.GetDNSConfig()
 
 	c.Assert(dnsConfig.Nameservers[0].String(), check.Equals, "1.1.1.1")
-
 	c.Assert(dnsConfig.Resolvers[0].Addr, check.Equals, "1.1.1.1")
+	c.Assert(dnsConfig.Proxied, check.Equals, true)
+	c.Assert(baseDomain, check.Equals, "example.com")
 }
 
 func writeConfig(c *check.C, tmpDir string, configYaml []byte) {

config.json.postgres.example

@@ -10,6 +10,8 @@
     "db_name": "headscale",
     "db_user": "foo",
     "db_pass": "bar",
+    "acme_url": "https://acme-v02.api.letsencrypt.org/directory",
+    "acme_email": "",
     "tls_letsencrypt_hostname": "",
     "tls_letsencrypt_listen": ":http",
     "tls_letsencrypt_cache_dir": ".cache",
@@ -20,6 +22,9 @@
     "dns_config": {
         "nameservers": [
             "1.1.1.1"
-        ]
+        ],
+        "domains": [],
+        "magic_dns": true,
+        "base_domain": "example.com"
     }
 }

config.json.sqlite.example

@@ -6,6 +6,8 @@
     "ephemeral_node_inactivity_timeout": "30m",
     "db_type": "sqlite3",
     "db_path": "db.sqlite",
+    "acme_url": "https://acme-v02.api.letsencrypt.org/directory",
+    "acme_email": "",
     "tls_letsencrypt_hostname": "",
     "tls_letsencrypt_listen": ":http",
     "tls_letsencrypt_cache_dir": ".cache",
@@ -16,6 +18,9 @@
     "dns_config": {
         "nameservers": [
             "1.1.1.1"
-        ]
+        ],
+        "domains": [],
+        "magic_dns": true,
+        "base_domain": "example.com"
     }
 }

dns.go

@@ -0,0 +1,73 @@
+package headscale
+
+import (
+	"fmt"
+	"strings"
+
+	"inet.af/netaddr"
+	"tailscale.com/util/dnsname"
+)
+
+// generateMagicDNSRootDomains generates a list of DNS entries to be included in `Routes` in `MapResponse`.
+// This list of reverse DNS entries instructs the OS on what subnets and domains the Tailscale embedded DNS
+// server (listening in 100.100.100.100 udp/53) should be used for.
+//
+// Tailscale.com includes in the list:
+// - the `BaseDomain` of the user
+// - the reverse DNS entry for IPv6 (0.e.1.a.c.5.1.1.a.7.d.f.ip6.arpa., see below more on IPv6)
+// - the reverse DNS entries for the IPv4 subnets covered by the user's `IPPrefix`.
+//   In the public SaaS this is [64-127].100.in-addr.arpa.
+//
+// The main purpose of this function is then generating the list of IPv4 entries. For the 100.64.0.0/10, this
+// is clear, and could be hardcoded. But we are allowing any range as `IPPrefix`, so we need to find out the
+// subnets when we have 172.16.0.0/16 (i.e., [0-255].16.172.in-addr.arpa.), or any other subnet.
+//
+// How IN-ADDR.ARPA domains work is defined in RFC1035 (section 3.5). Tailscale.com seems to adhere to this,
+// and do not make use of RFC2317 ("Classless IN-ADDR.ARPA delegation") - hence generating the entries for the next
+// class block only.
+
+// From the netmask we can find out the wildcard bits (the bits that are not set in the netmask).
+// This allows us to then calculate the subnets included in the subsequent class block and generate the entries.
+func generateMagicDNSRootDomains(ipPrefix netaddr.IPPrefix, baseDomain string) ([]dnsname.FQDN, error) {
+	base, err := dnsname.ToFQDN(baseDomain)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO(juanfont): we are not handing out IPv6 addresses yet
+	// and in fact this is Tailscale.com's range (note the fd7a:115c:a1e0: range in the fc00::/7 network)
+	ipv6base := dnsname.FQDN("0.e.1.a.c.5.1.1.a.7.d.f.ip6.arpa.")
+	fqdns := []dnsname.FQDN{base, ipv6base}
+
+	// Conversion to the std lib net.IPnet, a bit easier to operate
+	netRange := ipPrefix.IPNet()
+	maskBits, _ := netRange.Mask.Size()
+
+	// lastOctet is the last IP byte covered by the mask
+	lastOctet := maskBits / 8
+
+	// wildcardBits is the number of bits not under the mask in the lastOctet
+	wildcardBits := 8 - maskBits%8
+
+	// min is the value in the lastOctet byte of the IP
+	// max is basically 2^wildcardBits - i.e., the value when all the wildcardBits are set to 1
+	min := uint(netRange.IP[lastOctet])
+	max := uint((min + 1<<uint(wildcardBits)) - 1)
+
+	// here we generate the base domain (e.g., 100.in-addr.arpa., 16.172.in-addr.arpa., etc.)
+	rdnsSlice := []string{}
+	for i := lastOctet - 1; i >= 0; i-- {
+		rdnsSlice = append(rdnsSlice, fmt.Sprintf("%d", netRange.IP[i]))
+	}
+	rdnsSlice = append(rdnsSlice, "in-addr.arpa.")
+	rdnsBase := strings.Join(rdnsSlice, ".")
+
+	for i := min; i <= max; i++ {
+		fqdn, err := dnsname.ToFQDN(fmt.Sprintf("%d.%s", i, rdnsBase))
+		if err != nil {
+			continue
+		}
+		fqdns = append(fqdns, fqdn)
+	}
+	return fqdns, nil
+}

dns_test.go

@@ -0,0 +1,63 @@
+package headscale
+
+import (
+	"gopkg.in/check.v1"
+	"inet.af/netaddr"
+)
+
+func (s *Suite) TestMagicDNSRootDomains100(c *check.C) {
+	prefix := netaddr.MustParseIPPrefix("100.64.0.0/10")
+	domains, err := generateMagicDNSRootDomains(prefix, "headscale.net")
+	c.Assert(err, check.IsNil)
+
+	found := false
+	for _, domain := range domains {
+		if domain == "64.100.in-addr.arpa." {
+			found = true
+			break
+		}
+	}
+	c.Assert(found, check.Equals, true)
+
+	found = false
+	for _, domain := range domains {
+		if domain == "100.100.in-addr.arpa." {
+			found = true
+			break
+		}
+	}
+	c.Assert(found, check.Equals, true)
+
+	found = false
+	for _, domain := range domains {
+		if domain == "127.100.in-addr.arpa." {
+			found = true
+			break
+		}
+	}
+	c.Assert(found, check.Equals, true)
+}
+
+func (s *Suite) TestMagicDNSRootDomains172(c *check.C) {
+	prefix := netaddr.MustParseIPPrefix("172.16.0.0/16")
+	domains, err := generateMagicDNSRootDomains(prefix, "headscale.net")
+	c.Assert(err, check.IsNil)
+
+	found := false
+	for _, domain := range domains {
+		if domain == "0.16.172.in-addr.arpa." {
+			found = true
+			break
+		}
+	}
+	c.Assert(found, check.Equals, true)
+
+	found = false
+	for _, domain := range domains {
+		if domain == "255.16.172.in-addr.arpa." {
+			found = true
+			break
+		}
+	}
+	c.Assert(found, check.Equals, true)
+}

docs/DNS.md

@@ -0,0 +1,33 @@
+# DNS in Headscale
+
+Headscale supports Tailscale's DNS configuration and MagicDNS. Please have a look to their KB to better understand what this means:
+
+- https://tailscale.com/kb/1054/dns/
+- https://tailscale.com/kb/1081/magicdns/
+- https://tailscale.com/blog/2021-09-private-dns-with-magicdns/
+
+Long story short, you can define the DNS servers you want to use in your tailnets, activate MagicDNS (so you don't have to remember the IP addresses of your nodes), define search domains, as well as predefined hosts. Headscale will inject that settings into your nodes.
+
+
+## Configuration reference
+
+The setup is done via the `config.json` file, under the `dns_config` key. 
+
+```json
+{
+    "server_url": "http://127.0.0.1:8001",
+    "listen_addr": "0.0.0.0:8001",
+    "private_key_path": "private.key",
+    //...
+    "dns_config": {
+        "nameservers": ["1.1.1.1", "8.8.8.8"],
+        "domains": [],
+        "magic_dns": true,
+        "base_domain": "example.com"
+    }
+}
+```
+- `nameservers`:  The list of DNS servers to use.
+- `domains`: Search domains to inject.
+- `magic_dns`: Whether to use [MagicDNS](https://tailscale.com/kb/1081/magicdns/). Only works if there is at least a nameserver defined.
+- `base_domain`: Defines the base domain to create the hostnames for MagicDNS. `base_domain` must be a FQDNs, without the trailing dot. The FQDN of the hosts will be `hostname.namespace.base_domain` (e.g., _myhost.mynamespace.example.com_).

go.mod

@@ -10,20 +10,27 @@ require (
 	github.com/docker/cli v20.10.8+incompatible // indirect
 	github.com/docker/docker v20.10.8+incompatible // indirect
 	github.com/efekarakus/termcolor v1.0.1
+	github.com/fatih/set v0.2.1 // indirect
 	github.com/gin-gonic/gin v1.7.4
+	github.com/gofrs/uuid v4.0.0+incompatible
+	github.com/google/go-github v17.0.0+incompatible // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b
 	github.com/klauspost/compress v1.13.5
 	github.com/lib/pq v1.10.3 // indirect
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/opencontainers/runc v1.0.2 // indirect
 	github.com/ory/dockertest/v3 v3.7.0
+	github.com/prometheus/client_golang v1.11.0
 	github.com/pterm/pterm v0.12.30
 	github.com/rs/zerolog v1.25.0
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/viper v1.8.1
 	github.com/stretchr/testify v1.7.0
 	github.com/tailscale/hujson v0.0.0-20210818175511-7360507a6e88
+	github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/zsais/go-gin-prometheus v0.1.0
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
 	golang.org/x/net v0.0.0-20210913180222-943fd674d43e // indirect
 	golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 // indirect

go.sum

@@ -103,6 +103,7 @@ github.com/aws/aws-sdk-go v1.38.52/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2z
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
@@ -118,7 +119,9 @@ github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq
 github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ=
 github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -201,6 +204,8 @@ github.com/fanliao/go-promise v0.0.0-20141029170127-1890db352a72/go.mod h1:Pjfxu
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
+github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA=
+github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
@@ -338,6 +343,10 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
+github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f/go.mod h1:n1ej5+FqyEytMt/mugVDZLIiqTMO+vsrgY+kM6ohzN0=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f/go.mod h1:nOFQdrUlIlx6M6ODdSpBj1NVA+VgLC6kmw60mkw34H4=
@@ -405,6 +414,7 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -513,6 +523,7 @@ github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhB
 github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
 github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ=
 github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok=
@@ -526,6 +537,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -533,6 +545,7 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
@@ -611,6 +624,7 @@ github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGw
 github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxzIU=
 github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mbilski/exhaustivestruct v1.1.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc=
 github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7/go.mod h1:U6ZQobyTjI/tJyq2HG+i/dfSoFUt8/aZCM+GKtmFk/Y=
@@ -664,6 +678,7 @@ github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8q
 github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c=
 github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
 github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=
@@ -742,21 +757,32 @@ github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
+github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/pterm/pterm v0.12.27/go.mod h1:PhQ89w4i95rhgE+xedAoqous6K9X+r6aSOI2eFF7DZI=
 github.com/pterm/pterm v0.12.29/go.mod h1:WI3qxgvoQFFGKGjGnJR849gU0TsEOvKn5Q8LlY1U7lg=
@@ -871,6 +897,8 @@ github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2/go.mod h1:STqf+YV
 github.com/tailscale/hujson v0.0.0-20210818175511-7360507a6e88 h1:q5Sxx79nhG4xWsYEJBlLdqo1hNhUV31/NhA4qQ1SKAY=
 github.com/tailscale/hujson v0.0.0-20210818175511-7360507a6e88/go.mod h1:iTDXJsA6A2wNNjurgic2rk+is6uzU4U2NLm4T+edr6M=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
+github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e h1:IWllFTiDjjLIf2oeKxpIUmtiDV5sn71VgeQgg6vcE7k=
+github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e/go.mod h1:d7u6HkTYKSv5m6MCKkOQlHwaShTMl3HjqSGW3XtVhXM=
 github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
 github.com/tdakkota/asciicheck v0.0.0-20200416200610-e657995f937b/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
 github.com/tetafro/godot v1.3.0/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQxA5S+0=
@@ -922,6 +950,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 github.com/ziutek/telnet v0.0.0-20180329124119-c3b780dc415b/go.mod h1:IZpXDfkJ6tWD3PhBK5YzgQT+xJWh7OsdwiG8hA2MkO4=
+github.com/zsais/go-gin-prometheus v0.1.0 h1:bkLv1XCdzqVgQ36ScgRi09MA2UC1t3tAB6nsfErsGO4=
+github.com/zsais/go-gin-prometheus v0.1.0/go.mod h1:Slirjzuz8uM8Cw0jmPNqbneoqcUtY2GGjn2bEd4NRLY=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
@@ -1136,6 +1166,7 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1151,6 +1182,8 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1183,6 +1216,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1443,6 +1477,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

integration_test.go

@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -233,9 +234,6 @@ func (s *IntegrationTestSuite) SetupSuite() {
 		},
 		Networks: []*dockertest.Network{&network},
 		Cmd:      []string{"headscale", "serve"},
-		PortBindings: map[docker.Port][]docker.PortBinding{
-			"8080/tcp": {{HostPort: "8080"}},
-		},
 	}
 
 	fmt.Println("Creating headscale container")
@@ -270,7 +268,11 @@ func (s *IntegrationTestSuite) SetupSuite() {
 		}
 		return nil
 	}); err != nil {
-		log.Fatalf("Could not connect to docker: %s", err)
+		// TODO(kradalby): If we cannot access headscale, or any other fatal error during
+		// test setup, we need to abort and tear down. However, testify does not seem to
+		// support that at the moment:
+		// https://github.com/stretchr/testify/issues/849
+		return // fmt.Errorf("Could not connect to headscale: %s", err)
 	}
 	fmt.Println("headscale container is ready")
 
@@ -292,7 +294,7 @@ func (s *IntegrationTestSuite) SetupSuite() {
 		)
 		assert.Nil(s.T(), err)
 
-		headscaleEndpoint := fmt.Sprintf("http://headscale:%s", headscale.GetPort("8080/tcp"))
+		headscaleEndpoint := "http://headscale:8080"
 
 		fmt.Printf("Joining tailscale containers to headscale at %s\n", headscaleEndpoint)
 		for hostname, tailscale := range scales.tailscales {
@@ -353,15 +355,16 @@ func (s *IntegrationTestSuite) TestGetIpAddresses() {
 
 		for hostname := range scales.tailscales {
 			s.T().Run(hostname, func(t *testing.T) {
-				ip := ips[hostname]
+				ip, ok := ips[hostname]
+
+				assert.True(t, ok)
+				assert.NotNil(t, ip)
 
 				fmt.Printf("IP for %s: %s\n", hostname, ip)
 
 				// c.Assert(ip.Valid(), check.IsTrue)
 				assert.True(t, ip.Is4())
 				assert.True(t, ipPrefix.Contains(ip))
-
-				ips[hostname] = ip
 			})
 		}
 	}
@@ -433,7 +436,7 @@ func (s *IntegrationTestSuite) TestPingAllPeers() {
 						command := []string{
 							"tailscale", "ping",
 							"--timeout=1s",
-							"--c=20",
+							"--c=10",
 							"--until-direct=true",
 							ip.String(),
 						}
@@ -501,43 +504,43 @@ func (s *IntegrationTestSuite) TestSharedNodes() {
 		assert.Contains(s.T(), result, hostname)
 	}
 
-	// TODO(kradalby): Figure out why these connections are not set up
-	// // TODO: See if we can have a more deterministic wait here.
-	// time.Sleep(100 * time.Second)
+	// TODO(juanfont): We have to find out why do we need to wait
+	time.Sleep(100 * time.Second) // Wait for the nodes to receive updates
 
-	// mainIps, err := getIPs(main.tailscales)
-	// assert.Nil(s.T(), err)
+	mainIps, err := getIPs(main.tailscales)
+	assert.Nil(s.T(), err)
 
-	// sharedIps, err := getIPs(shared.tailscales)
-	// assert.Nil(s.T(), err)
+	sharedIps, err := getIPs(shared.tailscales)
+	assert.Nil(s.T(), err)
 
-	// for hostname, tailscale := range main.tailscales {
-	// 	for peername, ip := range sharedIps {
-	// 		s.T().Run(fmt.Sprintf("%s-%s", hostname, peername), func(t *testing.T) {
-	// 			// We currently cant ping ourselves, so skip that.
-	// 			if peername != hostname {
-	// 				// We are only interested in "direct ping" which means what we
-	// 				// might need a couple of more attempts before reaching the node.
-	// 				command := []string{
-	// 					"tailscale", "ping",
-	// 					"--timeout=1s",
-	// 					"--c=20",
-	// 					"--until-direct=true",
-	// 					ip.String(),
-	// 				}
+	for hostname, tailscale := range main.tailscales {
+		for peername, ip := range sharedIps {
+			s.T().Run(fmt.Sprintf("%s-%s", hostname, peername), func(t *testing.T) {
+				// We currently cant ping ourselves, so skip that.
+				if peername != hostname {
+					// We are only interested in "direct ping" which means what we
+					// might need a couple of more attempts before reaching the node.
+					command := []string{
+						"tailscale", "ping",
+						"--timeout=15s",
+						"--c=20",
+						"--until-direct=true",
+						ip.String(),
+					}
 
-	// 				fmt.Printf("Pinging from %s (%s) to %s (%s)\n", hostname, mainIps[hostname], peername, ip)
-	// 				result, err := executeCommand(
-	// 					&tailscale,
-	// 					command,
-	// 				)
-	// 				assert.Nil(t, err)
-	// 				fmt.Printf("Result for %s: %s\n", hostname, result)
-	// 				assert.Contains(t, result, "pong")
-	// 			}
-	// 		})
-	// 	}
-	// }
+					fmt.Printf("Pinging from %s (%s) to %s (%s)\n", hostname, mainIps[hostname], peername, ip)
+					result, err := executeCommand(
+						&tailscale,
+						command,
+						[]string{},
+					)
+					assert.Nil(t, err)
+					fmt.Printf("Result for %s: %s\n", hostname, result)
+					assert.Contains(t, result, "pong")
+				}
+			})
+		}
+	}
 }
 
 func (s *IntegrationTestSuite) TestTailDrop() {
@@ -589,7 +592,7 @@ func (s *IntegrationTestSuite) TestTailDrop() {
 							_, err = executeCommand(
 								&tailscale,
 								command,
-								[]string{"ALL_PROXY=socks5://localhost:1055/"},
+								[]string{"ALL_PROXY=socks5://localhost:1055"},
 							)
 							if err == nil {
 								break
@@ -642,6 +645,38 @@ func (s *IntegrationTestSuite) TestTailDrop() {
 	}
 }
 
+func (s *IntegrationTestSuite) TestMagicDNS() {
+	for namespace, scales := range s.namespaces {
+		ips, err := getIPs(scales.tailscales)
+		assert.Nil(s.T(), err)
+		for hostname, tailscale := range scales.tailscales {
+			for peername, ip := range ips {
+				s.T().Run(fmt.Sprintf("%s-%s", hostname, peername), func(t *testing.T) {
+					if peername != hostname {
+						command := []string{
+							"tailscale", "ping",
+							"--timeout=10s",
+							"--c=20",
+							"--until-direct=true",
+							fmt.Sprintf("%s.%s.headscale.net", peername, namespace),
+						}
+
+						fmt.Printf("Pinging using Hostname (magicdns) from %s (%s) to %s (%s)\n", hostname, ips[hostname], peername, ip)
+						result, err := executeCommand(
+							&tailscale,
+							command,
+							[]string{},
+						)
+						assert.Nil(t, err)
+						fmt.Printf("Result for %s: %s\n", hostname, result)
+						assert.Contains(t, result, "pong")
+					}
+				})
+			}
+		}
+	}
+}
+
 func getIPs(tailscales map[string]dockertest.Resource) (map[string]netaddr.IP, error) {
 	ips := make(map[string]netaddr.IP)
 	for hostname, tailscale := range tailscales {
@@ -692,6 +727,9 @@ func getAPIURLs(tailscales map[string]dockertest.Resource) (map[netaddr.IP]strin
 			n := ft.Node
 			for _, a := range n.Addresses { // just add all the addresses
 				if _, ok := fts[a.IP()]; !ok {
+					if ft.PeerAPIURL == "" {
+						return nil, errors.New("api url is empty")
+					}
 					fts[a.IP()] = ft.PeerAPIURL
 				}
 			}

integration_test/etc/config.json

@@ -7,5 +7,13 @@
   "db_type": "sqlite3",
   "db_path": "/tmp/integration_test_db.sqlite3",
   "acl_policy_path": "",
-  "log_level": "debug"
-}
+  "log_level": "trace",
+  "dns_config": {
+    "nameservers": [
+      "1.1.1.1"
+    ],
+    "domains": [],
+    "magic_dns": true,
+    "base_domain": "headscale.net"
+  }
+}

machine.go

@@ -2,12 +2,13 @@ package headscale
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"sort"
 	"strconv"
+	"strings"
 	"time"
 
+	"github.com/fatih/set"
 	"github.com/rs/zerolog/log"
 
 	"gorm.io/datatypes"
@@ -45,14 +46,304 @@ type Machine struct {
 	DeletedAt *time.Time
 }
 
+type (
+	Machines  []Machine
+	MachinesP []*Machine
+)
+
 // For the time being this method is rather naive
 func (m Machine) isAlreadyRegistered() bool {
 	return m.Registered
 }
 
+func (h *Headscale) getDirectPeers(m *Machine) (Machines, error) {
+	log.Trace().
+		Str("func", "getDirectPeers").
+		Str("machine", m.Name).
+		Msg("Finding direct peers")
+
+	machines := Machines{}
+	if err := h.db.Preload("Namespace").Where("namespace_id = ? AND machine_key <> ? AND registered",
+		m.NamespaceID, m.MachineKey).Find(&machines).Error; err != nil {
+		log.Error().Err(err).Msg("Error accessing db")
+		return Machines{}, err
+	}
+
+	sort.Slice(machines, func(i, j int) bool { return machines[i].ID < machines[j].ID })
+
+	log.Trace().
+		Str("func", "getDirectmachines").
+		Str("machine", m.Name).
+		Msgf("Found direct machines: %s", machines.String())
+	return machines, nil
+}
+
+// getShared fetches machines that are shared to the `Namespace` of the machine we are getting peers for
+func (h *Headscale) getShared(m *Machine) (Machines, error) {
+	log.Trace().
+		Str("func", "getShared").
+		Str("machine", m.Name).
+		Msg("Finding shared peers")
+
+	sharedMachines := []SharedMachine{}
+	if err := h.db.Preload("Namespace").Preload("Machine").Where("namespace_id = ?",
+		m.NamespaceID).Find(&sharedMachines).Error; err != nil {
+		return Machines{}, err
+	}
+
+	peers := make(Machines, 0)
+	for _, sharedMachine := range sharedMachines {
+		peers = append(peers, sharedMachine.Machine)
+	}
+
+	sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID })
+
+	log.Trace().
+		Str("func", "getShared").
+		Str("machine", m.Name).
+		Msgf("Found shared peers: %s", peers.String())
+	return peers, nil
+}
+
+// getSharedTo fetches the machines of the namespaces this machine is shared in
+func (h *Headscale) getSharedTo(m *Machine) (Machines, error) {
+	log.Trace().
+		Str("func", "getSharedTo").
+		Str("machine", m.Name).
+		Msg("Finding peers in namespaces this machine is shared with")
+
+	sharedMachines := []SharedMachine{}
+	if err := h.db.Preload("Namespace").Preload("Machine").Where("machine_id = ?",
+		m.ID).Find(&sharedMachines).Error; err != nil {
+		return Machines{}, err
+	}
+
+	peers := make(Machines, 0)
+	for _, sharedMachine := range sharedMachines {
+		namespaceMachines, err := h.ListMachinesInNamespace(sharedMachine.Namespace.Name)
+		if err != nil {
+			return Machines{}, err
+		}
+		peers = append(peers, *namespaceMachines...)
+	}
+
+	sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID })
+
+	log.Trace().
+		Str("func", "getSharedTo").
+		Str("machine", m.Name).
+		Msgf("Found peers we are shared with: %s", peers.String())
+	return peers, nil
+}
+
+func (h *Headscale) getPeers(m *Machine) (Machines, error) {
+	direct, err := h.getDirectPeers(m)
+	if err != nil {
+		log.Error().
+			Str("func", "getPeers").
+			Err(err).
+			Msg("Cannot fetch peers")
+		return Machines{}, err
+	}
+
+	shared, err := h.getShared(m)
+	if err != nil {
+		log.Error().
+			Str("func", "getShared").
+			Err(err).
+			Msg("Cannot fetch peers")
+		return Machines{}, err
+	}
+
+	sharedTo, err := h.getSharedTo(m)
+	if err != nil {
+		log.Error().
+			Str("func", "sharedTo").
+			Err(err).
+			Msg("Cannot fetch peers")
+		return Machines{}, err
+	}
+
+	peers := append(direct, shared...)
+	peers = append(peers, sharedTo...)
+
+	sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID })
+
+	log.Trace().
+		Str("func", "getShared").
+		Str("machine", m.Name).
+		Msgf("Found total peers: %s", peers.String())
+
+	return peers, nil
+}
+
+// GetMachine finds a Machine by name and namespace and returns the Machine struct
+func (h *Headscale) GetMachine(namespace string, name string) (*Machine, error) {
+	machines, err := h.ListMachinesInNamespace(namespace)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, m := range *machines {
+		if m.Name == name {
+			return &m, nil
+		}
+	}
+	return nil, fmt.Errorf("machine not found")
+}
+
+// GetMachineByID finds a Machine by ID and returns the Machine struct
+func (h *Headscale) GetMachineByID(id uint64) (*Machine, error) {
+	m := Machine{}
+	if result := h.db.Preload("Namespace").Find(&Machine{ID: id}).First(&m); result.Error != nil {
+		return nil, result.Error
+	}
+	return &m, nil
+}
+
+// GetMachineByMachineKey finds a Machine by ID and returns the Machine struct
+func (h *Headscale) GetMachineByMachineKey(mKey string) (*Machine, error) {
+	m := Machine{}
+	if result := h.db.Preload("Namespace").First(&m, "machine_key = ?", mKey); result.Error != nil {
+		return nil, result.Error
+	}
+	return &m, nil
+}
+
+// UpdateMachine takes a Machine struct pointer (typically already loaded from database
+// and updates it with the latest data from the database.
+func (h *Headscale) UpdateMachine(m *Machine) error {
+	if result := h.db.Find(m).First(&m); result.Error != nil {
+		return result.Error
+	}
+	return nil
+}
+
+// DeleteMachine softs deletes a Machine from the database
+func (h *Headscale) DeleteMachine(m *Machine) error {
+	err := h.RemoveSharedMachineFromAllNamespaces(m)
+	if err != nil && err != errorMachineNotShared {
+		return err
+	}
+
+	m.Registered = false
+	namespaceID := m.NamespaceID
+	h.db.Save(&m) // we mark it as unregistered, just in case
+	if err := h.db.Delete(&m).Error; err != nil {
+		return err
+	}
+
+	return h.RequestMapUpdates(namespaceID)
+}
+
+// HardDeleteMachine hard deletes a Machine from the database
+func (h *Headscale) HardDeleteMachine(m *Machine) error {
+	err := h.RemoveSharedMachineFromAllNamespaces(m)
+	if err != nil && err != errorMachineNotShared {
+		return err
+	}
+
+	namespaceID := m.NamespaceID
+	if err := h.db.Unscoped().Delete(&m).Error; err != nil {
+		return err
+	}
+
+	return h.RequestMapUpdates(namespaceID)
+}
+
+// GetHostInfo returns a Hostinfo struct for the machine
+func (m *Machine) GetHostInfo() (*tailcfg.Hostinfo, error) {
+	hostinfo := tailcfg.Hostinfo{}
+	if len(m.HostInfo) != 0 {
+		hi, err := m.HostInfo.MarshalJSON()
+		if err != nil {
+			return nil, err
+		}
+		err = json.Unmarshal(hi, &hostinfo)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return &hostinfo, nil
+}
+
+func (h *Headscale) isOutdated(m *Machine) bool {
+	err := h.UpdateMachine(m)
+	if err != nil {
+		// It does not seem meaningful to propagate this error as the end result
+		// will have to be that the machine has to be considered outdated.
+		return true
+	}
+
+	sharedMachines, _ := h.getShared(m)
+
+	namespaceSet := set.New(set.ThreadSafe)
+	namespaceSet.Add(m.Namespace.Name)
+
+	// Check if any of our shared namespaces has updates that we have
+	// not propagated.
+	for _, sharedMachine := range sharedMachines {
+		namespaceSet.Add(sharedMachine.Namespace.Name)
+	}
+
+	namespaces := make([]string, namespaceSet.Size())
+	for index, namespace := range namespaceSet.List() {
+		namespaces[index] = namespace.(string)
+	}
+
+	lastChange := h.getLastStateChange(namespaces...)
+	log.Trace().
+		Str("func", "keepAlive").
+		Str("machine", m.Name).
+		Time("last_successful_update", *m.LastSuccessfulUpdate).
+		Time("last_state_change", lastChange).
+		Msgf("Checking if %s is missing updates", m.Name)
+	return m.LastSuccessfulUpdate.Before(lastChange)
+}
+
+func (m Machine) String() string {
+	return m.Name
+}
+
+func (ms Machines) String() string {
+	temp := make([]string, len(ms))
+
+	for index, machine := range ms {
+		temp[index] = machine.Name
+	}
+
+	return fmt.Sprintf("[ %s ](%d)", strings.Join(temp, ", "), len(temp))
+}
+
+// TODO(kradalby): Remove when we have generics...
+func (ms MachinesP) String() string {
+	temp := make([]string, len(ms))
+
+	for index, machine := range ms {
+		temp[index] = machine.Name
+	}
+
+	return fmt.Sprintf("[ %s ](%d)", strings.Join(temp, ", "), len(temp))
+}
+
+func (ms Machines) toNodes(baseDomain string, dnsConfig *tailcfg.DNSConfig, includeRoutes bool) ([]*tailcfg.Node, error) {
+	nodes := make([]*tailcfg.Node, len(ms))
+
+	for index, machine := range ms {
+		node, err := machine.toNode(baseDomain, dnsConfig, includeRoutes)
+		if err != nil {
+			return nil, err
+		}
+
+		nodes[index] = node
+	}
+
+	return nodes, nil
+}
+
 // toNode converts a Machine into a Tailscale Node. includeRoutes is false for shared nodes
 // as per the expected behaviour in the official SaaS
-func (m Machine) toNode(includeRoutes bool) (*tailcfg.Node, error) {
+func (m Machine) toNode(baseDomain string, dnsConfig *tailcfg.DNSConfig, includeRoutes bool) (*tailcfg.Node, error) {
 	nKey, err := wgkey.ParseHex(m.NodeKey)
 	if err != nil {
 		return nil, err
@@ -147,10 +438,17 @@ func (m Machine) toNode(includeRoutes bool) (*tailcfg.Node, error) {
 		keyExpiry = time.Time{}
 	}
 
+	var hostname string
+	if dnsConfig != nil && dnsConfig.Proxied { // MagicDNS
+		hostname = fmt.Sprintf("%s.%s.%s", m.Name, m.Namespace.Name, baseDomain)
+	} else {
+		hostname = m.Name
+	}
+
 	n := tailcfg.Node{
 		ID:         tailcfg.NodeID(m.ID),                               // this is the actual ID
 		StableID:   tailcfg.StableNodeID(strconv.FormatUint(m.ID, 10)), // in headscale, unlike tailcontrol server, IDs are permanent
-		Name:       hostinfo.Hostname,
+		Name:       hostname,
 		User:       tailcfg.UserID(m.NamespaceID),
 		Key:        tailcfg.NodeKey(nKey),
 		KeyExpiry:  keyExpiry,
@@ -171,235 +469,3 @@ func (m Machine) toNode(includeRoutes bool) (*tailcfg.Node, error) {
 	}
 	return &n, nil
 }
-
-func (h *Headscale) getPeers(m Machine) (*[]*tailcfg.Node, error) {
-	log.Trace().
-		Str("func", "getPeers").
-		Str("machine", m.Name).
-		Msg("Finding peers")
-
-	machines := []Machine{}
-	if err := h.db.Where("namespace_id = ? AND machine_key <> ? AND registered",
-		m.NamespaceID, m.MachineKey).Find(&machines).Error; err != nil {
-		log.Error().Err(err).Msg("Error accessing db")
-		return nil, err
-	}
-
-	// We fetch here machines that are shared to the `Namespace` of the machine we are getting peers for
-	sharedMachines := []SharedMachine{}
-	if err := h.db.Preload("Namespace").Preload("Machine").Where("namespace_id = ?",
-		m.NamespaceID).Find(&sharedMachines).Error; err != nil {
-		return nil, err
-	}
-
-	peers := []*tailcfg.Node{}
-	for _, mn := range machines {
-		peer, err := mn.toNode(true)
-		if err != nil {
-			return nil, err
-		}
-		peers = append(peers, peer)
-	}
-	for _, sharedMachine := range sharedMachines {
-		peer, err := sharedMachine.Machine.toNode(false) // shared nodes do not expose their routes
-		if err != nil {
-			return nil, err
-		}
-		peers = append(peers, peer)
-	}
-	sort.Slice(peers, func(i, j int) bool { return peers[i].ID < peers[j].ID })
-
-	log.Trace().
-		Str("func", "getPeers").
-		Str("machine", m.Name).
-		Msgf("Found peers: %s", tailNodesToString(peers))
-	return &peers, nil
-}
-
-// GetMachine finds a Machine by name and namespace and returns the Machine struct
-func (h *Headscale) GetMachine(namespace string, name string) (*Machine, error) {
-	machines, err := h.ListMachinesInNamespace(namespace)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, m := range *machines {
-		if m.Name == name {
-			return &m, nil
-		}
-	}
-	return nil, fmt.Errorf("machine not found")
-}
-
-// GetMachineByID finds a Machine by ID and returns the Machine struct
-func (h *Headscale) GetMachineByID(id uint64) (*Machine, error) {
-	m := Machine{}
-	if result := h.db.Preload("Namespace").Find(&Machine{ID: id}).First(&m); result.Error != nil {
-		return nil, result.Error
-	}
-	return &m, nil
-}
-
-// UpdateMachine takes a Machine struct pointer (typically already loaded from database
-// and updates it with the latest data from the database.
-func (h *Headscale) UpdateMachine(m *Machine) error {
-	if result := h.db.Find(m).First(&m); result.Error != nil {
-		return result.Error
-	}
-	return nil
-}
-
-// DeleteMachine softs deletes a Machine from the database
-func (h *Headscale) DeleteMachine(m *Machine) error {
-	m.Registered = false
-	namespaceID := m.NamespaceID
-	h.db.Save(&m) // we mark it as unregistered, just in case
-	if err := h.db.Delete(&m).Error; err != nil {
-		return err
-	}
-
-	return h.RequestMapUpdates(namespaceID)
-}
-
-// HardDeleteMachine hard deletes a Machine from the database
-func (h *Headscale) HardDeleteMachine(m *Machine) error {
-	namespaceID := m.NamespaceID
-	if err := h.db.Unscoped().Delete(&m).Error; err != nil {
-		return err
-	}
-	return h.RequestMapUpdates(namespaceID)
-}
-
-// GetHostInfo returns a Hostinfo struct for the machine
-func (m *Machine) GetHostInfo() (*tailcfg.Hostinfo, error) {
-	hostinfo := tailcfg.Hostinfo{}
-	if len(m.HostInfo) != 0 {
-		hi, err := m.HostInfo.MarshalJSON()
-		if err != nil {
-			return nil, err
-		}
-		err = json.Unmarshal(hi, &hostinfo)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return &hostinfo, nil
-}
-
-func (h *Headscale) notifyChangesToPeers(m *Machine) {
-	peers, err := h.getPeers(*m)
-	if err != nil {
-		log.Error().
-			Str("func", "notifyChangesToPeers").
-			Str("machine", m.Name).
-			Msgf("Error getting peers: %s", err)
-		return
-	}
-	for _, p := range *peers {
-		log.Info().
-			Str("func", "notifyChangesToPeers").
-			Str("machine", m.Name).
-			Str("peer", p.Name).
-			Str("address", p.Addresses[0].String()).
-			Msgf("Notifying peer %s (%s)", p.Name, p.Addresses[0])
-		err := h.sendRequestOnUpdateChannel(p)
-		if err != nil {
-			log.Info().
-				Str("func", "notifyChangesToPeers").
-				Str("machine", m.Name).
-				Str("peer", p.Name).
-				Msgf("Peer %s does not appear to be polling", p.Name)
-		}
-		log.Trace().
-			Str("func", "notifyChangesToPeers").
-			Str("machine", m.Name).
-			Str("peer", p.Name).
-			Str("address", p.Addresses[0].String()).
-			Msgf("Notified peer %s (%s)", p.Name, p.Addresses[0])
-	}
-}
-
-func (h *Headscale) getOrOpenUpdateChannel(m *Machine) <-chan struct{} {
-	var updateChan chan struct{}
-	if storedChan, ok := h.clientsUpdateChannels.Load(m.ID); ok {
-		if unwrapped, ok := storedChan.(chan struct{}); ok {
-			updateChan = unwrapped
-		} else {
-			log.Error().
-				Str("handler", "openUpdateChannel").
-				Str("machine", m.Name).
-				Msg("Failed to convert update channel to struct{}")
-		}
-	} else {
-		log.Debug().
-			Str("handler", "openUpdateChannel").
-			Str("machine", m.Name).
-			Msg("Update channel not found, creating")
-
-		updateChan = make(chan struct{})
-		h.clientsUpdateChannels.Store(m.ID, updateChan)
-	}
-	return updateChan
-}
-
-func (h *Headscale) closeUpdateChannel(m *Machine) {
-	h.clientsUpdateChannelMutex.Lock()
-	defer h.clientsUpdateChannelMutex.Unlock()
-
-	if storedChan, ok := h.clientsUpdateChannels.Load(m.ID); ok {
-		if unwrapped, ok := storedChan.(chan struct{}); ok {
-			close(unwrapped)
-		}
-	}
-	h.clientsUpdateChannels.Delete(m.ID)
-}
-
-func (h *Headscale) sendRequestOnUpdateChannel(m *tailcfg.Node) error {
-	h.clientsUpdateChannelMutex.Lock()
-	defer h.clientsUpdateChannelMutex.Unlock()
-
-	pUp, ok := h.clientsUpdateChannels.Load(uint64(m.ID))
-	if ok {
-		log.Info().
-			Str("func", "requestUpdate").
-			Str("machine", m.Name).
-			Msgf("Notifying peer %s", m.Name)
-
-		if update, ok := pUp.(chan struct{}); ok {
-			log.Trace().
-				Str("func", "requestUpdate").
-				Str("machine", m.Name).
-				Msgf("Update channel is %#v", update)
-
-			update <- struct{}{}
-
-			log.Trace().
-				Str("func", "requestUpdate").
-				Str("machine", m.Name).
-				Msgf("Notified machine %s", m.Name)
-		}
-	} else {
-		log.Info().
-			Str("func", "requestUpdate").
-			Str("machine", m.Name).
-			Msgf("Machine %s does not appear to be polling", m.Name)
-		return errors.New("machine does not seem to be polling")
-	}
-	return nil
-}
-
-func (h *Headscale) isOutdated(m *Machine) bool {
-	err := h.UpdateMachine(m)
-	if err != nil {
-		return true
-	}
-
-	lastChange := h.getLastStateChange(m.Namespace.Name)
-	log.Trace().
-		Str("func", "keepAlive").
-		Str("machine", m.Name).
-		Time("last_successful_update", *m.LastSuccessfulUpdate).
-		Time("last_state_change", lastChange).
-		Msgf("Checking if %s is missing updates", m.Name)
-	return m.LastSuccessfulUpdate.Before(lastChange)
-}

machine_test.go

@@ -2,6 +2,7 @@ package headscale
 
 import (
 	"encoding/json"
+	"strconv"
 
 	"gopkg.in/check.v1"
 )
@@ -16,7 +17,7 @@ func (s *Suite) TestGetMachine(c *check.C) {
 	_, err = h.GetMachine("test", "testmachine")
 	c.Assert(err, check.NotNil)
 
-	m := Machine{
+	m := &Machine{
 		ID:             0,
 		MachineKey:     "foo",
 		NodeKey:        "bar",
@@ -27,7 +28,7 @@ func (s *Suite) TestGetMachine(c *check.C) {
 		RegisterMethod: "authKey",
 		AuthKeyID:      uint(pak.ID),
 	}
-	h.db.Save(&m)
+	h.db.Save(m)
 
 	m1, err := h.GetMachine("test", "testmachine")
 	c.Assert(err, check.IsNil)
@@ -116,3 +117,43 @@ func (s *Suite) TestHardDeleteMachine(c *check.C) {
 	_, err = h.GetMachine(n.Name, "testmachine3")
 	c.Assert(err, check.NotNil)
 }
+
+func (s *Suite) TestGetDirectPeers(c *check.C) {
+	n, err := h.CreateNamespace("test")
+	c.Assert(err, check.IsNil)
+
+	pak, err := h.CreatePreAuthKey(n.Name, false, false, nil)
+	c.Assert(err, check.IsNil)
+
+	_, err = h.GetMachineByID(0)
+	c.Assert(err, check.NotNil)
+
+	for i := 0; i <= 10; i++ {
+		m := Machine{
+			ID:             uint64(i),
+			MachineKey:     "foo" + strconv.Itoa(i),
+			NodeKey:        "bar" + strconv.Itoa(i),
+			DiscoKey:       "faa" + strconv.Itoa(i),
+			Name:           "testmachine" + strconv.Itoa(i),
+			NamespaceID:    n.ID,
+			Registered:     true,
+			RegisterMethod: "authKey",
+			AuthKeyID:      uint(pak.ID),
+		}
+		h.db.Save(&m)
+	}
+
+	m1, err := h.GetMachineByID(0)
+	c.Assert(err, check.IsNil)
+
+	_, err = m1.GetHostInfo()
+	c.Assert(err, check.IsNil)
+
+	peers, err := h.getDirectPeers(m1)
+	c.Assert(err, check.IsNil)
+
+	c.Assert(len(peers), check.Equals, 9)
+	c.Assert(peers[0].Name, check.Equals, "testmachine2")
+	c.Assert(peers[5].Name, check.Equals, "testmachine7")
+	c.Assert(peers[8].Name, check.Equals, "testmachine10")
+}

metrics.go

@@ -0,0 +1,41 @@
+package headscale
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+const prometheusNamespace = "headscale"
+
+var (
+	// This is a high cardinality metric (namespace x machines), we might want to make this
+	// configurable/opt-in in the future.
+	lastStateUpdate = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: prometheusNamespace,
+		Name:      "last_update_seconds",
+		Help:      "Time stamp in unix time when a machine or headscale was updated",
+	}, []string{"namespace", "machine"})
+
+	machineRegistrations = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Name:      "machine_registrations_total",
+		Help:      "The total amount of registered machine attempts",
+	}, []string{"action", "auth", "status", "namespace"})
+
+	updateRequestsFromNode = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Name:      "update_request_from_node_total",
+		Help:      "The number of updates requested by a node/update function",
+	}, []string{"namespace", "machine", "state"})
+	updateRequestsSentToNode = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Name:      "update_request_sent_to_node_total",
+		Help:      "The number of calls/messages issued on a specific nodes update channel",
+	}, []string{"namespace", "machine", "status"})
+	//TODO(kradalby): This is very debugging, we might want to remove it.
+	updateRequestsReceivedOnChannel = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Name:      "update_request_received_on_channel_total",
+		Help:      "The number of update requests received on an update channel",
+	}, []string{"namespace", "machine"})
+)

namespaces.go

@@ -91,7 +91,7 @@ func (h *Headscale) ListMachinesInNamespace(name string) (*[]Machine, error) {
 	}
 
 	machines := []Machine{}
-	if err := h.db.Preload("AuthKey").Preload("Namespace").Where(&Machine{NamespaceID: n.ID}).Find(&machines).Error; err != nil {
+	if err := h.db.Preload("AuthKey").Preload("AuthKey.Namespace").Preload("Namespace").Where(&Machine{NamespaceID: n.ID}).Find(&machines).Error; err != nil {
 		return nil, err
 	}
 	return &machines, nil
@@ -176,23 +176,17 @@ func (h *Headscale) checkForNamespacesPendingUpdates() {
 		return
 	}
 
-	names := []string{}
-	err = json.Unmarshal([]byte(v), &names)
+	namespaces := []string{}
+	err = json.Unmarshal([]byte(v), &namespaces)
 	if err != nil {
 		return
 	}
-	for _, name := range names {
+	for _, namespace := range namespaces {
 		log.Trace().
 			Str("func", "RequestMapUpdates").
-			Str("machine", name).
-			Msg("Sending updates to nodes in namespace")
-		machines, err := h.ListMachinesInNamespace(name)
-		if err != nil {
-			continue
-		}
-		for _, m := range *machines {
-			h.notifyChangesToPeers(&m)
-		}
+			Str("machine", namespace).
+			Msg("Sending updates to nodes in namespacespace")
+		h.setLastStateChangeToNow(namespace)
 	}
 	newV, err := h.getValue("namespaces_pending_updates")
 	if err != nil {

poll.go

@@ -51,13 +51,19 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 		return
 	}
 
-	var m Machine
-	if result := h.db.Preload("Namespace").First(&m, "machine_key = ?", mKey.HexString()); errors.Is(result.Error, gorm.ErrRecordNotFound) {
-		log.Warn().
+	m, err := h.GetMachineByMachineKey(mKey.HexString())
+	if err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			log.Warn().
+				Str("handler", "PollNetMap").
+				Msgf("Ignoring request, cannot find machine with key %s", mKey.HexString())
+			c.String(http.StatusUnauthorized, "")
+			return
+		}
+		log.Error().
 			Str("handler", "PollNetMap").
-			Msgf("Ignoring request, cannot find machine with key %s", mKey.HexString())
-		c.String(http.StatusUnauthorized, "")
-		return
+			Msgf("Failed to fetch machine from the database with Machine key: %s", mKey.HexString())
+		c.String(http.StatusInternalServerError, "")
 	}
 	log.Trace().
 		Str("handler", "PollNetMap").
@@ -117,7 +123,7 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 			Str("handler", "PollNetMap").
 			Str("machine", m.Name).
 			Msg("Client is starting up. Probably interested in a DERP map")
-		c.Data(200, "application/json; charset=utf-8", *data)
+		c.Data(200, "application/json; charset=utf-8", data)
 		return
 	}
 
@@ -134,10 +140,9 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 		Str("id", c.Param("id")).
 		Str("machine", m.Name).
 		Msg("Loading or creating update channel")
-	updateChan := h.getOrOpenUpdateChannel(&m)
+	updateChan := make(chan struct{})
 
 	pollDataChan := make(chan []byte)
-	// defer close(pollData)
 
 	keepAliveChan := make(chan []byte)
 
@@ -149,11 +154,12 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 			Str("handler", "PollNetMap").
 			Str("machine", m.Name).
 			Msg("Client sent endpoint update and is ok with a response without peer list")
-		c.Data(200, "application/json; charset=utf-8", *data)
+		c.Data(200, "application/json; charset=utf-8", data)
 
 		// It sounds like we should update the nodes when we have received a endpoint update
 		// even tho the comments in the tailscale code dont explicitly say so.
-		go h.notifyChangesToPeers(&m)
+		updateRequestsFromNode.WithLabelValues(m.Name, m.Namespace.Name, "endpoint-update").Inc()
+		go func() { updateChan <- struct{}{} }()
 		return
 	} else if req.OmitPeers && req.Stream {
 		log.Warn().
@@ -172,13 +178,14 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 		Str("handler", "PollNetMap").
 		Str("machine", m.Name).
 		Msg("Sending initial map")
-	go func() { pollDataChan <- *data }()
+	go func() { pollDataChan <- data }()
 
 	log.Info().
 		Str("handler", "PollNetMap").
 		Str("machine", m.Name).
 		Msg("Notifying peers")
-	go h.notifyChangesToPeers(&m)
+	updateRequestsFromNode.WithLabelValues(m.Name, m.Namespace.Name, "full-update").Inc()
+	go func() { updateChan <- struct{}{} }()
 
 	h.PollNetMapStream(c, m, req, mKey, pollDataChan, keepAliveChan, updateChan, cancelKeepAlive)
 	log.Trace().
@@ -193,15 +200,15 @@ func (h *Headscale) PollNetMapHandler(c *gin.Context) {
 // to the connected clients.
 func (h *Headscale) PollNetMapStream(
 	c *gin.Context,
-	m Machine,
+	m *Machine,
 	req tailcfg.MapRequest,
 	mKey wgkey.Key,
 	pollDataChan chan []byte,
 	keepAliveChan chan []byte,
-	updateChan <-chan struct{},
+	updateChan chan struct{},
 	cancelKeepAlive chan struct{},
 ) {
-	go h.scheduledPollWorker(cancelKeepAlive, keepAliveChan, mKey, req, m)
+	go h.scheduledPollWorker(cancelKeepAlive, updateChan, keepAliveChan, mKey, req, m)
 
 	c.Stream(func(w io.Writer) bool {
 		log.Trace().
@@ -230,6 +237,7 @@ func (h *Headscale) PollNetMapStream(
 					Str("channel", "pollData").
 					Err(err).
 					Msg("Cannot write data")
+				return false
 			}
 			log.Trace().
 				Str("handler", "PollNetMapStream").
@@ -237,10 +245,10 @@ func (h *Headscale) PollNetMapStream(
 				Str("channel", "pollData").
 				Int("bytes", len(data)).
 				Msg("Data from pollData channel written successfully")
-				// TODO: Abstract away all the database calls, this can cause race conditions
+				// TODO(kradalby): Abstract away all the database calls, this can cause race conditions
 				// when an outdated machine object is kept alive, e.g. db is update from
 				// command line, but then overwritten.
-			err = h.UpdateMachine(&m)
+			err = h.UpdateMachine(m)
 			if err != nil {
 				log.Error().
 					Str("handler", "PollNetMapStream").
@@ -251,14 +259,17 @@ func (h *Headscale) PollNetMapStream(
 			}
 			now := time.Now().UTC()
 			m.LastSeen = &now
+
+			lastStateUpdate.WithLabelValues(m.Namespace.Name, m.Name).Set(float64(now.Unix()))
 			m.LastSuccessfulUpdate = &now
+
 			h.db.Save(&m)
 			log.Trace().
 				Str("handler", "PollNetMapStream").
 				Str("machine", m.Name).
 				Str("channel", "pollData").
 				Int("bytes", len(data)).
-				Msg("Machine updated successfully after sending pollData")
+				Msg("Machine entry in database updated successfully after sending pollData")
 			return true
 
 		case data := <-keepAliveChan:
@@ -276,6 +287,7 @@ func (h *Headscale) PollNetMapStream(
 					Str("channel", "keepAlive").
 					Err(err).
 					Msg("Cannot write keep alive message")
+				return false
 			}
 			log.Trace().
 				Str("handler", "PollNetMapStream").
@@ -283,10 +295,10 @@ func (h *Headscale) PollNetMapStream(
 				Str("channel", "keepAlive").
 				Int("bytes", len(data)).
 				Msg("Keep alive sent successfully")
-				// TODO: Abstract away all the database calls, this can cause race conditions
+				// TODO(kradalby): Abstract away all the database calls, this can cause race conditions
 				// when an outdated machine object is kept alive, e.g. db is update from
 				// command line, but then overwritten.
-			err = h.UpdateMachine(&m)
+			err = h.UpdateMachine(m)
 			if err != nil {
 				log.Error().
 					Str("handler", "PollNetMapStream").
@@ -312,7 +324,8 @@ func (h *Headscale) PollNetMapStream(
 				Str("machine", m.Name).
 				Str("channel", "update").
 				Msg("Received a request for update")
-			if h.isOutdated(&m) {
+			updateRequestsReceivedOnChannel.WithLabelValues(m.Name, m.Namespace.Name).Inc()
+			if h.isOutdated(m) {
 				log.Debug().
 					Str("handler", "PollNetMapStream").
 					Str("machine", m.Name).
@@ -328,7 +341,7 @@ func (h *Headscale) PollNetMapStream(
 						Err(err).
 						Msg("Could not get the map update")
 				}
-				_, err = w.Write(*data)
+				_, err = w.Write(data)
 				if err != nil {
 					log.Error().
 						Str("handler", "PollNetMapStream").
@@ -336,21 +349,24 @@ func (h *Headscale) PollNetMapStream(
 						Str("channel", "update").
 						Err(err).
 						Msg("Could not write the map response")
+					updateRequestsSentToNode.WithLabelValues(m.Name, m.Namespace.Name, "failed").Inc()
+					return false
 				}
 				log.Trace().
 					Str("handler", "PollNetMapStream").
 					Str("machine", m.Name).
 					Str("channel", "update").
 					Msg("Updated Map has been sent")
+				updateRequestsSentToNode.WithLabelValues(m.Name, m.Namespace.Name, "success").Inc()
 
-					// Keep track of the last successful update,
-					// we sometimes end in a state were the update
-					// is not picked up by a client and we use this
-					// to determine if we should "force" an update.
-					// TODO: Abstract away all the database calls, this can cause race conditions
-					// when an outdated machine object is kept alive, e.g. db is update from
-					// command line, but then overwritten.
-				err = h.UpdateMachine(&m)
+				// Keep track of the last successful update,
+				// we sometimes end in a state were the update
+				// is not picked up by a client and we use this
+				// to determine if we should "force" an update.
+				// TODO(kradalby): Abstract away all the database calls, this can cause race conditions
+				// when an outdated machine object is kept alive, e.g. db is update from
+				// command line, but then overwritten.
+				err = h.UpdateMachine(m)
 				if err != nil {
 					log.Error().
 						Str("handler", "PollNetMapStream").
@@ -360,7 +376,10 @@ func (h *Headscale) PollNetMapStream(
 						Msg("Cannot update machine from database")
 				}
 				now := time.Now().UTC()
+
+				lastStateUpdate.WithLabelValues(m.Namespace.Name, m.Name).Set(float64(now.Unix()))
 				m.LastSuccessfulUpdate = &now
+
 				h.db.Save(&m)
 			} else {
 				log.Trace().
@@ -380,7 +399,7 @@ func (h *Headscale) PollNetMapStream(
 				// TODO: Abstract away all the database calls, this can cause race conditions
 				// when an outdated machine object is kept alive, e.g. db is update from
 				// command line, but then overwritten.
-			err := h.UpdateMachine(&m)
+			err := h.UpdateMachine(m)
 			if err != nil {
 				log.Error().
 					Str("handler", "PollNetMapStream").
@@ -393,12 +412,33 @@ func (h *Headscale) PollNetMapStream(
 			m.LastSeen = &now
 			h.db.Save(&m)
 
+			log.Trace().
+				Str("handler", "PollNetMapStream").
+				Str("machine", m.Name).
+				Str("channel", "Done").
+				Msg("Cancelling keepAlive channel")
 			cancelKeepAlive <- struct{}{}
 
-			h.closeUpdateChannel(&m)
+			log.Trace().
+				Str("handler", "PollNetMapStream").
+				Str("machine", m.Name).
+				Str("channel", "Done").
+				Msg("Closing update channel")
+			//h.closeUpdateChannel(m)
+			close(updateChan)
 
+			log.Trace().
+				Str("handler", "PollNetMapStream").
+				Str("machine", m.Name).
+				Str("channel", "Done").
+				Msg("Closing pollData channel")
 			close(pollDataChan)
 
+			log.Trace().
+				Str("handler", "PollNetMapStream").
+				Str("machine", m.Name).
+				Str("channel", "Done").
+				Msg("Closing keepAliveChan channel")
 			close(keepAliveChan)
 
 			return false
@@ -408,13 +448,14 @@ func (h *Headscale) PollNetMapStream(
 
 func (h *Headscale) scheduledPollWorker(
 	cancelChan <-chan struct{},
+	updateChan chan<- struct{},
 	keepAliveChan chan<- []byte,
 	mKey wgkey.Key,
 	req tailcfg.MapRequest,
-	m Machine,
+	m *Machine,
 ) {
 	keepAliveTicker := time.NewTicker(60 * time.Second)
-	updateCheckerTicker := time.NewTicker(30 * time.Second)
+	updateCheckerTicker := time.NewTicker(10 * time.Second)
 
 	for {
 		select {
@@ -435,20 +476,15 @@ func (h *Headscale) scheduledPollWorker(
 				Str("func", "keepAlive").
 				Str("machine", m.Name).
 				Msg("Sending keepalive")
-			keepAliveChan <- *data
+			keepAliveChan <- data
 
 		case <-updateCheckerTicker.C:
-			// Send an update request regardless of outdated or not, if data is sent
-			// to the node is determined in the updateChan consumer block
-			n, _ := m.toNode(true)
-			err := h.sendRequestOnUpdateChannel(n)
-			if err != nil {
-				log.Error().
-					Str("func", "keepAlive").
-					Str("machine", m.Name).
-					Err(err).
-					Msgf("Failed to send update request to %s", m.Name)
-			}
+			log.Debug().
+				Str("func", "scheduledPollWorker").
+				Str("machine", m.Name).
+				Msg("Sending update request")
+			updateRequestsFromNode.WithLabelValues(m.Name, m.Namespace.Name, "scheduled-update").Inc()
+			updateChan <- struct{}{}
 		}
 	}
 }

preauth_keys.go

@@ -11,7 +11,7 @@ import (
 
 const errorAuthKeyNotFound = Error("AuthKey not found")
 const errorAuthKeyExpired = Error("AuthKey expired")
-const errorAuthKeyNotReusableAlreadyUsed = Error("AuthKey not reusable already used")
+const errSingleUseAuthKeyHasBeenUsed = Error("AuthKey has already been used")
 
 // PreAuthKey describes a pre-authorization key usable in a particular namespace
 type PreAuthKey struct {
@@ -21,6 +21,7 @@ type PreAuthKey struct {
 	Namespace   Namespace
 	Reusable    bool
 	Ephemeral   bool `gorm:"default:false"`
+	Used        bool `gorm:"default:false"`
 
 	CreatedAt  *time.Time
 	Expiration *time.Time
@@ -110,11 +111,10 @@ func (h *Headscale) checkKeyValidity(k string) (*PreAuthKey, error) {
 		return nil, err
 	}
 
-	if len(machines) != 0 {
-		return nil, errorAuthKeyNotReusableAlreadyUsed
+	if len(machines) != 0 || pak.Used {
+		return nil, errSingleUseAuthKeyHasBeenUsed
 	}
 
-	// missing here validation on current usage
 	return &pak, nil
 }
 

preauth_keys_test.go

@@ -87,7 +87,7 @@ func (*Suite) TestAlreadyUsedKey(c *check.C) {
 	h.db.Save(&m)
 
 	p, err := h.checkKeyValidity(pak.Key)
-	c.Assert(err, check.Equals, errorAuthKeyNotReusableAlreadyUsed)
+	c.Assert(err, check.Equals, errSingleUseAuthKeyHasBeenUsed)
 	c.Assert(p, check.IsNil)
 }
 
@@ -180,3 +180,16 @@ func (*Suite) TestExpirePreauthKey(c *check.C) {
 	c.Assert(err, check.Equals, errorAuthKeyExpired)
 	c.Assert(p, check.IsNil)
 }
+
+func (*Suite) TestNotReusableMarkedAsUsed(c *check.C) {
+	n, err := h.CreateNamespace("test6")
+	c.Assert(err, check.IsNil)
+
+	pak, err := h.CreatePreAuthKey(n.Name, false, false, nil)
+	c.Assert(err, check.IsNil)
+	pak.Used = true
+	h.db.Save(&pak)
+
+	_, err = h.checkKeyValidity(pak.Key)
+	c.Assert(err, check.Equals, errSingleUseAuthKeyHasBeenUsed)
+}

sharing.go

@@ -4,6 +4,7 @@ import "gorm.io/gorm"
 
 const errorSameNamespace = Error("Destination namespace same as origin")
 const errorMachineAlreadyShared = Error("Node already shared to this namespace")
+const errorMachineNotShared = Error("Machine not shared to this namespace")
 
 // SharedMachine is a join table to support sharing nodes between namespaces
 type SharedMachine struct {
@@ -35,3 +36,13 @@ func (h *Headscale) AddSharedMachineToNamespace(m *Machine, ns *Namespace) error
 
 	return nil
 }
+
+// RemoveSharedMachineFromAllNamespaces removes a machine as a shared node from all namespaces
+func (h *Headscale) RemoveSharedMachineFromAllNamespaces(m *Machine) error {
+	sharedMachine := SharedMachine{}
+	if result := h.db.Where("machine_id = ?", m.ID).Unscoped().Delete(&sharedMachine); result.Error != nil {
+		return result.Error
+	}
+
+	return nil
+}

sharing_test.go

@@ -2,7 +2,6 @@ package headscale
 
 import (
 	"gopkg.in/check.v1"
-	"tailscale.com/tailcfg"
 )
 
 func (s *Suite) TestBasicSharedNodesInNamespace(c *check.C) {
@@ -21,7 +20,7 @@ func (s *Suite) TestBasicSharedNodesInNamespace(c *check.C) {
 	_, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1")
 	c.Assert(err, check.NotNil)
 
-	m1 := Machine{
+	m1 := &Machine{
 		ID:             0,
 		MachineKey:     "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
 		NodeKey:        "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
@@ -33,12 +32,12 @@ func (s *Suite) TestBasicSharedNodesInNamespace(c *check.C) {
 		IPAddress:      "100.64.0.1",
 		AuthKeyID:      uint(pak1.ID),
 	}
-	h.db.Save(&m1)
+	h.db.Save(m1)
 
 	_, err = h.GetMachine(n1.Name, m1.Name)
 	c.Assert(err, check.IsNil)
 
-	m2 := Machine{
+	m2 := &Machine{
 		ID:             1,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -50,22 +49,22 @@ func (s *Suite) TestBasicSharedNodesInNamespace(c *check.C) {
 		IPAddress:      "100.64.0.2",
 		AuthKeyID:      uint(pak2.ID),
 	}
-	h.db.Save(&m2)
+	h.db.Save(m2)
 
 	_, err = h.GetMachine(n2.Name, m2.Name)
 	c.Assert(err, check.IsNil)
 
 	p1s, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1s), check.Equals, 0)
+	c.Assert(len(p1s), check.Equals, 0)
 
-	err = h.AddSharedMachineToNamespace(&m2, n1)
+	err = h.AddSharedMachineToNamespace(m2, n1)
 	c.Assert(err, check.IsNil)
 
 	p1sAfter, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1sAfter), check.Equals, 1)
-	c.Assert((*p1sAfter)[0].ID, check.Equals, tailcfg.NodeID(m2.ID))
+	c.Assert(len(p1sAfter), check.Equals, 1)
+	c.Assert(p1sAfter[0].ID, check.Equals, m2.ID)
 }
 
 func (s *Suite) TestSameNamespace(c *check.C) {
@@ -84,7 +83,7 @@ func (s *Suite) TestSameNamespace(c *check.C) {
 	_, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1")
 	c.Assert(err, check.NotNil)
 
-	m1 := Machine{
+	m1 := &Machine{
 		ID:             0,
 		MachineKey:     "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
 		NodeKey:        "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
@@ -96,12 +95,12 @@ func (s *Suite) TestSameNamespace(c *check.C) {
 		IPAddress:      "100.64.0.1",
 		AuthKeyID:      uint(pak1.ID),
 	}
-	h.db.Save(&m1)
+	h.db.Save(m1)
 
 	_, err = h.GetMachine(n1.Name, m1.Name)
 	c.Assert(err, check.IsNil)
 
-	m2 := Machine{
+	m2 := &Machine{
 		ID:             1,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -113,16 +112,16 @@ func (s *Suite) TestSameNamespace(c *check.C) {
 		IPAddress:      "100.64.0.2",
 		AuthKeyID:      uint(pak2.ID),
 	}
-	h.db.Save(&m2)
+	h.db.Save(m2)
 
 	_, err = h.GetMachine(n2.Name, m2.Name)
 	c.Assert(err, check.IsNil)
 
 	p1s, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1s), check.Equals, 0)
+	c.Assert(len(p1s), check.Equals, 0)
 
-	err = h.AddSharedMachineToNamespace(&m1, n1)
+	err = h.AddSharedMachineToNamespace(m1, n1)
 	c.Assert(err, check.Equals, errorSameNamespace)
 }
 
@@ -142,7 +141,7 @@ func (s *Suite) TestAlreadyShared(c *check.C) {
 	_, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1")
 	c.Assert(err, check.NotNil)
 
-	m1 := Machine{
+	m1 := &Machine{
 		ID:             0,
 		MachineKey:     "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
 		NodeKey:        "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
@@ -154,12 +153,12 @@ func (s *Suite) TestAlreadyShared(c *check.C) {
 		IPAddress:      "100.64.0.1",
 		AuthKeyID:      uint(pak1.ID),
 	}
-	h.db.Save(&m1)
+	h.db.Save(m1)
 
 	_, err = h.GetMachine(n1.Name, m1.Name)
 	c.Assert(err, check.IsNil)
 
-	m2 := Machine{
+	m2 := &Machine{
 		ID:             1,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -171,18 +170,18 @@ func (s *Suite) TestAlreadyShared(c *check.C) {
 		IPAddress:      "100.64.0.2",
 		AuthKeyID:      uint(pak2.ID),
 	}
-	h.db.Save(&m2)
+	h.db.Save(m2)
 
 	_, err = h.GetMachine(n2.Name, m2.Name)
 	c.Assert(err, check.IsNil)
 
 	p1s, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1s), check.Equals, 0)
+	c.Assert(len(p1s), check.Equals, 0)
 
-	err = h.AddSharedMachineToNamespace(&m2, n1)
+	err = h.AddSharedMachineToNamespace(m2, n1)
 	c.Assert(err, check.IsNil)
-	err = h.AddSharedMachineToNamespace(&m2, n1)
+	err = h.AddSharedMachineToNamespace(m2, n1)
 	c.Assert(err, check.Equals, errorMachineAlreadyShared)
 }
 
@@ -202,7 +201,7 @@ func (s *Suite) TestDoNotIncludeRoutesOnShared(c *check.C) {
 	_, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1")
 	c.Assert(err, check.NotNil)
 
-	m1 := Machine{
+	m1 := &Machine{
 		ID:             0,
 		MachineKey:     "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
 		NodeKey:        "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
@@ -214,12 +213,12 @@ func (s *Suite) TestDoNotIncludeRoutesOnShared(c *check.C) {
 		IPAddress:      "100.64.0.1",
 		AuthKeyID:      uint(pak1.ID),
 	}
-	h.db.Save(&m1)
+	h.db.Save(m1)
 
 	_, err = h.GetMachine(n1.Name, m1.Name)
 	c.Assert(err, check.IsNil)
 
-	m2 := Machine{
+	m2 := &Machine{
 		ID:             1,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -231,22 +230,22 @@ func (s *Suite) TestDoNotIncludeRoutesOnShared(c *check.C) {
 		IPAddress:      "100.64.0.2",
 		AuthKeyID:      uint(pak2.ID),
 	}
-	h.db.Save(&m2)
+	h.db.Save(m2)
 
 	_, err = h.GetMachine(n2.Name, m2.Name)
 	c.Assert(err, check.IsNil)
 
 	p1s, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1s), check.Equals, 0)
+	c.Assert(len(p1s), check.Equals, 0)
 
-	err = h.AddSharedMachineToNamespace(&m2, n1)
+	err = h.AddSharedMachineToNamespace(m2, n1)
 	c.Assert(err, check.IsNil)
 
 	p1sAfter, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1sAfter), check.Equals, 1)
-	c.Assert(len((*p1sAfter)[0].AllowedIPs), check.Equals, 1)
+	c.Assert(len(p1sAfter), check.Equals, 1)
+	c.Assert(p1sAfter[0].Name, check.Equals, "test_get_shared_nodes_2")
 }
 
 func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
@@ -274,8 +273,8 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 	_, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1")
 	c.Assert(err, check.NotNil)
 
-	m1 := Machine{
-		ID:             0,
+	m1 := &Machine{
+		ID:             1,
 		MachineKey:     "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
 		NodeKey:        "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
 		DiscoKey:       "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
@@ -286,13 +285,13 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 		IPAddress:      "100.64.0.1",
 		AuthKeyID:      uint(pak1.ID),
 	}
-	h.db.Save(&m1)
+	h.db.Save(m1)
 
 	_, err = h.GetMachine(n1.Name, m1.Name)
 	c.Assert(err, check.IsNil)
 
-	m2 := Machine{
-		ID:             1,
+	m2 := &Machine{
+		ID:             2,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		DiscoKey:       "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -303,13 +302,13 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 		IPAddress:      "100.64.0.2",
 		AuthKeyID:      uint(pak2.ID),
 	}
-	h.db.Save(&m2)
+	h.db.Save(m2)
 
 	_, err = h.GetMachine(n2.Name, m2.Name)
 	c.Assert(err, check.IsNil)
 
-	m3 := Machine{
-		ID:             2,
+	m3 := &Machine{
+		ID:             3,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		DiscoKey:       "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -320,13 +319,13 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 		IPAddress:      "100.64.0.3",
 		AuthKeyID:      uint(pak3.ID),
 	}
-	h.db.Save(&m3)
+	h.db.Save(m3)
 
 	_, err = h.GetMachine(n3.Name, m3.Name)
 	c.Assert(err, check.IsNil)
 
-	m4 := Machine{
-		ID:             3,
+	m4 := &Machine{
+		ID:             4,
 		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
 		DiscoKey:       "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
@@ -337,23 +336,165 @@ func (s *Suite) TestComplexSharingAcrossNamespaces(c *check.C) {
 		IPAddress:      "100.64.0.4",
 		AuthKeyID:      uint(pak4.ID),
 	}
-	h.db.Save(&m4)
+	h.db.Save(m4)
 
 	_, err = h.GetMachine(n1.Name, m4.Name)
 	c.Assert(err, check.IsNil)
 
 	p1s, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1s), check.Equals, 1) // nodes 1 and 4
+	c.Assert(len(p1s), check.Equals, 1) // node1 can see node4
+	c.Assert(p1s[0].Name, check.Equals, "test_get_shared_nodes_4")
 
-	err = h.AddSharedMachineToNamespace(&m2, n1)
+	err = h.AddSharedMachineToNamespace(m2, n1)
 	c.Assert(err, check.IsNil)
 
 	p1sAfter, err := h.getPeers(m1)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*p1sAfter), check.Equals, 2) // nodes 1, 2, 4
+	c.Assert(len(p1sAfter), check.Equals, 2) // node1 can see node2 (shared) and node4 (same namespace)
+	c.Assert(p1sAfter[0].Name, check.Equals, "test_get_shared_nodes_2")
+	c.Assert(p1sAfter[1].Name, check.Equals, "test_get_shared_nodes_4")
+
+	node1shared, err := h.getShared(m1)
+	c.Assert(err, check.IsNil)
+	c.Assert(len(node1shared), check.Equals, 1) // node1 can see node2 as shared
+	c.Assert(node1shared[0].Name, check.Equals, "test_get_shared_nodes_2")
 
 	pAlone, err := h.getPeers(m3)
 	c.Assert(err, check.IsNil)
-	c.Assert(len(*pAlone), check.Equals, 0) // node 3 is alone
+	c.Assert(len(pAlone), check.Equals, 0) // node3 is alone
+
+	pSharedTo, err := h.getPeers(m2)
+	c.Assert(err, check.IsNil)
+	c.Assert(len(pSharedTo), check.Equals, 2) // node2 should see node1 (sharedTo) and node4 (sharedTo), as is shared in namespace1
+	c.Assert(pSharedTo[0].Name, check.Equals, "test_get_shared_nodes_1")
+	c.Assert(pSharedTo[1].Name, check.Equals, "test_get_shared_nodes_4")
+}
+
+func (s *Suite) TestDeleteSharedMachine(c *check.C) {
+	n1, err := h.CreateNamespace("shared1")
+	c.Assert(err, check.IsNil)
+
+	n2, err := h.CreateNamespace("shared2")
+	c.Assert(err, check.IsNil)
+
+	n3, err := h.CreateNamespace("shared3")
+	c.Assert(err, check.IsNil)
+
+	pak1n1, err := h.CreatePreAuthKey(n1.Name, false, false, nil)
+	c.Assert(err, check.IsNil)
+
+	pak2n2, err := h.CreatePreAuthKey(n2.Name, false, false, nil)
+	c.Assert(err, check.IsNil)
+
+	pak3n3, err := h.CreatePreAuthKey(n3.Name, false, false, nil)
+	c.Assert(err, check.IsNil)
+
+	pak4n1, err := h.CreatePreAuthKey(n1.Name, false, false, nil)
+	c.Assert(err, check.IsNil)
+
+	_, err = h.GetMachine(n1.Name, "test_get_shared_nodes_1")
+	c.Assert(err, check.NotNil)
+
+	m1 := &Machine{
+		ID:             1,
+		MachineKey:     "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
+		NodeKey:        "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
+		DiscoKey:       "686824e749f3b7f2a5927ee6c1e422aee5292592d9179a271ed7b3e659b44a66",
+		Name:           "test_get_shared_nodes_1",
+		NamespaceID:    n1.ID,
+		Registered:     true,
+		RegisterMethod: "authKey",
+		IPAddress:      "100.64.0.1",
+		AuthKeyID:      uint(pak1n1.ID),
+	}
+	h.db.Save(m1)
+
+	_, err = h.GetMachine(n1.Name, m1.Name)
+	c.Assert(err, check.IsNil)
+
+	m2 := &Machine{
+		ID:             2,
+		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
+		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
+		DiscoKey:       "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
+		Name:           "test_get_shared_nodes_2",
+		NamespaceID:    n2.ID,
+		Registered:     true,
+		RegisterMethod: "authKey",
+		IPAddress:      "100.64.0.2",
+		AuthKeyID:      uint(pak2n2.ID),
+	}
+	h.db.Save(m2)
+
+	_, err = h.GetMachine(n2.Name, m2.Name)
+	c.Assert(err, check.IsNil)
+
+	m3 := &Machine{
+		ID:             3,
+		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
+		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
+		DiscoKey:       "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
+		Name:           "test_get_shared_nodes_3",
+		NamespaceID:    n3.ID,
+		Registered:     true,
+		RegisterMethod: "authKey",
+		IPAddress:      "100.64.0.3",
+		AuthKeyID:      uint(pak3n3.ID),
+	}
+	h.db.Save(m3)
+
+	_, err = h.GetMachine(n3.Name, m3.Name)
+	c.Assert(err, check.IsNil)
+
+	m4 := &Machine{
+		ID:             4,
+		MachineKey:     "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
+		NodeKey:        "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
+		DiscoKey:       "dec46ef9dc45c7d2f03bfcd5a640d9e24e3cc68ce3d9da223867c9bc6d5e9863",
+		Name:           "test_get_shared_nodes_4",
+		NamespaceID:    n1.ID,
+		Registered:     true,
+		RegisterMethod: "authKey",
+		IPAddress:      "100.64.0.4",
+		AuthKeyID:      uint(pak4n1.ID),
+	}
+	h.db.Save(m4)
+
+	_, err = h.GetMachine(n1.Name, m4.Name)
+	c.Assert(err, check.IsNil)
+
+	p1s, err := h.getPeers(m1)
+	c.Assert(err, check.IsNil)
+	c.Assert(len(p1s), check.Equals, 1) // nodes 1 and 4
+	c.Assert(p1s[0].Name, check.Equals, "test_get_shared_nodes_4")
+
+	err = h.AddSharedMachineToNamespace(m2, n1)
+	c.Assert(err, check.IsNil)
+
+	p1sAfter, err := h.getPeers(m1)
+	c.Assert(err, check.IsNil)
+	c.Assert(len(p1sAfter), check.Equals, 2) // nodes 1, 2, 4
+	c.Assert(p1sAfter[0].Name, check.Equals, "test_get_shared_nodes_2")
+	c.Assert(p1sAfter[1].Name, check.Equals, "test_get_shared_nodes_4")
+
+	node1shared, err := h.getShared(m1)
+	c.Assert(err, check.IsNil)
+	c.Assert(len(node1shared), check.Equals, 1) // nodes 1, 2, 4
+	c.Assert(node1shared[0].Name, check.Equals, "test_get_shared_nodes_2")
+
+	pAlone, err := h.getPeers(m3)
+	c.Assert(err, check.IsNil)
+	c.Assert(len(pAlone), check.Equals, 0) // node 3 is alone
+
+	sharedMachines, err := h.ListSharedMachinesInNamespace(n1.Name)
+	c.Assert(err, check.IsNil)
+	c.Assert(len(*sharedMachines), check.Equals, 1)
+
+	err = h.DeleteMachine(m2)
+	c.Assert(err, check.IsNil)
+
+	sharedMachines, err = h.ListSharedMachinesInNamespace(n1.Name)
+	c.Assert(err, check.IsNil)
+	c.Assert(len(*sharedMachines), check.Equals, 0)
 }