2023-06-21 11:29:52 +02:00
|
|
|
package notifier
|
|
|
|
|
|
|
|
import (
|
2024-02-08 17:28:19 +01:00
|
|
|
"context"
|
2023-12-09 18:09:24 +01:00
|
|
|
"fmt"
|
2024-04-27 10:47:39 +02:00
|
|
|
"sort"
|
2023-12-09 18:09:24 +01:00
|
|
|
"strings"
|
2023-06-21 11:29:52 +02:00
|
|
|
"sync"
|
2024-04-21 18:28:17 +02:00
|
|
|
"time"
|
2023-06-21 11:29:52 +02:00
|
|
|
|
2023-06-29 11:20:22 +01:00
|
|
|
"github.com/juanfont/headscale/hscontrol/types"
|
2024-04-21 18:28:17 +02:00
|
|
|
"github.com/puzpuzpuz/xsync/v3"
|
2023-07-24 08:58:51 +02:00
|
|
|
"github.com/rs/zerolog/log"
|
2024-04-27 10:47:39 +02:00
|
|
|
"tailscale.com/tailcfg"
|
|
|
|
"tailscale.com/util/set"
|
2023-06-21 11:29:52 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
type Notifier struct {
|
2024-02-08 17:28:19 +01:00
|
|
|
l sync.RWMutex
|
2024-02-23 10:59:24 +01:00
|
|
|
nodes map[types.NodeID]chan<- types.StateUpdate
|
2024-04-21 18:28:17 +02:00
|
|
|
connected *xsync.MapOf[types.NodeID, bool]
|
2024-04-27 10:47:39 +02:00
|
|
|
b *batcher
|
2023-06-21 11:29:52 +02:00
|
|
|
}
|
|
|
|
|
2024-04-27 10:47:39 +02:00
|
|
|
func NewNotifier(cfg *types.Config) *Notifier {
|
|
|
|
n := &Notifier{
|
2024-02-23 10:59:24 +01:00
|
|
|
nodes: make(map[types.NodeID]chan<- types.StateUpdate),
|
2024-04-21 18:28:17 +02:00
|
|
|
connected: xsync.NewMapOf[types.NodeID, bool](),
|
2024-02-08 17:28:19 +01:00
|
|
|
}
|
2024-04-27 10:47:39 +02:00
|
|
|
b := newBatcher(cfg.Tuning.BatchChangeDelay, n)
|
|
|
|
n.b = b
|
|
|
|
// TODO(kradalby): clean this up
|
|
|
|
go b.doWork()
|
|
|
|
return n
|
2023-06-21 11:29:52 +02:00
|
|
|
}
|
|
|
|
|
2024-02-23 10:59:24 +01:00
|
|
|
func (n *Notifier) AddNode(nodeID types.NodeID, c chan<- types.StateUpdate) {
|
|
|
|
log.Trace().Caller().Uint64("node.id", nodeID.Uint64()).Msg("acquiring lock to add node")
|
2024-02-08 17:28:19 +01:00
|
|
|
defer log.Trace().
|
|
|
|
Caller().
|
2024-02-23 10:59:24 +01:00
|
|
|
Uint64("node.id", nodeID.Uint64()).
|
2024-02-08 17:28:19 +01:00
|
|
|
Msg("releasing lock to add node")
|
2023-09-11 06:08:44 -05:00
|
|
|
|
2024-04-21 18:28:17 +02:00
|
|
|
start := time.Now()
|
2023-06-21 11:29:52 +02:00
|
|
|
n.l.Lock()
|
|
|
|
defer n.l.Unlock()
|
2024-04-21 18:28:17 +02:00
|
|
|
notifierWaitForLock.WithLabelValues("add").Observe(time.Since(start).Seconds())
|
2023-06-21 11:29:52 +02:00
|
|
|
|
2024-02-23 10:59:24 +01:00
|
|
|
n.nodes[nodeID] = c
|
2024-04-21 18:28:17 +02:00
|
|
|
n.connected.Store(nodeID, true)
|
2023-07-24 08:58:51 +02:00
|
|
|
|
|
|
|
log.Trace().
|
2024-02-23 10:59:24 +01:00
|
|
|
Uint64("node.id", nodeID.Uint64()).
|
2023-07-24 08:58:51 +02:00
|
|
|
Int("open_chans", len(n.nodes)).
|
|
|
|
Msg("Added new channel")
|
2024-04-21 18:28:17 +02:00
|
|
|
notifierNodeUpdateChans.Inc()
|
2023-06-21 11:29:52 +02:00
|
|
|
}
|
|
|
|
|
2024-02-23 10:59:24 +01:00
|
|
|
func (n *Notifier) RemoveNode(nodeID types.NodeID) {
|
|
|
|
log.Trace().Caller().Uint64("node.id", nodeID.Uint64()).Msg("acquiring lock to remove node")
|
2024-02-08 17:28:19 +01:00
|
|
|
defer log.Trace().
|
|
|
|
Caller().
|
2024-02-23 10:59:24 +01:00
|
|
|
Uint64("node.id", nodeID.Uint64()).
|
2024-02-08 17:28:19 +01:00
|
|
|
Msg("releasing lock to remove node")
|
2023-09-11 06:08:44 -05:00
|
|
|
|
2024-04-21 18:28:17 +02:00
|
|
|
start := time.Now()
|
2023-06-21 11:29:52 +02:00
|
|
|
n.l.Lock()
|
|
|
|
defer n.l.Unlock()
|
2024-04-21 18:28:17 +02:00
|
|
|
notifierWaitForLock.WithLabelValues("remove").Observe(time.Since(start).Seconds())
|
2023-06-21 11:29:52 +02:00
|
|
|
|
2024-02-08 17:28:19 +01:00
|
|
|
if len(n.nodes) == 0 {
|
2023-06-21 11:29:52 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2024-02-23 10:59:24 +01:00
|
|
|
delete(n.nodes, nodeID)
|
2024-04-21 18:28:17 +02:00
|
|
|
n.connected.Store(nodeID, false)
|
2023-07-24 08:58:51 +02:00
|
|
|
|
|
|
|
log.Trace().
|
2024-02-23 10:59:24 +01:00
|
|
|
Uint64("node.id", nodeID.Uint64()).
|
2023-07-24 08:58:51 +02:00
|
|
|
Int("open_chans", len(n.nodes)).
|
|
|
|
Msg("Removed channel")
|
2024-04-21 18:28:17 +02:00
|
|
|
notifierNodeUpdateChans.Dec()
|
2023-06-21 11:29:52 +02:00
|
|
|
}
|
|
|
|
|
2023-12-09 18:09:24 +01:00
|
|
|
// IsConnected reports if a node is connected to headscale and has a
|
|
|
|
// poll session open.
|
2024-02-23 10:59:24 +01:00
|
|
|
func (n *Notifier) IsConnected(nodeID types.NodeID) bool {
|
2023-12-09 18:09:24 +01:00
|
|
|
n.l.RLock()
|
|
|
|
defer n.l.RUnlock()
|
|
|
|
|
2024-04-21 18:28:17 +02:00
|
|
|
if val, ok := n.connected.Load(nodeID); ok {
|
|
|
|
return val
|
|
|
|
}
|
|
|
|
return false
|
2024-02-23 10:59:24 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// IsLikelyConnected reports if a node is connected to headscale and has a
|
|
|
|
// poll session open, but doesnt lock, so might be wrong.
|
|
|
|
func (n *Notifier) IsLikelyConnected(nodeID types.NodeID) bool {
|
2024-04-21 18:28:17 +02:00
|
|
|
if val, ok := n.connected.Load(nodeID); ok {
|
|
|
|
return val
|
|
|
|
}
|
|
|
|
return false
|
2024-02-08 17:28:19 +01:00
|
|
|
}
|
2023-12-09 18:09:24 +01:00
|
|
|
|
2024-04-21 18:28:17 +02:00
|
|
|
func (n *Notifier) LikelyConnectedMap() *xsync.MapOf[types.NodeID, bool] {
|
2024-02-08 17:28:19 +01:00
|
|
|
return n.connected
|
2023-12-09 18:09:24 +01:00
|
|
|
}
|
|
|
|
|
2024-02-08 17:28:19 +01:00
|
|
|
func (n *Notifier) NotifyAll(ctx context.Context, update types.StateUpdate) {
|
|
|
|
n.NotifyWithIgnore(ctx, update)
|
2023-06-21 11:29:52 +02:00
|
|
|
}
|
|
|
|
|
2024-02-08 17:28:19 +01:00
|
|
|
func (n *Notifier) NotifyWithIgnore(
|
|
|
|
ctx context.Context,
|
|
|
|
update types.StateUpdate,
|
2024-02-23 10:59:24 +01:00
|
|
|
ignoreNodeIDs ...types.NodeID,
|
2024-02-08 17:28:19 +01:00
|
|
|
) {
|
2024-04-27 10:47:39 +02:00
|
|
|
notifierUpdateReceived.WithLabelValues(update.Type.String(), types.NotifyOriginKey.Value(ctx)).Inc()
|
|
|
|
n.b.addOrPassthrough(update)
|
2023-06-21 11:29:52 +02:00
|
|
|
}
|
2023-12-09 18:09:24 +01:00
|
|
|
|
2024-04-21 18:28:17 +02:00
|
|
|
func (n *Notifier) NotifyByNodeID(
|
2024-02-08 17:28:19 +01:00
|
|
|
ctx context.Context,
|
|
|
|
update types.StateUpdate,
|
2024-02-23 10:59:24 +01:00
|
|
|
nodeID types.NodeID,
|
2024-02-08 17:28:19 +01:00
|
|
|
) {
|
2024-02-23 10:59:24 +01:00
|
|
|
log.Trace().Caller().Str("type", update.Type.String()).Msg("acquiring lock to notify")
|
2024-01-05 10:41:56 +01:00
|
|
|
defer log.Trace().
|
|
|
|
Caller().
|
2024-02-23 10:59:24 +01:00
|
|
|
Str("type", update.Type.String()).
|
2024-02-08 17:28:19 +01:00
|
|
|
Msg("releasing lock, finished notifying")
|
2024-01-05 10:41:56 +01:00
|
|
|
|
2024-04-21 18:28:17 +02:00
|
|
|
start := time.Now()
|
2024-01-05 10:41:56 +01:00
|
|
|
n.l.RLock()
|
|
|
|
defer n.l.RUnlock()
|
2024-04-21 18:28:17 +02:00
|
|
|
notifierWaitForLock.WithLabelValues("notify").Observe(time.Since(start).Seconds())
|
2024-01-05 10:41:56 +01:00
|
|
|
|
2024-02-23 10:59:24 +01:00
|
|
|
if c, ok := n.nodes[nodeID]; ok {
|
2024-02-08 17:28:19 +01:00
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
log.Error().
|
|
|
|
Err(ctx.Err()).
|
2024-02-23 10:59:24 +01:00
|
|
|
Uint64("node.id", nodeID.Uint64()).
|
2024-04-27 10:47:39 +02:00
|
|
|
Any("origin", types.NotifyOriginKey.Value(ctx)).
|
|
|
|
Any("origin-hostname", types.NotifyHostnameKey.Value(ctx)).
|
2024-02-08 17:28:19 +01:00
|
|
|
Msgf("update not sent, context cancelled")
|
2024-04-27 10:47:39 +02:00
|
|
|
notifierUpdateSent.WithLabelValues("cancelled", update.Type.String(), types.NotifyOriginKey.Value(ctx)).Inc()
|
2024-02-08 17:28:19 +01:00
|
|
|
|
|
|
|
return
|
|
|
|
case c <- update:
|
|
|
|
log.Trace().
|
2024-02-23 10:59:24 +01:00
|
|
|
Uint64("node.id", nodeID.Uint64()).
|
2024-02-08 17:28:19 +01:00
|
|
|
Any("origin", ctx.Value("origin")).
|
2024-02-23 10:59:24 +01:00
|
|
|
Any("origin-hostname", ctx.Value("hostname")).
|
2024-02-08 17:28:19 +01:00
|
|
|
Msgf("update successfully sent on chan")
|
2024-04-27 10:47:39 +02:00
|
|
|
notifierUpdateSent.WithLabelValues("ok", update.Type.String(), types.NotifyOriginKey.Value(ctx)).Inc()
|
2024-02-08 17:28:19 +01:00
|
|
|
}
|
2024-01-05 10:41:56 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-04-27 10:47:39 +02:00
|
|
|
func (n *Notifier) sendAll(update types.StateUpdate) {
|
|
|
|
start := time.Now()
|
|
|
|
n.l.RLock()
|
|
|
|
defer n.l.RUnlock()
|
|
|
|
notifierWaitForLock.WithLabelValues("send-all").Observe(time.Since(start).Seconds())
|
|
|
|
|
|
|
|
for _, c := range n.nodes {
|
|
|
|
c <- update
|
|
|
|
notifierUpdateSent.WithLabelValues("ok", update.Type.String(), "send-all").Inc()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-12-09 18:09:24 +01:00
|
|
|
func (n *Notifier) String() string {
|
|
|
|
n.l.RLock()
|
|
|
|
defer n.l.RUnlock()
|
|
|
|
|
2024-04-10 15:35:09 +02:00
|
|
|
var b strings.Builder
|
|
|
|
b.WriteString("chans:\n")
|
2023-12-09 18:09:24 +01:00
|
|
|
|
|
|
|
for k, v := range n.nodes {
|
2024-04-10 15:35:09 +02:00
|
|
|
fmt.Fprintf(&b, "\t%d: %p\n", k, v)
|
2023-12-09 18:09:24 +01:00
|
|
|
}
|
|
|
|
|
2024-04-10 15:35:09 +02:00
|
|
|
b.WriteString("\n")
|
|
|
|
b.WriteString("connected:\n")
|
|
|
|
|
2024-04-21 18:28:17 +02:00
|
|
|
n.connected.Range(func(k types.NodeID, v bool) bool {
|
2024-04-10 15:35:09 +02:00
|
|
|
fmt.Fprintf(&b, "\t%d: %t\n", k, v)
|
2024-04-21 18:28:17 +02:00
|
|
|
return true
|
|
|
|
})
|
2024-04-10 15:35:09 +02:00
|
|
|
|
|
|
|
return b.String()
|
2023-12-09 18:09:24 +01:00
|
|
|
}
|
2024-04-27 10:47:39 +02:00
|
|
|
|
|
|
|
type batcher struct {
|
|
|
|
tick *time.Ticker
|
|
|
|
|
|
|
|
mu sync.Mutex
|
|
|
|
|
|
|
|
cancelCh chan struct{}
|
|
|
|
|
|
|
|
changedNodeIDs set.Slice[types.NodeID]
|
|
|
|
nodesChanged bool
|
|
|
|
patches map[types.NodeID]tailcfg.PeerChange
|
|
|
|
patchesChanged bool
|
|
|
|
|
|
|
|
n *Notifier
|
|
|
|
}
|
|
|
|
|
|
|
|
func newBatcher(batchTime time.Duration, n *Notifier) *batcher {
|
|
|
|
return &batcher{
|
|
|
|
tick: time.NewTicker(batchTime),
|
|
|
|
cancelCh: make(chan struct{}),
|
|
|
|
patches: make(map[types.NodeID]tailcfg.PeerChange),
|
|
|
|
n: n,
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
func (b *batcher) close() {
|
|
|
|
b.cancelCh <- struct{}{}
|
|
|
|
}
|
|
|
|
|
|
|
|
// addOrPassthrough adds the update to the batcher, if it is not a
|
|
|
|
// type that is currently batched, it will be sent immediately.
|
|
|
|
func (b *batcher) addOrPassthrough(update types.StateUpdate) {
|
|
|
|
b.mu.Lock()
|
|
|
|
defer b.mu.Unlock()
|
|
|
|
|
|
|
|
switch update.Type {
|
|
|
|
case types.StatePeerChanged:
|
|
|
|
b.changedNodeIDs.Add(update.ChangeNodes...)
|
|
|
|
b.nodesChanged = true
|
|
|
|
|
|
|
|
case types.StatePeerChangedPatch:
|
|
|
|
for _, newPatch := range update.ChangePatches {
|
|
|
|
if curr, ok := b.patches[types.NodeID(newPatch.NodeID)]; ok {
|
|
|
|
overwritePatch(&curr, newPatch)
|
|
|
|
b.patches[types.NodeID(newPatch.NodeID)] = curr
|
|
|
|
} else {
|
|
|
|
b.patches[types.NodeID(newPatch.NodeID)] = *newPatch
|
|
|
|
}
|
|
|
|
}
|
|
|
|
b.patchesChanged = true
|
|
|
|
|
|
|
|
default:
|
|
|
|
b.n.sendAll(update)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// flush sends all the accumulated patches to all
|
|
|
|
// nodes in the notifier.
|
|
|
|
func (b *batcher) flush() {
|
|
|
|
b.mu.Lock()
|
|
|
|
defer b.mu.Unlock()
|
|
|
|
|
|
|
|
if b.nodesChanged || b.patchesChanged {
|
|
|
|
var patches []*tailcfg.PeerChange
|
|
|
|
// If a node is getting a full update from a change
|
|
|
|
// node update, then the patch can be dropped.
|
|
|
|
for nodeID, patch := range b.patches {
|
|
|
|
if b.changedNodeIDs.Contains(nodeID) {
|
|
|
|
delete(b.patches, nodeID)
|
|
|
|
} else {
|
|
|
|
patches = append(patches, &patch)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
changedNodes := b.changedNodeIDs.Slice().AsSlice()
|
|
|
|
sort.Slice(changedNodes, func(i, j int) bool {
|
|
|
|
return changedNodes[i] < changedNodes[j]
|
|
|
|
})
|
|
|
|
|
|
|
|
if b.changedNodeIDs.Slice().Len() > 0 {
|
|
|
|
update := types.StateUpdate{
|
|
|
|
Type: types.StatePeerChanged,
|
|
|
|
ChangeNodes: changedNodes,
|
|
|
|
}
|
|
|
|
|
|
|
|
b.n.sendAll(update)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(patches) > 0 {
|
|
|
|
patchUpdate := types.StateUpdate{
|
|
|
|
Type: types.StatePeerChangedPatch,
|
|
|
|
ChangePatches: patches,
|
|
|
|
}
|
|
|
|
|
|
|
|
b.n.sendAll(patchUpdate)
|
|
|
|
}
|
|
|
|
|
|
|
|
b.changedNodeIDs = set.Slice[types.NodeID]{}
|
|
|
|
b.nodesChanged = false
|
|
|
|
b.patches = make(map[types.NodeID]tailcfg.PeerChange, len(b.patches))
|
|
|
|
b.patchesChanged = false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (b *batcher) doWork() {
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-b.cancelCh:
|
|
|
|
return
|
|
|
|
case <-b.tick.C:
|
|
|
|
b.flush()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// overwritePatch takes the current patch and a newer patch
|
|
|
|
// and override any field that has changed
|
|
|
|
func overwritePatch(currPatch, newPatch *tailcfg.PeerChange) {
|
|
|
|
if newPatch.DERPRegion != 0 {
|
|
|
|
currPatch.DERPRegion = newPatch.DERPRegion
|
|
|
|
}
|
|
|
|
|
|
|
|
if newPatch.Cap != 0 {
|
|
|
|
currPatch.Cap = newPatch.Cap
|
|
|
|
}
|
|
|
|
|
|
|
|
if newPatch.CapMap != nil {
|
|
|
|
currPatch.CapMap = newPatch.CapMap
|
|
|
|
}
|
|
|
|
|
|
|
|
if newPatch.Endpoints != nil {
|
|
|
|
currPatch.Endpoints = newPatch.Endpoints
|
|
|
|
}
|
|
|
|
|
|
|
|
if newPatch.Key != nil {
|
|
|
|
currPatch.Key = newPatch.Key
|
|
|
|
}
|
|
|
|
|
|
|
|
if newPatch.KeySignature != nil {
|
|
|
|
currPatch.KeySignature = newPatch.KeySignature
|
|
|
|
}
|
|
|
|
|
|
|
|
if newPatch.DiscoKey != nil {
|
|
|
|
currPatch.DiscoKey = newPatch.DiscoKey
|
|
|
|
}
|
|
|
|
|
|
|
|
if newPatch.Online != nil {
|
|
|
|
currPatch.Online = newPatch.Online
|
|
|
|
}
|
|
|
|
|
|
|
|
if newPatch.LastSeen != nil {
|
|
|
|
currPatch.LastSeen = newPatch.LastSeen
|
|
|
|
}
|
|
|
|
|
|
|
|
if newPatch.KeyExpiry != nil {
|
|
|
|
currPatch.KeyExpiry = newPatch.KeyExpiry
|
|
|
|
}
|
|
|
|
|
|
|
|
if newPatch.Capabilities != nil {
|
|
|
|
currPatch.Capabilities = newPatch.Capabilities
|
|
|
|
}
|
|
|
|
}
|