derp: add bytes dropped metric (#14698)

Add bytes dropped counter metric by reason and kind.

Fixes tailscale/corp#25918

Signed-off-by: Mike O'Driscoll <mikeo@tailscale.com>
This commit is contained in:
Mike O'Driscoll 2025-01-20 12:31:26 -05:00 committed by GitHub
parent 6c30840cac
commit 6e3c746942
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -357,6 +357,12 @@ var packetsDropped = metrics.NewMultiLabelMap[dropReasonKindLabels](
"counter",
"DERP packets dropped by reason and by kind")
var bytesDropped = metrics.NewMultiLabelMap[dropReasonKindLabels](
"derp_bytes_dropped",
"counter",
"DERP bytes dropped by reason and by kind",
)
// NewServer returns a new DERP server. It doesn't listen on its own.
// Connections are given to it via Server.Accept.
func NewServer(privateKey key.NodePrivate, logf logger.Logf) *Server {
@ -388,13 +394,13 @@ func NewServer(privateKey key.NodePrivate, logf logger.Logf) *Server {
s.packetsRecvDisco = s.packetsRecvByKind.Get(string(packetKindDisco))
s.packetsRecvOther = s.packetsRecvByKind.Get(string(packetKindOther))
genPacketsDroppedCounters()
genDroppedCounters()
s.perClientSendQueueDepth = getPerClientSendQueueDepth()
return s
}
func genPacketsDroppedCounters() {
func genDroppedCounters() {
initMetrics := func(reason dropReason) {
packetsDropped.Add(dropReasonKindLabels{
Kind: string(packetKindDisco),
@ -404,6 +410,14 @@ func genPacketsDroppedCounters() {
Kind: string(packetKindOther),
Reason: string(reason),
}, 0)
bytesDropped.Add(dropReasonKindLabels{
Kind: string(packetKindDisco),
Reason: string(reason),
}, 0)
bytesDropped.Add(dropReasonKindLabels{
Kind: string(packetKindOther),
Reason: string(reason),
}, 0)
}
getMetrics := func(reason dropReason) []expvar.Var {
return []expvar.Var{
@ -415,6 +429,14 @@ func genPacketsDroppedCounters() {
Kind: string(packetKindOther),
Reason: string(reason),
}),
bytesDropped.Get(dropReasonKindLabels{
Kind: string(packetKindDisco),
Reason: string(reason),
}),
bytesDropped.Get(dropReasonKindLabels{
Kind: string(packetKindOther),
Reason: string(reason),
}),
}
}
@ -431,12 +453,14 @@ func genPacketsDroppedCounters() {
for _, dr := range dropReasons {
initMetrics(dr)
m := getMetrics(dr)
if len(m) != 2 {
if len(m) != 4 {
panic("dropReason metrics out of sync")
}
if m[0] == nil || m[1] == nil {
panic("dropReason metrics out of sync")
for _, v := range m {
if v == nil {
panic("dropReason metrics out of sync")
}
}
}
}
@ -1207,6 +1231,7 @@ func (s *Server) recordDrop(packetBytes []byte, srcKey, dstKey key.NodePublic, r
labels.Kind = string(packetKindOther)
}
packetsDropped.Add(labels, 1)
bytesDropped.Add(labels, int64(len(packetBytes)))
if verboseDropKeys[dstKey] {
// Preformat the log string prior to calling limitedLogf. The