logtail: add metrics (#18184)

Add metrics about logtail uploading and underlying buffer.
Add metrics to the in-memory buffer implementation.

Updates tailscale/corp#21363

Signed-off-by: Joe Tsai <joetsai@digital-static.net>

logtail/buffer.go

@@ -8,8 +8,10 @@ package logtail
 import (
 	"bytes"
 	"errors"
+	"expvar"
 	"fmt"
 
+	"tailscale.com/metrics"
 	"tailscale.com/syncs"
 )
 
@@ -39,12 +41,42 @@ type memBuffer struct {
 
 	dropMu    syncs.Mutex
 	dropCount int
+
+	// Metrics (see [memBuffer.ExpVar] for details).
+	writeCalls   expvar.Int
+	readCalls    expvar.Int
+	writeBytes   expvar.Int
+	readBytes    expvar.Int
+	droppedBytes expvar.Int
+	storedBytes  expvar.Int
+}
+
+// ExpVar returns a [metrics.Set] with metrics about the buffer.
+//
+//   - counter_write_calls: Total number of write calls.
+//   - counter_read_calls: Total number of read calls.
+//   - counter_write_bytes: Total number of bytes written.
+//   - counter_read_bytes: Total number of bytes read.
+//   - counter_dropped_bytes: Total number of bytes dropped.
+//   - gauge_stored_bytes: Current number of bytes stored in memory.
+func (b *memBuffer) ExpVar() expvar.Var {
+	m := new(metrics.Set)
+	m.Set("counter_write_calls", &b.writeCalls)
+	m.Set("counter_read_calls", &b.readCalls)
+	m.Set("counter_write_bytes", &b.writeBytes)
+	m.Set("counter_read_bytes", &b.readBytes)
+	m.Set("counter_dropped_bytes", &b.droppedBytes)
+	m.Set("gauge_stored_bytes", &b.storedBytes)
+	return m
 }
 
 func (m *memBuffer) TryReadLine() ([]byte, error) {
+	m.readCalls.Add(1)
 	if m.next != nil {
 		msg := m.next
 		m.next = nil
+		m.readBytes.Add(int64(len(msg)))
+		m.storedBytes.Add(-int64(len(msg)))
 		return msg, nil
 	}
 
@@ -52,8 +84,13 @@ func (m *memBuffer) TryReadLine() ([]byte, error) {
 	case ent := <-m.pending:
 		if ent.dropCount > 0 {
 			m.next = ent.msg
-			return fmt.Appendf(nil, "----------- %d logs dropped ----------", ent.dropCount), nil
+			b := fmt.Appendf(nil, "----------- %d logs dropped ----------", ent.dropCount)
+			m.writeBytes.Add(int64(len(b))) // indicate pseudo-injected log message
+			m.readBytes.Add(int64(len(b)))
+			return b, nil
 		}
+		m.readBytes.Add(int64(len(ent.msg)))
+		m.storedBytes.Add(-int64(len(ent.msg)))
 		return ent.msg, nil
 	default:
 		return nil, nil
@@ -61,6 +98,7 @@ func (m *memBuffer) TryReadLine() ([]byte, error) {
 }
 
 func (m *memBuffer) Write(b []byte) (int, error) {
+	m.writeCalls.Add(1)
 	m.dropMu.Lock()
 	defer m.dropMu.Unlock()
 
@@ -70,10 +108,13 @@ func (m *memBuffer) Write(b []byte) (int, error) {
 	}
 	select {
 	case m.pending <- ent:
+		m.writeBytes.Add(int64(len(b)))
+		m.storedBytes.Add(+int64(len(b)))
 		m.dropCount = 0
 		return len(b), nil
 	default:
 		m.dropCount++
+		m.droppedBytes.Add(int64(len(b)))
 		return 0, errBufferFull
 	}
 }