From e371520cc5d1c0b18545980441f15bf5cbc13a6d Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 3 Mar 2020 11:33:22 -0800 Subject: [PATCH] tsweb, derp: add expvar http.Handler for Prometheus's format And add some opinions. Signed-off-by: Brad Fitzpatrick --- cmd/derper/derper.go | 15 ++++---- derp/derp_server.go | 81 +++++++++++++++----------------------------- metrics/metrics.go | 23 +++++++++++++ tsweb/tsweb.go | 79 ++++++++++++++++++++++++++++++++++++++---- 4 files changed, 131 insertions(+), 67 deletions(-) create mode 100644 metrics/metrics.go diff --git a/cmd/derper/derper.go b/cmd/derper/derper.go index 078afe4b9..e4fc7a142 100644 --- a/cmd/derper/derper.go +++ b/cmd/derper/derper.go @@ -104,6 +104,7 @@ func main() { *logCollection = "" *addr = ":3340" // above the keys DERP log.Printf("Running in dev mode.") + tsweb.DevMode = true } var logPol *logpolicy.Policy @@ -213,13 +214,13 @@ func serveSTUN() { } log.Printf("running STUN server on %v", pc.LocalAddr()) var ( - stunReadErrors = expvar.NewInt("stun-read-error") - stunWriteErrors = expvar.NewInt("stun-write-error") - stunReadNotSTUN = expvar.NewInt("stun-read-not-stun") - stunReadNotSTUNValid = expvar.NewInt("stun-read-not-stun-valid") - stunReadIPv4 = expvar.NewInt("stun-read-ipv4") - stunReadIPv6 = expvar.NewInt("stun-read-ipv6") - stunWrite = expvar.NewInt("stun-write") + stunReadErrors = expvar.NewInt("stun_read_error") + stunWriteErrors = expvar.NewInt("stun_write_error") + stunReadNotSTUN = expvar.NewInt("stun_read_not_stun") + stunReadNotSTUNValid = expvar.NewInt("stun_read_not_stun_valid") + stunReadIPv4 = expvar.NewInt("stun_read_ipv4") + stunReadIPv6 = expvar.NewInt("stun_read_ipv6") + stunWrite = expvar.NewInt("stun_write") ) var buf [64 << 10]byte for { diff --git a/derp/derp_server.go b/derp/derp_server.go index 1d6e5b96a..e6f4b8fdb 100644 --- a/derp/derp_server.go +++ b/derp/derp_server.go @@ -21,11 +21,11 @@ "os" "strconv" "sync" - "sync/atomic" "time" "golang.org/x/crypto/nacl/box" "golang.org/x/time/rate" + "tailscale.com/metrics" "tailscale.com/types/key" "tailscale.com/types/logger" ) @@ -43,13 +43,13 @@ type Server struct { logf logger.Logf // Counters: - packetsSent, bytesSent int64 - packetsRecv, bytesRecv int64 - packetsDropped int64 + packetsSent, bytesSent expvar.Int + packetsRecv, bytesRecv expvar.Int + packetsDropped expvar.Int + accepts expvar.Int mu sync.Mutex closed bool - accepts int64 netConns map[net.Conn]chan struct{} // chan is closed when conn closes clients map[key.Public]*sclient clientsEver map[key.Public]bool // never deleted from, for stats; fine for now @@ -111,8 +111,8 @@ func (s *Server) isClosed() bool { func (s *Server) Accept(nc net.Conn, brw *bufio.ReadWriter) { closed := make(chan struct{}) + s.accepts.Add(1) s.mu.Lock() - s.accepts++ s.netConns[nc] = closed s.mu.Unlock() @@ -229,7 +229,7 @@ func (s *Server) accept(nc net.Conn, brw *bufio.ReadWriter) error { s.mu.Unlock() if dst == nil { - atomic.AddInt64(&s.packetsDropped, 1) + s.packetsDropped.Add(1) if debug { s.logf("derp: %s: client %x: dropping packet for unknown %x", nc.RemoteAddr(), c.key, dstKey) } @@ -333,8 +333,8 @@ func (s *Server) recvClientKey(br *bufio.Reader) (clientKey key.Public, info *sc } func (s *Server) sendPacket(bw *bufio.Writer, srcKey key.Public, contents []byte) error { - atomic.AddInt64(&s.packetsSent, 1) - atomic.AddInt64(&s.bytesSent, int64(len(contents))) + s.packetsSent.Add(1) + s.bytesSent.Add(int64(len(contents))) if err := writeFrameHeader(bw, frameRecvPacket, uint32(len(contents))); err != nil { return err } @@ -362,8 +362,8 @@ func (s *Server) recvPacket(ctx context.Context, br *bufio.Reader, frameLen uint if _, err := io.ReadFull(br, contents); err != nil { return key.Public{}, nil, err } - atomic.AddInt64(&s.packetsRecv, 1) - atomic.AddInt64(&s.bytesRecv, int64(len(contents))) + s.packetsRecv.Add(1) + s.bytesRecv.Add(int64(len(contents))) return dstKey, contents, nil } @@ -424,51 +424,24 @@ type sclientInfo struct { type serverInfo struct { } -// Stats returns stats about the server. -func (s *Server) Stats() *ServerStats { - s.mu.Lock() - defer s.mu.Unlock() - return &ServerStats{ - BytesPerSecondLimit: s.BytesPerSecond, - CurrentConnections: len(s.netConns), - UniqueClientsEver: len(s.clientsEver), - TotalAccepts: s.accepts, - BytesReceived: atomic.LoadInt64(&s.bytesRecv), - BytesSent: atomic.LoadInt64(&s.bytesSent), - PacketsDropped: atomic.LoadInt64(&s.packetsDropped), - PacketsReceived: atomic.LoadInt64(&s.packetsRecv), - PacketsSent: atomic.LoadInt64(&s.packetsSent), - } +func (s *Server) expVarFunc(f func() interface{}) expvar.Func { + return expvar.Func(func() interface{} { + s.mu.Lock() + defer s.mu.Unlock() + return f() + }) } // ExpVar returns an expvar variable suitable for registering with expvar.Publish. func (s *Server) ExpVar() expvar.Var { - return expVar{s} -} - -type expVar struct{ *Server } - -// String implements the expvar.Var interface, returning the current server stats as JSON. -func (v expVar) String() string { - ss := v.Server.Stats() - j, err := json.MarshalIndent(ss, "", "\t") - if err != nil { - return "{}" - } - return string(j) -} - -// ServerStats are returned by Server.Stats. -// -// It is JSON-ified by expVar for the expvar package. -type ServerStats struct { - BytesPerSecondLimit int `json:"bytesPerSecondLimit"` - CurrentConnections int `json:"currentClients"` - UniqueClientsEver int `json:"uniqueClientsEver"` - TotalAccepts int64 `json:"totalAccepts"` - BytesReceived int64 `json:"bytesReceived"` - BytesSent int64 `json:"bytesSent"` - PacketsDropped int64 `json:"packetsDropped"` - PacketsReceived int64 `json:"packetsReceived"` - PacketsSent int64 `json:"packetsSent"` + m := new(metrics.Set) + m.Set("gauge_current_connnections", s.expVarFunc(func() interface{} { return len(s.netConns) })) + m.Set("counter_unique_clients_ever", s.expVarFunc(func() interface{} { return len(s.clientsEver) })) + m.Set("accepts", &s.accepts) + m.Set("bytes_received", &s.bytesRecv) + m.Set("bytes_sent", &s.bytesSent) + m.Set("packets_dropped", &s.packetsDropped) + m.Set("packets_sent", &s.packetsSent) + m.Set("packets_received", &s.packetsRecv) + return m } diff --git a/metrics/metrics.go b/metrics/metrics.go new file mode 100644 index 000000000..bb3c1bac6 --- /dev/null +++ b/metrics/metrics.go @@ -0,0 +1,23 @@ +// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package metrics contains expvar & Prometheus types and code used by +// Tailscale for monitoring. +package metrics + +import "expvar" + +// Map is a string-to-Var map variable that satisfies the expvar.Var +// interface. +// +// Semantically, this is mapped by tsweb's Prometheus exporter as a +// collection of unrelated variables exported with a common prefix. +// +// This lets us have tsweb recognize *expvar.Map for different +// purposes in the future. (Or perhaps all uses of expvar.Map will +// require explicit types like this one, declaring how we want tsweb +// to export it to Prometheus.) +type Set struct { + expvar.Map +} diff --git a/tsweb/tsweb.go b/tsweb/tsweb.go index 48461f5ee..d9138f263 100644 --- a/tsweb/tsweb.go +++ b/tsweb/tsweb.go @@ -18,8 +18,12 @@ "time" "tailscale.com/interfaces" + "tailscale.com/metrics" ) +// DevMode controls whether extra output in shown, for when the binary is being run in dev mode. +var DevMode bool + // NewMux returns a new ServeMux with debugHandler registered (and protected) at /debug/. func NewMux(debugHandler http.Handler) *http.ServeMux { mux := http.NewServeMux() @@ -29,9 +33,10 @@ func NewMux(debugHandler http.Handler) *http.ServeMux { } func RegisterCommonDebug(mux *http.ServeMux) { - expvar.Publish("uptime", uptimeVar{}) + expvar.Publish("counter_uptime_sec", expvar.Func(func() interface{} { return int64(Uptime().Seconds()) })) mux.Handle("/debug/pprof/", Protected(http.DefaultServeMux)) // to net/http/pprof mux.Handle("/debug/vars", Protected(http.DefaultServeMux)) // to expvar + mux.Handle("/debug/varz", Protected(http.HandlerFunc(varzHandler))) } func DefaultCertDir(leafDir string) string { @@ -69,7 +74,12 @@ func AllowDebugAccess(r *http.Request) bool { func Protected(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if !AllowDebugAccess(r) { - http.Error(w, "debug access denied", http.StatusForbidden) + msg := "debug access denied" + if DevMode { + ipStr, _, _ := net.SplitHostPort(r.RemoteAddr) + msg += fmt.Sprintf("; to permit access, set ALLOW_DEBUG_IP=%v", ipStr) + } + http.Error(w, msg, http.StatusForbidden) return } h.ServeHTTP(w, r) @@ -80,10 +90,6 @@ func Protected(h http.Handler) http.Handler { func Uptime() time.Duration { return time.Since(timeStart).Round(time.Second) } -type uptimeVar struct{} - -func (uptimeVar) String() string { return fmt.Sprint(int64(Uptime().Seconds())) } - // Port80Handler is the handler to be given to // autocert.Manager.HTTPHandler. The inner handler is the mux // returned by NewMux containing registered /debug handlers. @@ -114,3 +120,64 @@ func stripPort(hostport string) string { } return net.JoinHostPort(host, "443") } + +// varzHandler is an HTTP handler to write expvar values into the +// prometheus export format: +// +// https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md +// +// It makes the following assumptions: +// +// * *expvar.Int are counters. +// * a *tailscale/metrics.Set is descended into, joining keys with +// underscores. So use underscores as your metric names. +// * an expvar named starting with "gauge_" or "counter_" is of that +// Prometheus type, and has that prefix stripped. +// * anything else is untyped and thus not exported. +// * expvar.Func can return an int or int64 (for now) and anything else +// is not exported. +// +// This will evolve over time, or perhaps be replaced. +func varzHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; version=0.0.4") + + var dump func(prefix string, kv expvar.KeyValue) + dump = func(prefix string, kv expvar.KeyValue) { + name := prefix + kv.Key + var typ string + switch v := kv.Value.(type) { + case *expvar.Int: + // Fast path for common value type. + fmt.Fprintf(w, "# TYPE %s counter\n%s %v\n", name, name, v.Value()) + return + case *metrics.Set: + v.Do(func(kv expvar.KeyValue) { + dump(name+"_", kv) + }) + return + } + if strings.HasPrefix(kv.Key, "gauge_") { + typ = "gauge" + name = prefix + strings.TrimPrefix(kv.Key, "gauge_") + } else if strings.HasPrefix(kv.Key, "counter_") { + typ = "counter" + name = prefix + strings.TrimPrefix(kv.Key, "counter_") + } + if fn, ok := kv.Value.(expvar.Func); ok { + val := fn() + switch val.(type) { + case int64, int: + if typ != "" { + fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, val) + return + } + } + fmt.Fprintf(w, "# skipping expvar func %q returning unknown type %T\n", name, val) + return + } + fmt.Fprintf(w, "# skipping func %q returning unknown type %T\n", name, kv.Value) + } + expvar.Do(func(kv expvar.KeyValue) { + dump("", kv) + }) +}