From d21c00205d6af2648a5b2723a234277e281e6dcc Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 3 Jun 2024 13:42:06 -0700 Subject: [PATCH] cmd/stunstamp: implement service to measure DERP STUN RTT (#12241) stunstamp timestamping includes userspace and SO_TIMESTAMPING kernel timestamping where available. Measurements are written locally to a sqlite DB, exposed over an HTTP API, and written to prometheus via remote-write protocol. Updates tailscale/corp#20344 Signed-off-by: Jordan Whited --- cmd/stunstamp/api.go | 141 ++++ cmd/stunstamp/stunstamp.go | 782 ++++++++++++++++++++++ cmd/stunstamp/stunstamp_db_default.go | 26 + cmd/stunstamp/stunstamp_db_windows_386.go | 17 + cmd/stunstamp/stunstamp_default.go | 25 + cmd/stunstamp/stunstamp_linux.go | 126 ++++ go.mod | 28 +- go.sum | 70 +- 8 files changed, 1197 insertions(+), 18 deletions(-) create mode 100644 cmd/stunstamp/api.go create mode 100644 cmd/stunstamp/stunstamp.go create mode 100644 cmd/stunstamp/stunstamp_db_default.go create mode 100644 cmd/stunstamp/stunstamp_db_windows_386.go create mode 100644 cmd/stunstamp/stunstamp_default.go create mode 100644 cmd/stunstamp/stunstamp_linux.go diff --git a/cmd/stunstamp/api.go b/cmd/stunstamp/api.go new file mode 100644 index 000000000..849d5a32c --- /dev/null +++ b/cmd/stunstamp/api.go @@ -0,0 +1,141 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "compress/gzip" + "encoding/json" + "errors" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + sq "github.com/Masterminds/squirrel" +) + +type api struct { + db *db + mux *http.ServeMux +} + +func newAPI(db *db) *api { + a := &api{ + db: db, + } + mux := http.NewServeMux() + mux.HandleFunc("/query", a.query) + a.mux = mux + return a +} + +type apiResult struct { + At int `json:"at"` // time.Time.Unix() + RegionID int `json:"regionID"` + Hostname string `json:"hostname"` + Af int `json:"af"` // 4 or 6 + Addr string `json:"addr"` + Source int `json:"source"` // timestampSourceUserspace (0) or timestampSourceKernel (1) + StableConn bool `json:"stableConn"` + RttNS *int `json:"rttNS"` +} + +func getTimeBounds(vals url.Values) (from time.Time, to time.Time, err error) { + lastForm, ok := vals["last"] + if ok && len(lastForm) > 0 { + dur, err := time.ParseDuration(lastForm[0]) + if err != nil { + return time.Time{}, time.Time{}, err + } + now := time.Now() + return now.Add(-dur), now, nil + } + + fromForm, ok := vals["from"] + if ok && len(fromForm) > 0 { + fromUnixSec, err := strconv.Atoi(fromForm[0]) + if err != nil { + return time.Time{}, time.Time{}, err + } + from = time.Unix(int64(fromUnixSec), 0) + toForm, ok := vals["to"] + if ok && len(toForm) > 0 { + toUnixSec, err := strconv.Atoi(toForm[0]) + if err != nil { + return time.Time{}, time.Time{}, err + } + to = time.Unix(int64(toUnixSec), 0) + } else { + return time.Time{}, time.Time{}, errors.New("from specified without to") + } + return from, to, nil + } + + // no time bounds specified, default to last 1h + now := time.Now() + return now.Add(-time.Hour), now, nil +} + +func (a *api) ServeHTTP(w http.ResponseWriter, r *http.Request) { + a.mux.ServeHTTP(w, r) +} + +func (a *api) query(w http.ResponseWriter, r *http.Request) { + err := r.ParseForm() + if err != nil { + http.Error(w, err.Error(), 500) + return + } + from, to, err := getTimeBounds(r.Form) + if err != nil { + http.Error(w, err.Error(), 500) + return + } + + sb := sq.Select("at_unix", "region_id", "hostname", "af", "address", "timestamp_source", "stable_conn", "rtt_ns").From("rtt") + sb = sb.Where(sq.And{ + sq.GtOrEq{"at_unix": from.Unix()}, + sq.LtOrEq{"at_unix": to.Unix()}, + }) + query, args, err := sb.ToSql() + if err != nil { + return + } + + rows, err := a.db.Query(query, args...) + if err != nil { + http.Error(w, err.Error(), 500) + return + } + results := make([]apiResult, 0) + for rows.Next() { + rtt := 0 + result := apiResult{ + RttNS: &rtt, + } + err = rows.Scan(&result.At, &result.RegionID, &result.Hostname, &result.Af, &result.Addr, &result.Source, &result.StableConn, &result.RttNS) + if err != nil { + http.Error(w, err.Error(), 500) + return + } + results = append(results, result) + } + if rows.Err() != nil { + http.Error(w, rows.Err().Error(), 500) + return + } + if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + gz := gzip.NewWriter(w) + defer gz.Close() + w.Header().Set("Content-Encoding", "gzip") + err = json.NewEncoder(gz).Encode(&results) + } else { + err = json.NewEncoder(w).Encode(&results) + } + if err != nil { + http.Error(w, err.Error(), 500) + return + } +} diff --git a/cmd/stunstamp/stunstamp.go b/cmd/stunstamp/stunstamp.go new file mode 100644 index 000000000..3390a9c13 --- /dev/null +++ b/cmd/stunstamp/stunstamp.go @@ -0,0 +1,782 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// The stunstamp binary measures STUN round-trip latency with DERPs. +package main + +import ( + "bytes" + "cmp" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "log" + "math" + "math/rand" + "net" + "net/http" + "net/netip" + "net/url" + "os" + "os/signal" + "slices" + "sync" + "syscall" + "time" + + "github.com/golang/snappy" + "github.com/prometheus/prometheus/prompb" + "tailscale.com/logtail/backoff" + "tailscale.com/net/stun" + "tailscale.com/tailcfg" +) + +var ( + flagDERPMap = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map") + flagOut = flag.String("out", "", "output sqlite filename") + flagInterval = flag.Duration("interval", time.Minute, "interval to probe at in time.ParseDuration() format") + flagAPI = flag.String("api", "", "listen addr for HTTP API") + flagIPv6 = flag.Bool("ipv6", false, "probe IPv6 addresses") + flagRetention = flag.Duration("retention", time.Hour*24*7, "sqlite retention period in time.ParseDuration() format") + flagRemoteWriteURL = flag.String("rw-url", "", "prometheus remote write URL") + flagInstance = flag.String("instance", "", "instance label value; defaults to hostname if unspecified") +) + +const ( + minInterval = time.Second + maxBufferDuration = time.Hour +) + +func getDERPMap(ctx context.Context, url string) (*tailcfg.DERPMap, error) { + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + dm := tailcfg.DERPMap{} + err = json.NewDecoder(resp.Body).Decode(&dm) + if err != nil { + return nil, nil + } + return &dm, nil +} + +type timestampSource int + +const ( + timestampSourceUserspace timestampSource = iota + timestampSourceKernel +) + +func (t timestampSource) String() string { + switch t { + case timestampSourceUserspace: + return "userspace" + case timestampSourceKernel: + return "kernel" + default: + return "unknown" + } +} + +type result struct { + at time.Time + meta nodeMeta + timestampSource timestampSource + connStability connStability + rtt *time.Duration // nil signifies failure, e.g. timeout +} + +func measureRTT(conn io.ReadWriteCloser, dst *net.UDPAddr, req []byte) (resp []byte, rtt time.Duration, err error) { + uconn, ok := conn.(*net.UDPConn) + if !ok { + return nil, 0, fmt.Errorf("unexpected conn type: %T", conn) + } + err = uconn.SetReadDeadline(time.Now().Add(time.Second * 2)) + if err != nil { + return nil, 0, fmt.Errorf("error setting read deadline: %w", err) + } + txAt := time.Now() + _, err = uconn.WriteToUDP(req, dst) + if err != nil { + return nil, 0, fmt.Errorf("error writing to udp socket: %w", err) + } + b := make([]byte, 1460) + n, err := uconn.Read(b) + rxAt := time.Now() + if err != nil { + return nil, 0, fmt.Errorf("error reading from udp socket: %w", err) + } + return b[:n], rxAt.Sub(txAt), nil +} + +func isTemporaryOrTimeoutErr(err error) bool { + if errors.Is(err, os.ErrDeadlineExceeded) || errors.Is(err, context.DeadlineExceeded) { + return true + } + if err, ok := err.(interface{ Temporary() bool }); ok { + return err.Temporary() + } + return false +} + +type nodeMeta struct { + regionID int + regionCode string + hostname string + addr netip.Addr +} + +type measureFn func(conn io.ReadWriteCloser, dst *net.UDPAddr, req []byte) (resp []byte, rtt time.Duration, err error) + +func probe(meta nodeMeta, conn io.ReadWriteCloser, fn measureFn) (*time.Duration, error) { + ua := &net.UDPAddr{ + IP: net.IP(meta.addr.AsSlice()), + Port: 3478, + } + + var ( + resp []byte + rtt time.Duration + ) + txID := stun.NewTxID() + req := stun.Request(txID) + time.Sleep(time.Millisecond * time.Duration(rand.Intn(200))) // jitter across tx + resp, rtt, err := fn(conn, ua, req) + if err != nil { + if isTemporaryOrTimeoutErr(err) { + log.Printf("temp error measuring RTT to %s(%s): %v", meta.hostname, meta.addr, err) + return nil, nil + } + } + _, _, err = stun.ParseResponse(resp) + if err != nil { + log.Printf("invalid stun response from %s: %v", meta.hostname, err) + return nil, nil + } + return &rtt, nil +} + +func nodeMetaFromDERPMap(dm *tailcfg.DERPMap, nodeMetaByAddr map[netip.Addr]nodeMeta, ipv6 bool) (stale []nodeMeta, err error) { + // Parse the new derp map before making any state changes in nodeMetaByAddr. + // If parse fails we just stick with the old state. + updated := make(map[netip.Addr]nodeMeta) + for regionID, region := range dm.Regions { + for _, node := range region.Nodes { + v4, err := netip.ParseAddr(node.IPv4) + if err != nil || !v4.Is4() { + return nil, fmt.Errorf("invalid ipv4 addr for node in derp map: %v", node.Name) + } + metas := make([]nodeMeta, 0, 2) + metas = append(metas, nodeMeta{ + regionID: regionID, + regionCode: region.RegionCode, + hostname: node.HostName, + addr: v4, + }) + if ipv6 { + v6, err := netip.ParseAddr(node.IPv6) + if err != nil || !v6.Is6() { + return nil, fmt.Errorf("invalid ipv6 addr for node in derp map: %v", node.Name) + } + metas = append(metas, metas[0]) + metas[1].addr = v6 + } + for _, meta := range metas { + updated[meta.addr] = meta + } + } + } + + // Find nodeMeta that have changed + for addr, updatedMeta := range updated { + previousMeta, ok := nodeMetaByAddr[addr] + if ok { + if previousMeta == updatedMeta { + continue + } + stale = append(stale, previousMeta) + nodeMetaByAddr[addr] = updatedMeta + } else { + nodeMetaByAddr[addr] = updatedMeta + } + } + + // Find nodeMeta that no longer exist + for addr, potentialStale := range nodeMetaByAddr { + _, ok := updated[addr] + if !ok { + stale = append(stale, potentialStale) + } + } + + return stale, nil +} + +func getStableConns(stableConns map[netip.Addr][2]io.ReadWriteCloser, addr netip.Addr) ([2]io.ReadWriteCloser, error) { + conns, ok := stableConns[addr] + if ok { + return conns, nil + } + if supportsKernelTS() { + kconn, err := getConnKernelTimestamp() + if err != nil { + return conns, err + } + conns[timestampSourceKernel] = kconn + } + uconn, err := net.ListenUDP("udp", &net.UDPAddr{}) + if err != nil { + return conns, err + } + conns[timestampSourceUserspace] = uconn + stableConns[addr] = conns + return conns, nil +} + +// probeNodes measures the round-trip time for STUN binding requests against the +// DERP nodes described by nodeMetaByAddr while using/updating stableConns for +// UDP sockets that should be recycled across runs. It returns the results or +// an error if one occurs. +func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Addr][2]io.ReadWriteCloser) ([]result, error) { + wg := sync.WaitGroup{} + results := make([]result, 0) + resultsCh := make(chan result) + errCh := make(chan error) + doneCh := make(chan struct{}) + numProbes := 0 + at := time.Now() + addrsToProbe := make(map[netip.Addr]bool) + + doProbe := func(conn io.ReadWriteCloser, meta nodeMeta, source timestampSource) { + defer wg.Done() + r := result{} + if conn == nil { + var err error + if source == timestampSourceKernel { + conn, err = getConnKernelTimestamp() + } else { + conn, err = net.ListenUDP("udp", &net.UDPAddr{}) + } + if err != nil { + select { + case <-doneCh: + return + case errCh <- err: + return + } + } + defer conn.Close() + } else { + r.connStability = stableConn + } + fn := measureRTT + if source == timestampSourceKernel { + fn = measureRTTKernel + } + rtt, err := probe(meta, conn, fn) + if err != nil { + select { + case <-doneCh: + return + case errCh <- err: + return + } + } + r.at = at + r.meta = meta + r.timestampSource = source + r.rtt = rtt + select { + case <-doneCh: + case resultsCh <- r: + } + } + + for _, meta := range nodeMetaByAddr { + addrsToProbe[meta.addr] = true + stable, err := getStableConns(stableConns, meta.addr) + if err != nil { + close(doneCh) + wg.Wait() + return nil, err + } + + wg.Add(2) + numProbes += 2 + go doProbe(stable[timestampSourceUserspace], meta, timestampSourceUserspace) + go doProbe(nil, meta, timestampSourceUserspace) + if supportsKernelTS() { + wg.Add(2) + numProbes += 2 + go doProbe(stable[timestampSourceKernel], meta, timestampSourceKernel) + go doProbe(nil, meta, timestampSourceKernel) + } + } + + // cleanup conns we no longer need + for k, conns := range stableConns { + if !addrsToProbe[k] { + if conns[timestampSourceKernel] != nil { + conns[timestampSourceKernel].Close() + } + conns[timestampSourceUserspace].Close() + delete(stableConns, k) + } + } + + for { + select { + case err := <-errCh: + close(doneCh) + wg.Wait() + return nil, err + case result := <-resultsCh: + results = append(results, result) + if len(results) == numProbes { + return results, nil + } + } + } +} + +type connStability bool + +const ( + unstableConn connStability = false + stableConn connStability = true +) + +func timeSeriesLabels(meta nodeMeta, instance string, source timestampSource, stability connStability) []prompb.Label { + addressFamily := "ipv4" + if meta.addr.Is6() { + addressFamily = "ipv6" + } + labels := make([]prompb.Label, 0) + labels = append(labels, prompb.Label{ + Name: "job", + Value: "stunstamp-rw", + }) + labels = append(labels, prompb.Label{ + Name: "instance", + Value: instance, + }) + labels = append(labels, prompb.Label{ + Name: "region_id", + Value: fmt.Sprintf("%d", meta.regionID), + }) + labels = append(labels, prompb.Label{ + Name: "region_code", + Value: meta.regionCode, + }) + labels = append(labels, prompb.Label{ + Name: "address_family", + Value: addressFamily, + }) + labels = append(labels, prompb.Label{ + Name: "hostname", + Value: meta.hostname, + }) + labels = append(labels, prompb.Label{ + Name: "__name__", + Value: "stunstamp_derp_stun_rtt_ns", + }) + labels = append(labels, prompb.Label{ + Name: "timestamp_source", + Value: source.String(), + }) + labels = append(labels, prompb.Label{ + Name: "stable_conn", + Value: fmt.Sprintf("%v", stability), + }) + slices.SortFunc(labels, func(a, b prompb.Label) int { + // prometheus remote-write spec requires lexicographically sorted label names + return cmp.Compare(a.Name, b.Name) + }) + return labels +} + +const ( + // https://prometheus.io/docs/concepts/remote_write_spec/#stale-markers + staleNaN uint64 = 0x7ff0000000000002 +) + +func staleMarkersFromNodeMeta(stale []nodeMeta, instance string) []prompb.TimeSeries { + staleMarkers := make([]prompb.TimeSeries, 0) + now := time.Now() + for _, s := range stale { + samples := []prompb.Sample{ + { + Timestamp: now.UnixMilli(), + Value: math.Float64frombits(staleNaN), + }, + } + staleMarkers = append(staleMarkers, prompb.TimeSeries{ + Labels: timeSeriesLabels(s, instance, timestampSourceUserspace, unstableConn), + Samples: samples, + }) + staleMarkers = append(staleMarkers, prompb.TimeSeries{ + Labels: timeSeriesLabels(s, instance, timestampSourceUserspace, stableConn), + Samples: samples, + }) + if supportsKernelTS() { + staleMarkers = append(staleMarkers, prompb.TimeSeries{ + Labels: timeSeriesLabels(s, instance, timestampSourceKernel, unstableConn), + Samples: samples, + }) + staleMarkers = append(staleMarkers, prompb.TimeSeries{ + Labels: timeSeriesLabels(s, instance, timestampSourceKernel, stableConn), + Samples: samples, + }) + } + } + return staleMarkers +} + +func resultToPromTimeSeries(r result, instance string) prompb.TimeSeries { + labels := timeSeriesLabels(r.meta, instance, r.timestampSource, r.connStability) + samples := make([]prompb.Sample, 1) + samples[0].Timestamp = r.at.UnixMilli() + if r.rtt != nil { + samples[0].Value = float64(*r.rtt) + } else { + samples[0].Value = math.NaN() + // TODO: timeout counter + } + ts := prompb.TimeSeries{ + Labels: labels, + Samples: samples, + } + slices.SortFunc(ts.Labels, func(a, b prompb.Label) int { + // prometheus remote-write spec requires lexicographically sorted label names + return cmp.Compare(a.Name, b.Name) + }) + return ts +} + +type remoteWriteClient struct { + c *http.Client + url string +} + +type recoverableErr struct { + error +} + +func newRemoteWriteClient(url string) *remoteWriteClient { + return &remoteWriteClient{ + c: &http.Client{ + Timeout: time.Second * 30, + }, + url: url, + } +} + +func (r *remoteWriteClient) write(ctx context.Context, ts []prompb.TimeSeries) error { + wr := &prompb.WriteRequest{ + Timeseries: ts, + } + b, err := wr.Marshal() + if err != nil { + return fmt.Errorf("unable to marshal write request: %w", err) + } + compressed := snappy.Encode(nil, b) + req, err := http.NewRequestWithContext(ctx, "POST", r.url, bytes.NewReader(compressed)) + if err != nil { + return fmt.Errorf("unable to create write request: %w", err) + } + req.Header.Add("Content-Encoding", "snappy") + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("User-Agent", "stunstamp") + req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0") + resp, err := r.c.Do(req) + if err != nil { + return recoverableErr{fmt.Errorf("error performing write request: %w", err)} + } + if resp.StatusCode/100 != 2 { + err = fmt.Errorf("remote server %s returned HTTP status %d", r.url, resp.StatusCode) + } + if resp.StatusCode/100 == 5 || resp.StatusCode == http.StatusTooManyRequests { + return recoverableErr{err} + } + return err +} + +func remoteWriteTimeSeries(client *remoteWriteClient, tsCh chan []prompb.TimeSeries) { + bo := backoff.NewBackoff("remote-write", log.Printf, time.Second*30) + for ts := range tsCh { + for { + reqCtx, cancel := context.WithTimeout(context.Background(), time.Second*30) + err := client.write(reqCtx, ts) + cancel() + // we could parse the Retry-After header, but use a simple exp + // backoff for now + bo.BackOff(context.Background(), err) + if err == nil { + break + } + var re recoverableErr + if !errors.Is(err, &re) { + log.Printf("unrecoverable remote write error: %v", err) + break + } + } + } +} + +func main() { + flag.Parse() + if len(*flagDERPMap) < 1 { + log.Fatal("derp-map flag is unset") + } + if len(*flagOut) < 1 { + log.Fatal("out flag is unset") + } + if *flagInterval < minInterval || *flagInterval > maxBufferDuration { + log.Fatalf("interval must be >= %s and <= %s", minInterval, maxBufferDuration) + } + if *flagRetention < *flagInterval { + log.Fatalf("retention must be >= interval") + } + if len(*flagRemoteWriteURL) < 1 { + log.Fatalf("rw-url flag is unset") + } + _, err := url.Parse(*flagRemoteWriteURL) + if err != nil { + log.Fatalf("invalid rw-url flag value: %v", err) + } + if len(*flagInstance) < 1 { + hostname, err := os.Hostname() + if err != nil { + log.Fatalf("failed to get hostname: %v", err) + } + *flagInstance = hostname + } + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + dmCh := make(chan *tailcfg.DERPMap) + + go func() { + bo := backoff.NewBackoff("derp-map", log.Printf, time.Second*30) + for { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + dm, err := getDERPMap(ctx, *flagDERPMap) + cancel() + bo.BackOff(context.Background(), err) + if err != nil { + continue + } + dmCh <- dm + return + } + }() + + nodeMetaByAddr := make(map[netip.Addr]nodeMeta) + select { + case <-sigCh: + return + case dm := <-dmCh: + _, err := nodeMetaFromDERPMap(dm, nodeMetaByAddr, *flagIPv6) + if err != nil { + log.Fatalf("error parsing derp map on startup: %v", err) + } + } + + db, err := newDB(*flagOut) + if err != nil { + log.Fatalf("error opening output file for writing: %v", err) + } + defer db.Close() + + _, err = db.Exec("PRAGMA journal_mode=WAL") + if err != nil { + log.Fatalf("error enabling WAL mode: %v", err) + } + + // No indices or primary key. Keep it simple for now. Reads will be full + // scans. We can AUTOINCREMENT rowid in the future and hold an in-memory + // index to at_unix if needed as reads are almost always going to be + // time-bound (e.g. WHERE at_unix >= ?). At the time of authorship we have + // ~300 data points per-interval w/o ipv6 w/kernel timestamping resulting + // in ~2.6m rows in 24h w/a 10s probe interval. + _, err = db.Exec(` +CREATE TABLE IF NOT EXISTS rtt(at_unix INT, region_id INT, hostname TEXT, af INT, address TEXT, timestamp_source INT, stable_conn INT, rtt_ns INT) +`) + if err != nil { + log.Fatalf("error initializing db: %v", err) + } + + wg := sync.WaitGroup{} + httpErrCh := make(chan error, 1) + var httpServer *http.Server + if len(*flagAPI) > 0 { + api := newAPI(db) + httpServer = &http.Server{ + Addr: *flagAPI, + Handler: api, + ReadTimeout: time.Second * 60, + WriteTimeout: time.Second * 60, + } + wg.Add(1) + go func() { + err := httpServer.ListenAndServe() + httpErrCh <- err + wg.Done() + }() + } + + tsCh := make(chan []prompb.TimeSeries, maxBufferDuration / *flagInterval) + remoteWriteDoneCh := make(chan struct{}) + rwc := newRemoteWriteClient(*flagRemoteWriteURL) + go func() { + remoteWriteTimeSeries(rwc, tsCh) + close(remoteWriteDoneCh) + }() + + shutdown := func() { + if httpServer != nil { + httpServer.Close() + } + close(tsCh) + select { + case <-time.After(time.Second * 10): // give goroutine some time to flush + case <-remoteWriteDoneCh: + } + + // send stale markers on shutdown + staleMeta := make([]nodeMeta, 0, len(nodeMetaByAddr)) + for _, v := range nodeMetaByAddr { + staleMeta = append(staleMeta, v) + } + staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance) + if len(staleMarkers) > 0 { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + rwc.write(ctx, staleMarkers) + cancel() + } + + wg.Wait() + return + } + + log.Println("stunstamp started") + + // Re-using sockets means we get the same 5-tuple across runs. This results + // in a higher probability of the packets traversing the same underlay path. + // Comparison of stable and unstable 5-tuple results can shed light on + // differences between paths where hashing (multipathing/load balancing) + // comes into play. + stableConns := make(map[netip.Addr][2]io.ReadWriteCloser) + + derpMapTicker := time.NewTicker(time.Minute * 5) + defer derpMapTicker.Stop() + probeTicker := time.NewTicker(*flagInterval) + defer probeTicker.Stop() + cleanupTicker := time.NewTicker(time.Hour) + defer cleanupTicker.Stop() + + for { + select { + case <-cleanupTicker.C: + older := time.Now().Add(-*flagRetention) + log.Printf("cleaning up measurements older than %v", older) + _, err := db.Exec("DELETE FROM rtt WHERE at_unix < ?", older.Unix()) + if err != nil { + log.Printf("error cleaning up old data: %v", err) + shutdown() + return + } + case <-probeTicker.C: + results, err := probeNodes(nodeMetaByAddr, stableConns) + if err != nil { + log.Printf("unrecoverable error while probing: %v", err) + shutdown() + return + } + ts := make([]prompb.TimeSeries, 0, len(results)) + for _, r := range results { + ts = append(ts, resultToPromTimeSeries(r, *flagInstance)) + } + select { + case tsCh <- ts: + default: + select { + case <-tsCh: + log.Println("prometheus remote-write buffer full, dropped measurements") + default: + tsCh <- ts + } + } + tx, err := db.Begin() + if err != nil { + log.Printf("error beginning sqlite tx: %v", err) + shutdown() + return + } + for _, result := range results { + af := 4 + if result.meta.addr.Is6() { + af = 6 + } + _, err = tx.Exec("INSERT INTO rtt(at_unix, region_id, hostname, af, address, timestamp_source, stable_conn, rtt_ns) VALUES(?, ?, ?, ?, ?, ?, ?, ?)", + result.at.Unix(), result.meta.regionID, result.meta.hostname, af, result.meta.addr.String(), result.timestampSource, result.connStability, result.rtt) + if err != nil { + tx.Rollback() + log.Printf("error adding result to tx: %v", err) + shutdown() + return + } + } + err = tx.Commit() + if err != nil { + log.Printf("error committing tx: %v", err) + shutdown() + return + } + case dm := <-dmCh: + staleMeta, err := nodeMetaFromDERPMap(dm, nodeMetaByAddr, *flagIPv6) + if err != nil { + log.Printf("error parsing DERP map, continuing with stale map: %v", err) + continue + } + staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance) + if len(staleMarkers) < 1 { + continue + } + select { + case tsCh <- staleMarkers: + default: + select { + case <-tsCh: + log.Println("prometheus remote-write buffer full, dropped measurements") + default: + tsCh <- staleMarkers + } + } + case <-derpMapTicker.C: + go func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + updatedDM, err := getDERPMap(ctx, *flagDERPMap) + if err != nil { + dmCh <- updatedDM + } + }() + case err := <-httpErrCh: + log.Printf("http server error: %v", err) + shutdown() + return + case <-sigCh: + shutdown() + return + } + } +} diff --git a/cmd/stunstamp/stunstamp_db_default.go b/cmd/stunstamp/stunstamp_db_default.go new file mode 100644 index 000000000..3de3a0197 --- /dev/null +++ b/cmd/stunstamp/stunstamp_db_default.go @@ -0,0 +1,26 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !(windows && 386) + +package main + +import ( + "database/sql" + + _ "modernc.org/sqlite" +) + +type db struct { + *sql.DB +} + +func newDB(path string) (*db, error) { + d, err := sql.Open("sqlite", *flagOut) + if err != nil { + return nil, err + } + return &db{ + DB: d, + }, nil +} diff --git a/cmd/stunstamp/stunstamp_db_windows_386.go b/cmd/stunstamp/stunstamp_db_windows_386.go new file mode 100644 index 000000000..67dba08ca --- /dev/null +++ b/cmd/stunstamp/stunstamp_db_windows_386.go @@ -0,0 +1,17 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "database/sql" + "errors" +) + +type db struct { + *sql.DB +} + +func newDB(path string) (*db, error) { + return nil, errors.New("unsupported platform") +} diff --git a/cmd/stunstamp/stunstamp_default.go b/cmd/stunstamp/stunstamp_default.go new file mode 100644 index 000000000..74f2ca4c9 --- /dev/null +++ b/cmd/stunstamp/stunstamp_default.go @@ -0,0 +1,25 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !linux + +package main + +import ( + "errors" + "io" + "net" + "time" +) + +func getConnKernelTimestamp() (io.ReadWriteCloser, error) { + return nil, errors.New("unimplemented") +} + +func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr, req []byte) (resp []byte, rtt time.Duration, err error) { + return nil, 0, errors.New("unimplemented") +} + +func supportsKernelTS() bool { + return false +} diff --git a/cmd/stunstamp/stunstamp_linux.go b/cmd/stunstamp/stunstamp_linux.go new file mode 100644 index 000000000..b9b78b07e --- /dev/null +++ b/cmd/stunstamp/stunstamp_linux.go @@ -0,0 +1,126 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "time" + + "github.com/mdlayher/socket" + "golang.org/x/sys/unix" +) + +const ( + flags = unix.SOF_TIMESTAMPING_TX_SOFTWARE | // tx timestamp generation in device driver + unix.SOF_TIMESTAMPING_RX_SOFTWARE | // rx timestamp generation in the kernel + unix.SOF_TIMESTAMPING_SOFTWARE // report software timestamps +) + +func getConnKernelTimestamp() (io.ReadWriteCloser, error) { + sconn, err := socket.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_UDP, "udp", nil) + if err != nil { + return nil, err + } + sa := unix.SockaddrInet6{} + err = sconn.Bind(&sa) + if err != nil { + return nil, err + } + err = sconn.SetsockoptInt(unix.SOL_SOCKET, unix.SO_TIMESTAMPING_NEW, flags) + if err != nil { + return nil, err + } + return sconn, nil +} + +func parseTimestampFromCmsgs(oob []byte) (time.Time, error) { + msgs, err := unix.ParseSocketControlMessage(oob) + if err != nil { + return time.Time{}, fmt.Errorf("error parsing oob as cmsgs: %w", err) + } + for _, msg := range msgs { + if msg.Header.Level == unix.SOL_SOCKET && msg.Header.Type == unix.SO_TIMESTAMPING_NEW && len(msg.Data) >= 16 { + sec := int64(binary.NativeEndian.Uint64(msg.Data[:8])) + ns := int64(binary.NativeEndian.Uint64(msg.Data[8:16])) + return time.Unix(sec, ns), nil + } + } + return time.Time{}, errors.New("failed to parse timestamp from cmsgs") +} + +func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr, req []byte) (resp []byte, rtt time.Duration, err error) { + sconn, ok := conn.(*socket.Conn) + if !ok { + return nil, 0, fmt.Errorf("conn of unexpected type: %T", conn) + } + + var to unix.Sockaddr + to4 := dst.IP.To4() + if to4 != nil { + to = &unix.SockaddrInet4{ + Port: 3478, + } + copy(to.(*unix.SockaddrInet4).Addr[:], to4) + } else { + to = &unix.SockaddrInet6{ + Port: 3478, + } + copy(to.(*unix.SockaddrInet6).Addr[:], dst.IP) + } + + err = sconn.Sendto(context.Background(), req, 0, to) + if err != nil { + return nil, 0, fmt.Errorf("sendto error: %v", err) // don't wrap + } + + txCtx, txCancel := context.WithTimeout(context.Background(), time.Second*2) + defer txCancel() + + buf := make([]byte, 1024) + oob := make([]byte, 1024) + var txAt time.Time + + for { + n, oobn, _, _, err := sconn.Recvmsg(txCtx, buf, oob, unix.MSG_ERRQUEUE) + if err != nil { + return nil, 0, fmt.Errorf("recvmsg (MSG_ERRQUEUE) error: %v", err) // don't wrap + } + + buf = buf[:n] + if n < len(req) || !bytes.Equal(req, buf[len(buf)-len(req):]) { + // Spin until we find the message we sent. We get the full packet + // looped including eth header so match against the tail. + continue + } + txAt, err = parseTimestampFromCmsgs(oob[:oobn]) + if err != nil { + return nil, 0, fmt.Errorf("failed to get tx timestamp: %v", err) // don't wrap + } + break + } + + rxCtx, rxCancel := context.WithTimeout(context.Background(), time.Second*2) + defer rxCancel() + n, oobn, _, _, err := sconn.Recvmsg(rxCtx, buf, oob, 0) + if err != nil { + return nil, 0, fmt.Errorf("recvmsg error: %w", err) // wrap for timeout-related error unwrapping + } + + rxAt, err := parseTimestampFromCmsgs(oob[:oobn]) + if err != nil { + return nil, 0, fmt.Errorf("failed to get rx timestamp: %v", err) // don't wrap + } + + return buf[:n], rxAt.Sub(txAt), nil +} + +func supportsKernelTS() bool { + return true +} diff --git a/go.mod b/go.mod index 922667e5e..35900ee33 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.22.0 require ( filippo.io/mkcert v1.4.4 fybrik.io/crdoc v0.6.3 + github.com/Masterminds/squirrel v1.5.4 github.com/akutz/memconn v0.1.0 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa github.com/andybalholm/brotli v1.1.0 @@ -34,6 +35,7 @@ require ( github.com/go-ole/go-ole v1.3.0 github.com/godbus/dbus/v5 v5.1.1-0.20230522191255-76236955d466 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da + github.com/golang/snappy v0.0.4 github.com/golangci/golangci-lint v1.52.2 github.com/google/go-cmp v0.6.0 github.com/google/go-containerregistry v0.18.0 @@ -64,6 +66,7 @@ require ( github.com/prometheus-community/pro-bing v0.4.0 github.com/prometheus/client_golang v1.18.0 github.com/prometheus/common v0.46.0 + github.com/prometheus/prometheus v0.49.2-0.20240125131847-c3b8ef1694ff github.com/safchain/ethtool v0.3.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/studio-b12/gowebdav v0.9.0 @@ -91,14 +94,14 @@ require ( go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/crypto v0.21.0 golang.org/x/exp v0.0.0-20240119083558-1b970713d09a - golang.org/x/mod v0.14.0 + golang.org/x/mod v0.16.0 golang.org/x/net v0.23.0 golang.org/x/oauth2 v0.16.0 golang.org/x/sync v0.6.0 - golang.org/x/sys v0.18.0 + golang.org/x/sys v0.19.0 golang.org/x/term v0.18.0 golang.org/x/time v0.5.0 - golang.org/x/tools v0.17.0 + golang.org/x/tools v0.19.0 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 golang.zx2c4.com/wireguard/windows v0.5.3 gopkg.in/square/go-jose.v2 v2.6.0 @@ -108,6 +111,7 @@ require ( k8s.io/apimachinery v0.29.1 k8s.io/apiserver v0.29.1 k8s.io/client-go v0.29.1 + modernc.org/sqlite v1.29.10 nhooyr.io/websocket v1.8.10 sigs.k8s.io/controller-runtime v0.16.2 sigs.k8s.io/controller-tools v0.13.0 @@ -121,9 +125,21 @@ require ( github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14 // indirect github.com/dave/brenda v1.1.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/gobuffalo/flect v1.0.2 // indirect github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect github.com/gorilla/securecookie v1.1.2 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect + github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect + modernc.org/libc v1.49.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect + modernc.org/strutil v1.2.0 // indirect + modernc.org/token v1.1.0 // indirect ) require ( @@ -179,7 +195,7 @@ require ( github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect github.com/curioswitch/go-reassign v0.2.0 // indirect github.com/daixiang0/gci v0.10.1 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denis-tingaikin/go-header v0.4.3 // indirect github.com/docker/cli v25.0.0+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect @@ -276,7 +292,7 @@ require ( github.com/matoous/godox v0.0.0-20230222163458-006bad1f9d26 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/mbilski/exhaustivestruct v1.2.0 // indirect - github.com/mdlayher/socket v0.5.0 // indirect + github.com/mdlayher/socket v0.5.0 github.com/mgechev/revive v1.3.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -299,7 +315,7 @@ require ( github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polyfloyd/go-errorlint v1.4.1 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect diff --git a/go.sum b/go.sum index 59d1a8034..562a1dff3 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= +github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= +github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= @@ -240,8 +242,9 @@ github.com/dave/jennifer v1.7.0/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3 github.com/dave/patsy v0.0.0-20210517141501-957256f50cba h1:1o36L4EKbZzazMk8iGC4kXpVnZ6TPxR2mZ9qVKjNNAs= github.com/dave/patsy v0.0.0-20210517141501-957256f50cba/go.mod h1:qfR88CgEGLoiqDaE+xxDCi5QA5v4vUoW0UCX2Nd5Tlc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa h1:h8TfIT1xc8FWbwwpmHn1J5i43Y0uZP97GqasGCzSRJk= github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa/go.mod h1:Nx87SkVqTKd8UtT+xu7sM/l+LgXs6c0aHrlKusR+2EQ= github.com/denis-tingaikin/go-header v0.4.3 h1:tEaZKAlqql6SKCY++utLmkPLd6K8IBM20Ha7UVm+mtU= @@ -260,6 +263,8 @@ github.com/docker/docker-credential-helpers v0.8.1 h1:j/eKUktUltBtMzKqmfLB0PAgqY github.com/docker/docker-credential-helpers v0.8.1/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI= github.com/dsnet/try v0.0.3/go.mod h1:WBM8tRpUmnXXhY1U6/S8dt6UWdHTQ7y8A5YSkRCkq40= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emicklei/go-restful/v3 v3.11.2 h1:1onLa9DcsMYO9P+CXaL0dStDqQ2EHHXLiz+BtnqkLAU= @@ -400,6 +405,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0= github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM= @@ -463,8 +470,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/rpmpack v0.5.0 h1:L16KZ3QvkFGpYhmp23iQip+mx1X39foEsqszjMNBm8A= github.com/google/rpmpack v0.5.0/go.mod h1:uqVAUVQLq8UY2hCDfmJ/+rtO3aw7qyhc90rCVEabEfI= @@ -511,6 +518,8 @@ github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mO github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hdevalence/ed25519consensus v0.2.0 h1:37ICyZqdyj0lAZ8P4D1d1id3HqbbG1N3iBb1Tb4rdcU= @@ -610,6 +619,10 @@ github.com/kunwardeep/paralleltest v1.0.6 h1:FCKYMF1OF2+RveWlABsdnmsvJrei5aoyZoa github.com/kunwardeep/paralleltest v1.0.6/go.mod h1:Y0Y0XISdZM5IKm3TREQMZ6iteqn1YuwCsJO/0kL9Zes= github.com/kyoh86/exportloopref v0.1.11 h1:1Z0bcmTypkL3Q4k+IDHMWTcnCliEZcaPiIe0/ymEyhQ= github.com/kyoh86/exportloopref v0.1.11/go.mod h1:qkV4UF1zGl6EkF1ox8L5t9SwyeBAZ3qLMd6up458uqA= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= github.com/ldez/gomoddirectives v0.2.3 h1:y7MBaisZVDYmKvt9/l1mjNCiSA1BVn34U0ObUcJwlhA= github.com/ldez/gomoddirectives v0.2.3/go.mod h1:cpgBogWITnCfRq2qGoDkKMEVSaarhdBr6g8G04uz6d0= github.com/ldez/tagliatelle v0.5.0 h1:epgfuYt9v0CG3fms0pEgIMNPuFf/LpPIfjk4kyqSioo= @@ -682,6 +695,8 @@ github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81 github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4Ngq6aY7OE= github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 h1:4kuARK6Y6FxaNu/BnU2OAaLF86eTVhP2hjTB6iMvItA= github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354/go.mod h1:KSVJerMDfblTH7p5MZaTt+8zaT2iEk3AkVb9PQdZuE8= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/nishanths/exhaustive v0.10.0 h1:BMznKAcVa9WOoLq/kTGp4NJOJSMwEpcpjFNAVRfPlSo= @@ -729,8 +744,9 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polyfloyd/go-errorlint v1.4.1 h1:r8ru5FhXSn34YU1GJDOuoJv2LdsQkPmK325EOpPMJlM= github.com/polyfloyd/go-errorlint v1.4.1/go.mod h1:k6fU/+fQe38ednoZS51T7gSIGQW1y94d6TkSr35OzH8= github.com/prometheus-community/pro-bing v0.4.0 h1:YMbv+i08gQz97OZZBwLyvmmQEEzyfyrrjEaAchdy3R4= @@ -761,6 +777,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/prometheus v0.49.2-0.20240125131847-c3b8ef1694ff h1:X1Tly81aZ22DA1fxBdfvR3iw8+yFoUBUHMEd+AX/ZXI= +github.com/prometheus/prometheus v0.49.2-0.20240125131847-c3b8ef1694ff/go.mod h1:FvE8dtQ1Ww63IlyKBn1V4s+zMwF9kHkVNkQBR1pM4CU= github.com/quasilyte/go-ruleguard v0.3.19 h1:tfMnabXle/HzOb5Xe9CUZYWXKfkS1KwRmZyPmD9nVcc= github.com/quasilyte/go-ruleguard v0.3.19/go.mod h1:lHSn69Scl48I7Gt9cX3VrbsZYvYiBYszZOZW4A+oTEw= github.com/quasilyte/gogrep v0.5.0 h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo= @@ -769,6 +787,8 @@ github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 h1:TCg2WBOl github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -964,8 +984,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= @@ -1038,8 +1058,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1179,8 +1199,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1285,8 +1305,8 @@ golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1453,6 +1473,32 @@ k8s.io/kube-openapi v0.0.0-20240117194847-208609032b15 h1:m6dl1pkxz3HuE2mP9MUYPC k8s.io/kube-openapi v0.0.0-20240117194847-208609032b15/go.mod h1:Pa1PvrP7ACSkuX6I7KYomY6cmMA0Tx86waBhDUgoKPw= k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +modernc.org/cc/v4 v4.20.0 h1:45Or8mQfbUqJOG9WaxvlFYOAQO0lQ5RvqBcFCXngjxk= +modernc.org/cc/v4 v4.20.0/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.16.0 h1:ofwORa6vx2FMm0916/CkZjpFPSR70VwTjUCe2Eg5BnA= +modernc.org/ccgo/v4 v4.16.0/go.mod h1:dkNyWIjFrVIZ68DTo36vHK+6/ShBn4ysU61So6PIqCI= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= +modernc.org/libc v1.49.3 h1:j2MRCRdwJI2ls/sGbeSk0t2bypOG/uvPZUsGQFDulqg= +modernc.org/libc v1.49.3/go.mod h1:yMZuGkn7pXbKfoT/M35gFJOAEdSKdxL0q64sF7KqCDo= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.29.10 h1:3u93dz83myFnMilBGCOLbr+HjklS6+5rJLx4q86RDAg= +modernc.org/sqlite v1.29.10/go.mod h1:ItX2a1OVGgNsFh6Dv60JQvGfJfTPHPVpV6DF59akYOA= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= mvdan.cc/gofumpt v0.5.0 h1:0EQ+Z56k8tXjj/6TQD25BFNKQXpCvT0rnansIc7Ug5E= mvdan.cc/gofumpt v0.5.0/go.mod h1:HBeVDtMKRZpXyxFciAirzdKklDlGu8aAy1wEbH5Y9js= mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=