From c9a5d638e9d8895eda0cb175afcb8dc738e75a09 Mon Sep 17 00:00:00 2001 From: Fran Bull Date: Tue, 27 May 2025 08:06:45 -0700 Subject: [PATCH] tsconsensus: enable writing state to disk The comments in the raft code say to only use the InMemStore for tests. Updates #16027 Signed-off-by: Fran Bull --- go.mod | 3 +++ go.sum | 10 +++++++++ tsconsensus/bolt_store.go | 19 ++++++++++++++++ tsconsensus/bolt_store_no_bolt.go | 18 +++++++++++++++ tsconsensus/tsconsensus.go | 37 +++++++++++++++++++++++++------ 5 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 tsconsensus/bolt_store.go create mode 100644 tsconsensus/bolt_store_no_bolt.go diff --git a/go.mod b/go.mod index d44a14aef..9ea25446b 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( github.com/goreleaser/nfpm/v2 v2.33.1 github.com/hashicorp/go-hclog v1.6.2 github.com/hashicorp/raft v1.7.2 + github.com/hashicorp/raft-boltdb/v2 v2.3.1 github.com/hdevalence/ed25519consensus v0.2.0 github.com/illarion/gonotify/v3 v3.0.2 github.com/inetaf/tcpproxy v0.0.0-20250203165043-ded522cbd03f @@ -135,6 +136,7 @@ require ( github.com/alecthomas/go-check-sumtype v0.1.4 // indirect github.com/alexkohler/nakedret/v2 v2.0.4 // indirect github.com/armon/go-metrics v0.4.1 // indirect + github.com/boltdb/bolt v1.3.1 // indirect github.com/bombsimon/wsl/v4 v4.2.1 // indirect github.com/butuzov/mirror v1.1.0 // indirect github.com/catenacyber/perfsprint v0.7.1 // indirect @@ -166,6 +168,7 @@ require ( github.com/ykadowak/zerologlint v0.1.5 // indirect go-simpler.org/musttag v0.9.0 // indirect go-simpler.org/sloglint v0.5.0 // indirect + go.etcd.io/bbolt v1.3.11 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect go.opentelemetry.io/otel v1.33.0 // indirect diff --git a/go.sum b/go.sum index 73d87fd66..318eae1ea 100644 --- a/go.sum +++ b/go.sum @@ -180,6 +180,8 @@ github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= github.com/blizzy78/varnamelen v0.8.0 h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ089M= github.com/blizzy78/varnamelen v0.8.0/go.mod h1:V9TzQZ4fLJ1DSrjVDfl89H7aMnTvKkApdHeyESmyR7k= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bombsimon/wsl/v4 v4.2.1 h1:Cxg6u+XDWff75SIFFmNsqnIOgob+Q9hG6y/ioKbRFiM= github.com/bombsimon/wsl/v4 v4.2.1/go.mod h1:Xu/kDxGZTofQcDGCtQe9KCzhHphIe0fDuyWTxER9Feo= github.com/bramvdbogaerde/go-scp v1.4.0 h1:jKMwpwCbcX1KyvDbm/PDJuXcMuNVlLGi0Q0reuzjyKY= @@ -555,6 +557,8 @@ github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJ github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-metrics v0.5.4 h1:8mmPiIJkTPPEbAiV97IxdAGNdRdaWwVap1BU6elejKY= github.com/hashicorp/go-metrics v0.5.4/go.mod h1:CG5yz4NZ/AI/aQt9Ucm/vdBnbh7fvmv4lxZ350i+QQI= +github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack/v2 v2.1.2 h1:4Ee8FTp834e+ewB71RDrQ0VKpyFdrKOjvYtnQ/ltVj0= github.com/hashicorp/go-msgpack/v2 v2.1.2/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= @@ -571,6 +575,10 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/raft v1.7.2 h1:pyvxhfJ4R8VIAlHKvLoKQWElZspsCVT6YWuxVxsPAgc= github.com/hashicorp/raft v1.7.2/go.mod h1:DfvCGFxpAUPE0L4Uc8JLlTPtc3GzSbdH0MTJCLgnmJQ= +github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702 h1:RLKEcCuKcZ+qp2VlaaZsYZfLOmIiuJNpEi48Rl8u9cQ= +github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0= +github.com/hashicorp/raft-boltdb/v2 v2.3.1 h1:ackhdCNPKblmOhjEU9+4lHSJYFkJd6Jqyvj6eW9pwkc= +github.com/hashicorp/raft-boltdb/v2 v2.3.1/go.mod h1:n4S+g43dXF1tqDT+yzcXHhXM6y7MrlUd3TTwGRcUvQE= github.com/hdevalence/ed25519consensus v0.2.0 h1:37ICyZqdyj0lAZ8P4D1d1id3HqbbG1N3iBb1Tb4rdcU= github.com/hdevalence/ed25519consensus v0.2.0/go.mod h1:w3BHWjwJbFU29IRHL1Iqkw3sus+7FctEyM4RqDxYNzo= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -1046,6 +1054,8 @@ go-simpler.org/musttag v0.9.0 h1:Dzt6/tyP9ONr5g9h9P3cnYWCxeBFRkd0uJL/w+1Mxos= go-simpler.org/musttag v0.9.0/go.mod h1:gA9nThnalvNSKpEoyp3Ko4/vCX2xTpqKoUtNqXOnVR4= go-simpler.org/sloglint v0.5.0 h1:2YCcd+YMuYpuqthCgubcF5lBSjb6berc5VMOYUHKrpY= go-simpler.org/sloglint v0.5.0/go.mod h1:EUknX5s8iXqf18KQxKnaBHUPVriiPnOrPjjJcsaTcSQ= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/tsconsensus/bolt_store.go b/tsconsensus/bolt_store.go new file mode 100644 index 000000000..ca347cfc0 --- /dev/null +++ b/tsconsensus/bolt_store.go @@ -0,0 +1,19 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !loong64 + +package tsconsensus + +import ( + "github.com/hashicorp/raft" + raftboltdb "github.com/hashicorp/raft-boltdb/v2" +) + +func boltStore(path string) (raft.StableStore, raft.LogStore, error) { + store, err := raftboltdb.NewBoltStore(path) + if err != nil { + return nil, nil, err + } + return store, store, nil +} diff --git a/tsconsensus/bolt_store_no_bolt.go b/tsconsensus/bolt_store_no_bolt.go new file mode 100644 index 000000000..33b3bd6c7 --- /dev/null +++ b/tsconsensus/bolt_store_no_bolt.go @@ -0,0 +1,18 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build loong64 + +package tsconsensus + +import ( + "errors" + + "github.com/hashicorp/raft" +) + +func boltStore(path string) (raft.StableStore, raft.LogStore, error) { + // "github.com/hashicorp/raft-boltdb/v2" doesn't build on loong64 + // see https://github.com/hashicorp/raft-boltdb/issues/27 + return nil, nil, errors.New("not implemented") +} diff --git a/tsconsensus/tsconsensus.go b/tsconsensus/tsconsensus.go index 74094782f..b6bf37310 100644 --- a/tsconsensus/tsconsensus.go +++ b/tsconsensus/tsconsensus.go @@ -32,6 +32,7 @@ import ( "net" "net/http" "net/netip" + "path/filepath" "time" "github.com/hashicorp/go-hclog" @@ -71,6 +72,7 @@ type Config struct { MaxConnPool int ConnTimeout time.Duration ServeDebugMonitor bool + StateDirPath string } // DefaultConfig returns a Config populated with default values ready for use. @@ -223,10 +225,31 @@ func Start(ctx context.Context, ts *tsnet.Server, fsm raft.FSM, clusterTag strin func startRaft(shutdownCtx context.Context, ts *tsnet.Server, fsm *raft.FSM, self selfRaftNode, auth *authorization, cfg Config) (*raft.Raft, error) { cfg.Raft.LocalID = raft.ServerID(self.id) - // no persistence (for now?) - logStore := raft.NewInmemStore() - stableStore := raft.NewInmemStore() - snapshots := raft.NewInmemSnapshotStore() + var logStore raft.LogStore + var stableStore raft.StableStore + var snapStore raft.SnapshotStore + + if cfg.StateDirPath == "" { + // comments in raft code say to only use for tests + logStore = raft.NewInmemStore() + stableStore = raft.NewInmemStore() + snapStore = raft.NewInmemSnapshotStore() + } else { + var err error + stableStore, logStore, err = boltStore(filepath.Join(cfg.StateDirPath, "store")) + if err != nil { + return nil, err + } + snaplogger := hclog.New(&hclog.LoggerOptions{ + Name: "raft-snap", + Output: cfg.Raft.LogOutput, + Level: hclog.LevelFromString(cfg.Raft.LogLevel), + }) + snapStore, err = raft.NewFileSnapshotStoreWithLogger(filepath.Join(cfg.StateDirPath, "snapstore"), 2, snaplogger) + if err != nil { + return nil, err + } + } // opens the listener on the raft port, raft will close it when it thinks it's appropriate ln, err := ts.Listen("tcp", raftAddr(self.hostAddr, cfg)) @@ -234,7 +257,7 @@ func startRaft(shutdownCtx context.Context, ts *tsnet.Server, fsm *raft.FSM, sel return nil, err } - logger := hclog.New(&hclog.LoggerOptions{ + transportLogger := hclog.New(&hclog.LoggerOptions{ Name: "raft-net", Output: cfg.Raft.LogOutput, Level: hclog.LevelFromString(cfg.Raft.LogLevel), @@ -248,9 +271,9 @@ func startRaft(shutdownCtx context.Context, ts *tsnet.Server, fsm *raft.FSM, sel }, cfg.MaxConnPool, cfg.ConnTimeout, - logger) + transportLogger) - return raft.NewRaft(cfg.Raft, *fsm, logStore, stableStore, snapshots, transport) + return raft.NewRaft(cfg.Raft, *fsm, logStore, stableStore, snapStore, transport) } // A Consensus is the consensus algorithm for a tsnet.Server