ipn/store: add common package for instantiating ipn.StateStores

Also move KubeStore and MemStore into their own package.

RELNOTE: tsnet now supports providing a custom ipn.StateStore.

Signed-off-by: Maisem Ali <maisem@tailscale.com>

cmd/tailscale/depaware.txt

@@ -42,7 +42,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/hostinfo                                       from tailscale.com/net/interfaces+
         tailscale.com/ipn                                            from tailscale.com/cmd/tailscale/cli+
         tailscale.com/ipn/ipnstate                                   from tailscale.com/cmd/tailscale/cli+
-        tailscale.com/kube                                           from tailscale.com/ipn
      💣 tailscale.com/metrics                                        from tailscale.com/derp
         tailscale.com/net/dnscache                                   from tailscale.com/derp/derphttp
         tailscale.com/net/flowtrack                                  from tailscale.com/wgengine/filter+

cmd/tailscaled/depaware.txt

@@ -6,7 +6,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
   LD    github.com/anmitsu/go-shlex                                  from github.com/gliderlabs/ssh
    L    github.com/aws/aws-sdk-go-v2                                 from github.com/aws/aws-sdk-go-v2/internal/ini
    L    github.com/aws/aws-sdk-go-v2/aws                             from github.com/aws/aws-sdk-go-v2/aws/middleware+
-   L    github.com/aws/aws-sdk-go-v2/aws/arn                         from tailscale.com/ipn/store/aws
+   L    github.com/aws/aws-sdk-go-v2/aws/arn                         from tailscale.com/ipn/store/awsstore
    L    github.com/aws/aws-sdk-go-v2/aws/middleware                  from github.com/aws/aws-sdk-go-v2/aws/retry+
    L    github.com/aws/aws-sdk-go-v2/aws/protocol/query              from github.com/aws/aws-sdk-go-v2/service/sts
    L    github.com/aws/aws-sdk-go-v2/aws/protocol/restjson           from github.com/aws/aws-sdk-go-v2/service/ssm+
@@ -16,7 +16,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4          from github.com/aws/aws-sdk-go-v2/aws/signer/v4
    L    github.com/aws/aws-sdk-go-v2/aws/signer/v4                   from github.com/aws/aws-sdk-go-v2/service/internal/presigned-url+
    L    github.com/aws/aws-sdk-go-v2/aws/transport/http              from github.com/aws/aws-sdk-go-v2/config+
-   L    github.com/aws/aws-sdk-go-v2/config                          from tailscale.com/ipn/store/aws
+   L    github.com/aws/aws-sdk-go-v2/config                          from tailscale.com/ipn/store/awsstore
    L    github.com/aws/aws-sdk-go-v2/credentials                     from github.com/aws/aws-sdk-go-v2/config
    L    github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds        from github.com/aws/aws-sdk-go-v2/config
    L    github.com/aws/aws-sdk-go-v2/credentials/endpointcreds       from github.com/aws/aws-sdk-go-v2/config
@@ -36,7 +36,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
    L    github.com/aws/aws-sdk-go-v2/internal/sync/singleflight      from github.com/aws/aws-sdk-go-v2/aws
    L    github.com/aws/aws-sdk-go-v2/internal/timeconv               from github.com/aws/aws-sdk-go-v2/aws/retry
    L    github.com/aws/aws-sdk-go-v2/service/internal/presigned-url  from github.com/aws/aws-sdk-go-v2/service/sts
-   L    github.com/aws/aws-sdk-go-v2/service/ssm                     from tailscale.com/ipn/store/aws
+   L    github.com/aws/aws-sdk-go-v2/service/ssm                     from tailscale.com/ipn/store/awsstore
    L    github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints  from github.com/aws/aws-sdk-go-v2/service/ssm
    L    github.com/aws/aws-sdk-go-v2/service/ssm/types               from github.com/aws/aws-sdk-go-v2/service/ssm+
    L    github.com/aws/aws-sdk-go-v2/service/sso                     from github.com/aws/aws-sdk-go-v2/config+
@@ -185,8 +185,11 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/ipn/ipnstate                                   from tailscale.com/client/tailscale+
         tailscale.com/ipn/localapi                                   from tailscale.com/ipn/ipnserver
         tailscale.com/ipn/policy                                     from tailscale.com/ipn/ipnlocal
-        tailscale.com/ipn/store/aws                                  from tailscale.com/ipn/ipnserver
+        tailscale.com/ipn/store                                      from tailscale.com/cmd/tailscaled
-        tailscale.com/kube                                           from tailscale.com/ipn
+   L    tailscale.com/ipn/store/awsstore                             from tailscale.com/ipn/store
+   L    tailscale.com/ipn/store/kubestore                            from tailscale.com/ipn/store
+        tailscale.com/ipn/store/mem                                  from tailscale.com/ipn/store+
+   L    tailscale.com/kube                                           from tailscale.com/ipn/store/kubestore
         tailscale.com/log/filelogger                                 from tailscale.com/logpolicy
         tailscale.com/log/logheap                                    from tailscale.com/control/controlclient
         tailscale.com/logpolicy                                      from tailscale.com/cmd/tailscaled+
@@ -337,7 +340,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         crypto/sha256                                                from crypto/tls+
         crypto/sha512                                                from crypto/ecdsa+
         crypto/subtle                                                from crypto/aes+
-        crypto/tls                                                   from github.com/aws/aws-sdk-go-v2/aws/transport/http+
+        crypto/tls                                                   from github.com/tcnksm/go-httpstat+
         crypto/x509                                                  from crypto/tls+
         crypto/x509/pkix                                             from crypto/x509+
         embed                                                        from tailscale.com/net/dns+
@@ -348,7 +351,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         encoding/hex                                                 from crypto/x509+
         encoding/json                                                from expvar+
         encoding/pem                                                 from crypto/tls+
-        encoding/xml                                                 from github.com/aws/aws-sdk-go-v2/aws/protocol/xml+
+        encoding/xml                                                 from github.com/tailscale/goupnp+
         errors                                                       from bufio+
         expvar                                                       from tailscale.com/derp+
         flag                                                         from tailscale.com/cmd/tailscaled+
@@ -372,19 +375,19 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         net                                                          from crypto/tls+
         net/http                                                     from expvar+
         net/http/httptrace                                           from github.com/tcnksm/go-httpstat+
-        net/http/httputil                                            from github.com/aws/smithy-go/transport/http+
+        net/http/httputil                                            from tailscale.com/cmd/tailscaled+
         net/http/internal                                            from net/http+
         net/http/pprof                                               from tailscale.com/cmd/tailscaled+
-        net/textproto                                                from github.com/aws/aws-sdk-go-v2/aws/signer/v4+
+        net/textproto                                                from golang.org/x/net/http/httpguts+
         net/url                                                      from crypto/x509+
         os                                                           from crypto/rand+
-        os/exec                                                      from github.com/aws/aws-sdk-go-v2/credentials/processcreds+
+        os/exec                                                      from github.com/coreos/go-iptables/iptables+
         os/signal                                                    from tailscale.com/cmd/tailscaled+
         os/user                                                      from github.com/godbus/dbus/v5+
-        path                                                         from github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds+
+        path                                                         from github.com/gliderlabs/ssh+
         path/filepath                                                from crypto/x509+
         reflect                                                      from crypto/x509+
-        regexp                                                       from github.com/aws/aws-sdk-go-v2/internal/endpoints/v2+
+        regexp                                                       from github.com/coreos/go-iptables/iptables+
         regexp/syntax                                                from regexp
         runtime/debug                                                from github.com/klauspost/compress/zstd+
         runtime/pprof                                                from net/http/pprof+

cmd/tailscaled/tailscaled.go

@@ -33,6 +33,7 @@
 	"tailscale.com/envknob"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnserver"
+	"tailscale.com/ipn/store"
 	"tailscale.com/logpolicy"
 	"tailscale.com/logtail"
 	"tailscale.com/net/dns"
@@ -394,9 +395,9 @@ func run() error {
 
 	opts := ipnServerOpts()
 
-	store, err := ipnserver.StateStore(statePathOrDefault(), logf)
+	store, err := store.New(logf, statePathOrDefault())
 	if err != nil {
-		return fmt.Errorf("ipnserver.StateStore: %w", err)
+		return fmt.Errorf("store.New: %w", err)
 	}
 	srv, err := ipnserver.New(logf, pol.PublicID.String(), store, e, dialer, nil, opts)
 	if err != nil {

cmd/tailscaled/tailscaled_windows.go

@@ -31,6 +31,7 @@
 	"inet.af/netaddr"
 	"tailscale.com/envknob"
 	"tailscale.com/ipn/ipnserver"
+	"tailscale.com/ipn/store"
 	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
 	"tailscale.com/net/tsdial"
@@ -335,8 +336,7 @@ type engineOrError struct {
 			return nil, fmt.Errorf("%w\n\nlogid: %v", res.Err, logid)
 		}
 	}
-
+	store, err := store.New(logf, statePathOrDefault())
-	store, err := ipnserver.StateStore(statePathOrDefault(), logf)
 	if err != nil {
 		return err
 	}

ipn/ipnlocal/local_test.go

@@ -14,6 +14,7 @@
 
 	"inet.af/netaddr"
 	"tailscale.com/ipn"
+	"tailscale.com/ipn/store/mem"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/tailcfg"
@@ -457,7 +458,7 @@ func TestLazyMachineKeyGeneration(t *testing.T) {
 	panicOnMachineKeyGeneration = true
 
 	var logf logger.Logf = logger.Discard
-	store := new(ipn.MemoryStore)
+	store := new(mem.Store)
 	eng, err := wgengine.NewFakeUserspaceEngine(logf, 0)
 	if err != nil {
 		t.Fatalf("NewFakeUserspaceEngine: %v", err)

ipn/ipnlocal/loglines_test.go

@@ -11,6 +11,7 @@
 
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/ipn/store/mem"
 	"tailscale.com/logtail"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
@@ -47,7 +48,7 @@ func TestLocalLogLines(t *testing.T) {
 	idA := logid(0xaa)
 
 	// set up a LocalBackend, super bare bones. No functional data.
-	store := &ipn.MemoryStore{}
+	store := new(mem.Store)
 	e, err := wgengine.NewFakeUserspaceEngine(logf, 0)
 	if err != nil {
 		t.Fatal(err)

ipn/ipnlocal/state_test.go

@@ -14,6 +14,7 @@
 
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn"
+	"tailscale.com/ipn/store/mem"
 	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/empty"
@@ -922,7 +923,7 @@ func TestStateMachine(t *testing.T) {
 }
 
 type testStateStorage struct {
-	mem     ipn.MemoryStore
+	mem     mem.Store
 	written syncs.AtomicBool
 }
 
@@ -954,7 +955,7 @@ func TestWGEngineStatusRace(t *testing.T) {
 	eng, err := wgengine.NewFakeUserspaceEngine(logf, 0)
 	c.Assert(err, qt.IsNil)
 	t.Cleanup(eng.Close)
-	b, err := NewLocalBackend(logf, "logid", new(ipn.MemoryStore), nil, eng, 0)
+	b, err := NewLocalBackend(logf, "logid", new(mem.Store), nil, eng, 0)
 	c.Assert(err, qt.IsNil)
 
 	cc := newMockControl(t)

ipn/ipnserver/server.go

@@ -36,12 +36,10 @@
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnlocal"
 	"tailscale.com/ipn/localapi"
-	"tailscale.com/ipn/store/aws"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/net/netstat"
 	"tailscale.com/net/netutil"
 	"tailscale.com/net/tsdial"
-	"tailscale.com/paths"
 	"tailscale.com/safesocket"
 	"tailscale.com/smallzstd"
 	"tailscale.com/types/logger"
@@ -657,75 +655,6 @@ func (s *Server) writeToClients(n ipn.Notify) {
 	}
 }
 
-// tryWindowsAppDataMigration attempts to copy the Windows state file
-// from its old location to the new location. (Issue 2856)
-//
-// Tailscale 1.14 and before stored state under %LocalAppData%
-// (usually "C:\WINDOWS\system32\config\systemprofile\AppData\Local"
-// when tailscaled.exe is running as a non-user system service).
-// However it is frequently cleared for almost any reason: Windows
-// updates, System Restore, even various System Cleaner utilities.
-//
-// Returns a string of the path to use for the state file.
-// This will be a fallback %LocalAppData% path if migration fails,
-// a %ProgramData% path otherwise.
-func tryWindowsAppDataMigration(logf logger.Logf, path string) string {
-	if path != paths.DefaultTailscaledStateFile() {
-		// If they're specifying a non-default path, just trust that they know
-		// what they are doing.
-		return path
-	}
-	oldFile := paths.LegacyStateFilePath()
-	return paths.TryConfigFileMigration(logf, oldFile, path)
-}
-
-// StateStore returns a StateStore from path.
-//
-// The path should be an absolute path to a file.
-//
-// Special cases:
-//
-//   * empty string means to use an in-memory store
-//   * if the string begins with "mem:", the suffix
-//     is ignored and an in-memory store is used.
-//   * if the string begins with "kube:", the suffix
-//     is a Kubernetes secret name
-//   * if the string begins with "arn:", the value is
-//     an AWS ARN for an SSM.
-func StateStore(path string, logf logger.Logf) (ipn.StateStore, error) {
-	if path == "" {
-		return &ipn.MemoryStore{}, nil
-	}
-	const memPrefix = "mem:"
-	const kubePrefix = "kube:"
-	const arnPrefix = "arn:"
-	switch {
-	case strings.HasPrefix(path, memPrefix):
-		return &ipn.MemoryStore{}, nil
-	case strings.HasPrefix(path, kubePrefix):
-		secretName := strings.TrimPrefix(path, kubePrefix)
-		store, err := ipn.NewKubeStore(secretName)
-		if err != nil {
-			return nil, fmt.Errorf("ipn.NewKubeStore(%q): %v", secretName, err)
-		}
-		return store, nil
-	case strings.HasPrefix(path, arnPrefix):
-		store, err := aws.NewStore(path)
-		if err != nil {
-			return nil, fmt.Errorf("aws.NewStore(%q): %v", path, err)
-		}
-		return store, nil
-	}
-	if runtime.GOOS == "windows" {
-		path = tryWindowsAppDataMigration(logf, path)
-	}
-	store, err := ipn.NewFileStore(path)
-	if err != nil {
-		return nil, fmt.Errorf("ipn.NewFileStore(%q): %v", path, err)
-	}
-	return store, nil
-}
-
 // Run runs a Tailscale backend service.
 // The getEngine func is called repeatedly, once per connection, until it returns an engine successfully.
 //

ipn/ipnserver/server_test.go

@@ -13,6 +13,7 @@
 
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnserver"
+	"tailscale.com/ipn/store/mem"
 	"tailscale.com/net/tsdial"
 	"tailscale.com/safesocket"
 	"tailscale.com/wgengine"
@@ -66,7 +67,7 @@ func TestRunMultipleAccepts(t *testing.T) {
 
 	opts := ipnserver.Options{}
 	t.Logf("pre-Run")
-	store := new(ipn.MemoryStore)
+	store := new(mem.Store)
 
 	ln, _, err := safesocket.Listen(socketPath, 0)
 	if err != nil {

ipn/store.go

@@ -5,21 +5,7 @@
 package ipn
 
 import (
-	"bytes"
-	"context"
-	"encoding/json"
 	"errors"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"path/filepath"
-	"sync"
-	"time"
-
-	"tailscale.com/atomicfile"
-	"tailscale.com/kube"
-	"tailscale.com/paths"
 )
 
 // ErrStateNotExist is returned by StateStore.ReadState when the
@@ -58,203 +44,3 @@ type StateStore interface {
 	// WriteState saves bs as the state associated with ID.
 	WriteState(id StateKey, bs []byte) error
 }
-
-// KubeStore is a StateStore that uses a Kubernetes Secret for persistence.
-type KubeStore struct {
-	client     *kube.Client
-	secretName string
-}
-
-// NewKubeStore returns a new KubeStore that persists to the named secret.
-func NewKubeStore(secretName string) (*KubeStore, error) {
-	c, err := kube.New()
-	if err != nil {
-		return nil, err
-	}
-	return &KubeStore{
-		client:     c,
-		secretName: secretName,
-	}, nil
-}
-
-func (s *KubeStore) String() string { return "KubeStore" }
-
-// ReadState implements the StateStore interface.
-func (s *KubeStore) ReadState(id StateKey) ([]byte, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	secret, err := s.client.GetSecret(ctx, s.secretName)
-	if err != nil {
-		if st, ok := err.(*kube.Status); ok && st.Code == 404 {
-			return nil, ErrStateNotExist
-		}
-		return nil, err
-	}
-	b, ok := secret.Data[string(id)]
-	if !ok {
-		return nil, ErrStateNotExist
-	}
-	return b, nil
-}
-
-// WriteState implements the StateStore interface.
-func (s *KubeStore) WriteState(id StateKey, bs []byte) error {
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	secret, err := s.client.GetSecret(ctx, s.secretName)
-	if err != nil {
-		if st, ok := err.(*kube.Status); ok && st.Code == 404 {
-			return s.client.CreateSecret(ctx, &kube.Secret{
-				TypeMeta: kube.TypeMeta{
-					APIVersion: "v1",
-					Kind:       "Secret",
-				},
-				ObjectMeta: kube.ObjectMeta{
-					Name: s.secretName,
-				},
-				Data: map[string][]byte{
-					string(id): bs,
-				},
-			})
-		}
-		return err
-	}
-	secret.Data[string(id)] = bs
-	if err := s.client.UpdateSecret(ctx, secret); err != nil {
-		return err
-	}
-	return err
-}
-
-// MemoryStore is a store that keeps state in memory only.
-type MemoryStore struct {
-	mu    sync.Mutex
-	cache map[StateKey][]byte
-}
-
-func (s *MemoryStore) String() string { return "MemoryStore" }
-
-// ReadState implements the StateStore interface.
-func (s *MemoryStore) ReadState(id StateKey) ([]byte, error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	bs, ok := s.cache[id]
-	if !ok {
-		return nil, ErrStateNotExist
-	}
-	return bs, nil
-}
-
-// WriteState implements the StateStore interface.
-func (s *MemoryStore) WriteState(id StateKey, bs []byte) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if s.cache == nil {
-		s.cache = map[StateKey][]byte{}
-	}
-	s.cache[id] = append([]byte(nil), bs...)
-	return nil
-}
-
-// LoadFromJSON attempts to unmarshal json content into the
-// in-memory cache.
-func (s *MemoryStore) LoadFromJSON(data []byte) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	return json.Unmarshal(data, &s.cache)
-}
-
-// ExportToJSON exports the content of the cache to
-// JSON formatted []byte.
-func (s *MemoryStore) ExportToJSON() ([]byte, error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if len(s.cache) == 0 {
-		// Avoid "null" serialization.
-		return []byte("{}"), nil
-	}
-	return json.MarshalIndent(s.cache, "", "  ")
-}
-
-// FileStore is a StateStore that uses a JSON file for persistence.
-type FileStore struct {
-	path string
-
-	mu    sync.RWMutex
-	cache map[StateKey][]byte
-}
-
-// Path returns the path that NewFileStore was called with.
-func (s *FileStore) Path() string { return s.path }
-
-func (s *FileStore) String() string { return fmt.Sprintf("FileStore(%q)", s.path) }
-
-// NewFileStore returns a new file store that persists to path.
-func NewFileStore(path string) (*FileStore, error) {
-	// We unconditionally call this to ensure that our perms are correct
-	if err := paths.MkStateDir(filepath.Dir(path)); err != nil {
-		return nil, fmt.Errorf("creating state directory: %w", err)
-	}
-
-	bs, err := ioutil.ReadFile(path)
-
-	// Treat an empty file as a missing file.
-	// (https://github.com/tailscale/tailscale/issues/895#issuecomment-723255589)
-	if err == nil && len(bs) == 0 {
-		log.Printf("ipn.NewFileStore(%q): file empty; treating it like a missing file [warning]", path)
-		err = os.ErrNotExist
-	}
-
-	if err != nil {
-		if os.IsNotExist(err) {
-			// Write out an initial file, to verify that we can write
-			// to the path.
-			if err = atomicfile.WriteFile(path, []byte("{}"), 0600); err != nil {
-				return nil, err
-			}
-			return &FileStore{
-				path:  path,
-				cache: map[StateKey][]byte{},
-			}, nil
-		}
-		return nil, err
-	}
-
-	ret := &FileStore{
-		path:  path,
-		cache: map[StateKey][]byte{},
-	}
-	if err := json.Unmarshal(bs, &ret.cache); err != nil {
-		return nil, err
-	}
-
-	return ret, nil
-}
-
-// ReadState implements the StateStore interface.
-func (s *FileStore) ReadState(id StateKey) ([]byte, error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-	bs, ok := s.cache[id]
-	if !ok {
-		return nil, ErrStateNotExist
-	}
-	return bs, nil
-}
-
-// WriteState implements the StateStore interface.
-func (s *FileStore) WriteState(id StateKey, bs []byte) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if bytes.Equal(s.cache[id], bs) {
-		return nil
-	}
-	s.cache[id] = append([]byte(nil), bs...)
-	bs, err := json.MarshalIndent(s.cache, "", "  ")
-	if err != nil {
-		return err
-	}
-	return atomicfile.WriteFile(s.path, bs, 0600)
-}

ipn/store/awsstore/store_aws.go

@@ -5,8 +5,8 @@
 //go:build linux
 // +build linux
 
-// Package aws contains an AWS SSM StateStore implementation.
+// Package awsstore contains an ipn.StateStore implementation using AWS SSM.
-package aws
+package awsstore
 
 import (
 	"context"
@@ -20,6 +20,8 @@
 	"github.com/aws/aws-sdk-go-v2/service/ssm"
 	ssmTypes "github.com/aws/aws-sdk-go-v2/service/ssm/types"
 	"tailscale.com/ipn"
+	"tailscale.com/ipn/store/mem"
+	"tailscale.com/types/logger"
 )
 
 const (
@@ -46,12 +48,12 @@ type awsStore struct {
 	ssmClient awsSSMClient
 	ssmARN    arn.ARN
 
-	memory ipn.MemoryStore
+	memory mem.Store
 }
 
-// NewStore returns a new ipn.StateStore using the AWS SSM storage
+// New returns a new ipn.StateStore using the AWS SSM storage
 // location given by ssmARN.
-func NewStore(ssmARN string) (ipn.StateStore, error) {
+func New(_ logger.Logf, ssmARN string) (ipn.StateStore, error) {
 	return newStore(ssmARN, nil)
 }
 

ipn/store/awsstore/store_aws_stub.go

@@ -5,7 +5,7 @@
 //go:build !linux
 // +build !linux
 
-package aws
+package awsstore
 
 import (
 	"fmt"

ipn/store/awsstore/store_aws_test.go

@@ -5,7 +5,7 @@
 //go:build linux
 // +build linux
 
-package aws
+package awsstore
 
 import (
 	"context"

ipn/store/kubestore/store_kube.go

@@ -0,0 +1,85 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package kubestore contains an ipn.StateStore implementation using Kubernetes Secrets.
+
+package kubestore
+
+import (
+	"context"
+	"time"
+
+	"tailscale.com/ipn"
+	"tailscale.com/kube"
+	"tailscale.com/types/logger"
+)
+
+// Store is an ipn.StateStore that uses a Kubernetes Secret for persistence.
+type Store struct {
+	client     *kube.Client
+	secretName string
+}
+
+// New returns a new Store that persists to the named secret.
+func New(_ logger.Logf, secretName string) (*Store, error) {
+	c, err := kube.New()
+	if err != nil {
+		return nil, err
+	}
+	return &Store{
+		client:     c,
+		secretName: secretName,
+	}, nil
+}
+
+func (s *Store) String() string { return "kube.Store" }
+
+// ReadState implements the StateStore interface.
+func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	secret, err := s.client.GetSecret(ctx, s.secretName)
+	if err != nil {
+		if st, ok := err.(*kube.Status); ok && st.Code == 404 {
+			return nil, ipn.ErrStateNotExist
+		}
+		return nil, err
+	}
+	b, ok := secret.Data[string(id)]
+	if !ok {
+		return nil, ipn.ErrStateNotExist
+	}
+	return b, nil
+}
+
+// WriteState implements the StateStore interface.
+func (s *Store) WriteState(id ipn.StateKey, bs []byte) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	secret, err := s.client.GetSecret(ctx, s.secretName)
+	if err != nil {
+		if st, ok := err.(*kube.Status); ok && st.Code == 404 {
+			return s.client.CreateSecret(ctx, &kube.Secret{
+				TypeMeta: kube.TypeMeta{
+					APIVersion: "v1",
+					Kind:       "Secret",
+				},
+				ObjectMeta: kube.ObjectMeta{
+					Name: s.secretName,
+				},
+				Data: map[string][]byte{
+					string(id): bs,
+				},
+			})
+		}
+		return err
+	}
+	secret.Data[string(id)] = bs
+	if err := s.client.UpdateSecret(ctx, secret); err != nil {
+		return err
+	}
+	return err
+}

ipn/store/mem/store_mem.go

@@ -0,0 +1,69 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package mem provides an in-memory ipn.StateStore implementation.
+package mem
+
+import (
+	"encoding/json"
+	"sync"
+
+	"tailscale.com/ipn"
+	"tailscale.com/types/logger"
+)
+
+// New returns a new Store.
+func New(logger.Logf, string) (ipn.StateStore, error) {
+	return new(Store), nil
+}
+
+// Store is an ipn.StateStore that keeps state in memory only.
+type Store struct {
+	mu    sync.Mutex
+	cache map[ipn.StateKey][]byte
+}
+
+func (s *Store) String() string { return "mem.Store" }
+
+// ReadState implements the StateStore interface.
+func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	bs, ok := s.cache[id]
+	if !ok {
+		return nil, ipn.ErrStateNotExist
+	}
+	return bs, nil
+}
+
+// WriteState implements the StateStore interface.
+func (s *Store) WriteState(id ipn.StateKey, bs []byte) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.cache == nil {
+		s.cache = map[ipn.StateKey][]byte{}
+	}
+	s.cache[id] = append([]byte(nil), bs...)
+	return nil
+}
+
+// LoadFromJSON attempts to unmarshal json content into the
+// in-memory cache.
+func (s *Store) LoadFromJSON(data []byte) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return json.Unmarshal(data, &s.cache)
+}
+
+// ExportToJSON exports the content of the cache to
+// JSON formatted []byte.
+func (s *Store) ExportToJSON() ([]byte, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if len(s.cache) == 0 {
+		// Avoid "null" serialization.
+		return []byte("{}"), nil
+	}
+	return json.MarshalIndent(s.cache, "", "  ")
+}

ipn/store/stores.go

@@ -0,0 +1,193 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package store provides various implementation of ipn.StateStore.
+package store
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+
+	"tailscale.com/atomicfile"
+	"tailscale.com/ipn"
+	"tailscale.com/ipn/store/mem"
+	"tailscale.com/paths"
+	"tailscale.com/types/logger"
+)
+
+// Provider returns a StateStore for the provided path.
+// The arg is of the form "prefix:rest", where prefix was previously registered with Register.
+type Provider func(logf logger.Logf, arg string) (ipn.StateStore, error)
+
+var regOnce sync.Once
+
+var registerAvailableExternalStores func()
+
+func registerDefaultStores() {
+	Register("mem:", mem.New)
+
+	if registerAvailableExternalStores != nil {
+		registerAvailableExternalStores()
+	}
+}
+
+var knownStores map[string]Provider
+
+// New returns a StateStore based on the provided arg
+// and registered stores.
+// The arg is of the form "prefix:rest", where prefix was previously
+// registered with Register.
+//
+// By default the following stores are registered:
+//
+//   * if the string begins with "mem:", the suffix
+//     is ignored and an in-memory store is used.
+//   * (Linux-only) if the string begins with "arn:",
+//     the suffix an AWS ARN for an SSM.
+//   * (Linux-only) if the string begins with "kube:",
+//     the suffix is a Kubernetes secret name
+//   * In all other cases, the path is treated as a filepath.
+func New(logf logger.Logf, path string) (ipn.StateStore, error) {
+	regOnce.Do(registerDefaultStores)
+	for prefix, sf := range knownStores {
+		if strings.HasPrefix(path, prefix) {
+			// We can't strip the prefix here as some NewStoreFunc (like arn:)
+			// expect the prefix.
+			return sf(logf, path)
+		}
+	}
+	if runtime.GOOS == "windows" {
+		path = TryWindowsAppDataMigration(logf, path)
+	}
+	return NewFileStore(logf, path)
+}
+
+// Register registers a prefix to be used for
+// NewStore. It panics if the prefix is empty, or if the
+// prefix is already registered.
+// The provided fn is called with the path passed to NewStore;
+// the prefix is not stripped.
+func Register(prefix string, fn Provider) {
+	if len(prefix) == 0 {
+		panic("prefix is empty")
+	}
+	if _, ok := knownStores[prefix]; ok {
+		panic(fmt.Sprintf("%q already registered", prefix))
+	}
+	if knownStores == nil {
+		knownStores = make(map[string]Provider)
+	}
+	knownStores[prefix] = fn
+}
+
+// TryWindowsAppDataMigration attempts to copy the Windows state file
+// from its old location to the new location. (Issue 2856)
+//
+// Tailscale 1.14 and before stored state under %LocalAppData%
+// (usually "C:\WINDOWS\system32\config\systemprofile\AppData\Local"
+// when tailscaled.exe is running as a non-user system service).
+// However it is frequently cleared for almost any reason: Windows
+// updates, System Restore, even various System Cleaner utilities.
+//
+// Returns a string of the path to use for the state file.
+// This will be a fallback %LocalAppData% path if migration fails,
+// a %ProgramData% path otherwise.
+func TryWindowsAppDataMigration(logf logger.Logf, path string) string {
+	if path != paths.DefaultTailscaledStateFile() {
+		// If they're specifying a non-default path, just trust that they know
+		// what they are doing.
+		return path
+	}
+	oldFile := paths.LegacyStateFilePath()
+	return paths.TryConfigFileMigration(logf, oldFile, path)
+}
+
+// FileStore is a StateStore that uses a JSON file for persistence.
+type FileStore struct {
+	path string
+
+	mu    sync.RWMutex
+	cache map[ipn.StateKey][]byte
+}
+
+// Path returns the path that NewFileStore was called with.
+func (s *FileStore) Path() string { return s.path }
+
+func (s *FileStore) String() string { return fmt.Sprintf("FileStore(%q)", s.path) }
+
+// NewFileStore returns a new file store that persists to path.
+func NewFileStore(_ logger.Logf, path string) (ipn.StateStore, error) {
+	// We unconditionally call this to ensure that our perms are correct
+	if err := paths.MkStateDir(filepath.Dir(path)); err != nil {
+		return nil, fmt.Errorf("creating state directory: %w", err)
+	}
+
+	bs, err := ioutil.ReadFile(path)
+
+	// Treat an empty file as a missing file.
+	// (https://github.com/tailscale/tailscale/issues/895#issuecomment-723255589)
+	if err == nil && len(bs) == 0 {
+		log.Printf("ipn.NewFileStore(%q): file empty; treating it like a missing file [warning]", path)
+		err = os.ErrNotExist
+	}
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Write out an initial file, to verify that we can write
+			// to the path.
+			if err = atomicfile.WriteFile(path, []byte("{}"), 0600); err != nil {
+				return nil, err
+			}
+			return &FileStore{
+				path:  path,
+				cache: map[ipn.StateKey][]byte{},
+			}, nil
+		}
+		return nil, err
+	}
+
+	ret := &FileStore{
+		path:  path,
+		cache: map[ipn.StateKey][]byte{},
+	}
+	if err := json.Unmarshal(bs, &ret.cache); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+// ReadState implements the StateStore interface.
+func (s *FileStore) ReadState(id ipn.StateKey) ([]byte, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	bs, ok := s.cache[id]
+	if !ok {
+		return nil, ipn.ErrStateNotExist
+	}
+	return bs, nil
+}
+
+// WriteState implements the StateStore interface.
+func (s *FileStore) WriteState(id ipn.StateKey, bs []byte) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if bytes.Equal(s.cache[id], bs) {
+		return nil
+	}
+	s.cache[id] = append([]byte(nil), bs...)
+	bs, err := json.MarshalIndent(s.cache, "", "  ")
+	if err != nil {
+		return err
+	}
+	return atomicfile.WriteFile(s.path, bs, 0600)
+}

ipn/store/stores_linux.go

@@ -0,0 +1,26 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package store
+
+import (
+	"strings"
+
+	"tailscale.com/ipn"
+	"tailscale.com/ipn/store/awsstore"
+	"tailscale.com/ipn/store/kubestore"
+	"tailscale.com/types/logger"
+)
+
+func init() {
+	registerAvailableExternalStores = registerExternalStores
+}
+
+func registerExternalStores() {
+	Register("kube:", func(logf logger.Logf, path string) (ipn.StateStore, error) {
+		secretName := strings.TrimPrefix(path, "kube:")
+		return kubestore.New(logf, secretName)
+	})
+	Register("arn:", awsstore.New)
+}

ipn/store/stores_test.go

@@ -0,0 +1,191 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package store
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"tailscale.com/ipn"
+	"tailscale.com/ipn/store/mem"
+	"tailscale.com/tstest"
+	"tailscale.com/types/logger"
+)
+
+func TestNewStore(t *testing.T) {
+	regOnce.Do(registerDefaultStores)
+	t.Cleanup(func() {
+		knownStores = map[string]Provider{}
+		registerDefaultStores()
+	})
+	knownStores = map[string]Provider{}
+
+	type store1 struct {
+		ipn.StateStore
+		path string
+	}
+
+	type store2 struct {
+		ipn.StateStore
+		path string
+	}
+
+	Register("arn:", func(_ logger.Logf, path string) (ipn.StateStore, error) {
+		return &store1{new(mem.Store), path}, nil
+	})
+	Register("kube:", func(_ logger.Logf, path string) (ipn.StateStore, error) {
+		return &store2{new(mem.Store), path}, nil
+	})
+	Register("mem:", func(_ logger.Logf, path string) (ipn.StateStore, error) {
+		return new(mem.Store), nil
+	})
+
+	path := "mem:abcd"
+	if s, err := New(t.Logf, path); err != nil {
+		t.Fatalf("%q: %v", path, err)
+	} else if _, ok := s.(*mem.Store); !ok {
+		t.Fatalf("%q: got: %T, want: %T", path, s, new(mem.Store))
+	}
+
+	path = "arn:foo"
+	if s, err := New(t.Logf, path); err != nil {
+		t.Fatalf("%q: %v", path, err)
+	} else if _, ok := s.(*store1); !ok {
+		t.Fatalf("%q: got: %T, want: %T", path, s, new(store1))
+	}
+
+	path = "kube:abcd"
+	if s, err := New(t.Logf, path); err != nil {
+		t.Fatalf("%q: %v", path, err)
+	} else if _, ok := s.(*store2); !ok {
+		t.Fatalf("%q: got: %T, want: %T", path, s, new(store2))
+	}
+	f, err := os.CreateTemp("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		os.Remove(f.Name())
+	})
+
+	path = f.Name()
+	if s, err := New(t.Logf, path); err != nil {
+		t.Fatalf("%q: %v", path, err)
+	} else if _, ok := s.(*FileStore); !ok {
+		t.Fatalf("%q: got: %T, want: %T", path, s, new(FileStore))
+	}
+}
+
+func testStoreSemantics(t *testing.T, store ipn.StateStore) {
+	t.Helper()
+
+	tests := []struct {
+		// if true, data is data to write. If false, data is expected
+		// output of read.
+		write bool
+		id    ipn.StateKey
+		data  string
+		// If write=false, true if we expect a not-exist error.
+		notExists bool
+	}{
+		{
+			id:        "foo",
+			notExists: true,
+		},
+		{
+			write: true,
+			id:    "foo",
+			data:  "bar",
+		},
+		{
+			id:   "foo",
+			data: "bar",
+		},
+		{
+			id:        "baz",
+			notExists: true,
+		},
+		{
+			write: true,
+			id:    "baz",
+			data:  "quux",
+		},
+		{
+			id:   "foo",
+			data: "bar",
+		},
+		{
+			id:   "baz",
+			data: "quux",
+		},
+	}
+
+	for _, test := range tests {
+		if test.write {
+			if err := store.WriteState(test.id, []byte(test.data)); err != nil {
+				t.Errorf("writing %q to %q: %v", test.data, test.id, err)
+			}
+		} else {
+			bs, err := store.ReadState(test.id)
+			if err != nil {
+				if test.notExists && err == ipn.ErrStateNotExist {
+					continue
+				}
+				t.Errorf("reading %q: %v", test.id, err)
+				continue
+			}
+			if string(bs) != test.data {
+				t.Errorf("reading %q: got %q, want %q", test.id, string(bs), test.data)
+			}
+		}
+	}
+}
+
+func TestMemoryStore(t *testing.T) {
+	tstest.PanicOnLog()
+
+	store := new(mem.Store)
+	testStoreSemantics(t, store)
+}
+
+func TestFileStore(t *testing.T) {
+	tstest.PanicOnLog()
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test-file-store.conf")
+
+	store, err := NewFileStore(nil, path)
+	if err != nil {
+		t.Fatalf("creating file store failed: %v", err)
+	}
+
+	testStoreSemantics(t, store)
+
+	// Build a brand new file store and check that both IDs written
+	// above are still there.
+	store, err = NewFileStore(nil, path)
+	if err != nil {
+		t.Fatalf("creating second file store failed: %v", err)
+	}
+
+	expected := map[ipn.StateKey]string{
+		"foo": "bar",
+		"baz": "quux",
+	}
+	for key, want := range expected {
+		bs, err := store.ReadState(key)
+		if err != nil {
+			t.Errorf("reading %q (2nd store): %v", key, err)
+			continue
+		}
+		if string(bs) != want {
+			t.Errorf("reading %q (2nd store): got %q, want %q", key, bs, want)
+		}
+	}
+}

ipn/store_test.go

@@ -1,120 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package ipn
-
-import (
-	"path/filepath"
-	"testing"
-
-	"tailscale.com/tstest"
-)
-
-func testStoreSemantics(t *testing.T, store StateStore) {
-	t.Helper()
-
-	tests := []struct {
-		// if true, data is data to write. If false, data is expected
-		// output of read.
-		write bool
-		id    StateKey
-		data  string
-		// If write=false, true if we expect a not-exist error.
-		notExists bool
-	}{
-		{
-			id:        "foo",
-			notExists: true,
-		},
-		{
-			write: true,
-			id:    "foo",
-			data:  "bar",
-		},
-		{
-			id:   "foo",
-			data: "bar",
-		},
-		{
-			id:        "baz",
-			notExists: true,
-		},
-		{
-			write: true,
-			id:    "baz",
-			data:  "quux",
-		},
-		{
-			id:   "foo",
-			data: "bar",
-		},
-		{
-			id:   "baz",
-			data: "quux",
-		},
-	}
-
-	for _, test := range tests {
-		if test.write {
-			if err := store.WriteState(test.id, []byte(test.data)); err != nil {
-				t.Errorf("writing %q to %q: %v", test.data, test.id, err)
-			}
-		} else {
-			bs, err := store.ReadState(test.id)
-			if err != nil {
-				if test.notExists && err == ErrStateNotExist {
-					continue
-				}
-				t.Errorf("reading %q: %v", test.id, err)
-				continue
-			}
-			if string(bs) != test.data {
-				t.Errorf("reading %q: got %q, want %q", test.id, string(bs), test.data)
-			}
-		}
-	}
-}
-
-func TestMemoryStore(t *testing.T) {
-	tstest.PanicOnLog()
-
-	store := &MemoryStore{}
-	testStoreSemantics(t, store)
-}
-
-func TestFileStore(t *testing.T) {
-	tstest.PanicOnLog()
-
-	dir := t.TempDir()
-	path := filepath.Join(dir, "test-file-store.conf")
-
-	store, err := NewFileStore(path)
-	if err != nil {
-		t.Fatalf("creating file store failed: %v", err)
-	}
-
-	testStoreSemantics(t, store)
-
-	// Build a brand new file store and check that both IDs written
-	// above are still there.
-	store, err = NewFileStore(path)
-	if err != nil {
-		t.Fatalf("creating second file store failed: %v", err)
-	}
-
-	expected := map[StateKey]string{
-		"foo": "bar",
-		"baz": "quux",
-	}
-	for key, want := range expected {
-		bs, err := store.ReadState(key)
-		if err != nil {
-			t.Errorf("reading %q (2nd store): %v", key, err)
-			continue
-		}
-		if string(bs) != want {
-			t.Errorf("reading %q (2nd store): got %q, want %q", key, bs, want)
-		}
-	}
-}

ssh/tailssh/tailssh_test.go

@@ -22,8 +22,8 @@
 
 	"github.com/gliderlabs/ssh"
 	"inet.af/netaddr"
-	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnlocal"
+	"tailscale.com/ipn/store/mem"
 	"tailscale.com/net/tsdial"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
@@ -181,7 +181,7 @@ func TestSSH(t *testing.T) {
 		t.Fatal(err)
 	}
 	lb, err := ipnlocal.NewLocalBackend(logf, "",
-		new(ipn.MemoryStore),
+		new(mem.Store),
 		new(tsdial.Dialer),
 		eng, 0)
 	if err != nil {

tsnet/tsnet.go

@@ -27,6 +27,8 @@
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnlocal"
 	"tailscale.com/ipn/localapi"
+	"tailscale.com/ipn/store"
+	"tailscale.com/ipn/store/mem"
 	"tailscale.com/net/nettest"
 	"tailscale.com/net/tsdial"
 	"tailscale.com/smallzstd"
@@ -46,6 +48,12 @@ type Server struct {
 	// based on the name of the binary.
 	Dir string
 
+	// Store specifies the state store to use.
+	//
+	// If nil, a new FileStore is initialized at `Dir/tailscaled.state`.
+	// See tailscale.com/ipn/store for supported stores.
+	Store ipn.StateStore
+
 	// Hostname is the hostname to present to the control server.
 	// If empty, the binary name is used.
 	Hostname string
@@ -62,7 +70,7 @@ type Server struct {
 	initErr  error
 	lb       *ipnlocal.LocalBackend
 	// the state directory
-	dir      string
+	rootPath string
 	hostname string
 
 	mu        sync.Mutex
@@ -79,12 +87,6 @@ func (s *Server) Dial(ctx context.Context, network, address string) (net.Conn, e
 	return s.dialer.UserDial(ctx, network, address)
 }
 
-func (s *Server) doInit() {
-	if err := s.start(); err != nil {
-		s.initErr = fmt.Errorf("tsnet: %w", err)
-	}
-}
-
 // Start connects the server to the tailnet.
 // Optional: any calls to Dial/Listen will also call Start.
 func (s *Server) Start() error {
@@ -92,6 +94,12 @@ func (s *Server) Start() error {
 	return s.initErr
 }
 
+func (s *Server) doInit() {
+	if err := s.start(); err != nil {
+		s.initErr = fmt.Errorf("tsnet: %w", err)
+	}
+}
+
 func (s *Server) start() error {
 	if !envknob.UseWIPCode() {
 		return errors.New("code disabled without environment variable TAILSCALE_USE_WIP_CODE set true")
@@ -108,21 +116,26 @@ func (s *Server) start() error {
 		s.hostname = prog
 	}
 
-	s.dir = s.Dir
+	s.rootPath = s.Dir
-	if s.dir == "" {
+	if s.Store != nil && !s.Ephemeral {
+		if _, ok := s.Store.(*mem.Store); !ok {
+			return fmt.Errorf("in-memory store is only supported for Ephemeral nodes")
+		}
+	}
+	if s.rootPath == "" {
 		confDir, err := os.UserConfigDir()
 		if err != nil {
 			return err
 		}
-		s.dir = filepath.Join(confDir, "tslib-"+prog)
+		s.rootPath = filepath.Join(confDir, "tslib-"+prog)
-		if err := os.MkdirAll(s.dir, 0700); err != nil {
+		if err := os.MkdirAll(s.rootPath, 0700); err != nil {
 			return err
 		}
 	}
-	if fi, err := os.Stat(s.dir); err != nil {
+	if fi, err := os.Stat(s.rootPath); err != nil {
 		return err
 	} else if !fi.IsDir() {
-		return fmt.Errorf("%v is not a directory", s.dir)
+		return fmt.Errorf("%v is not a directory", s.rootPath)
 	}
 
 	logf := s.Logf
@@ -170,10 +183,11 @@ func (s *Server) start() error {
 		return ns.DialContextTCP(ctx, dst)
 	}
 
-	statePath := filepath.Join(s.dir, "tailscaled.state")
+	if s.Store == nil {
-	store, err := ipn.NewFileStore(statePath)
+		s.Store, err = store.New(logf, filepath.Join(s.rootPath, "tailscaled.state"))
-	if err != nil {
+		if err != nil {
-		return err
+			return err
+		}
 	}
 	logid := "tslib-TODO"
 
@@ -181,11 +195,11 @@ func (s *Server) start() error {
 	if s.Ephemeral {
 		loginFlags = controlclient.LoginEphemeral
 	}
-	lb, err := ipnlocal.NewLocalBackend(logf, logid, store, s.dialer, eng, loginFlags)
+	lb, err := ipnlocal.NewLocalBackend(logf, logid, s.Store, s.dialer, eng, loginFlags)
 	if err != nil {
 		return fmt.Errorf("NewLocalBackend: %v", err)
 	}
-	lb.SetVarRoot(s.dir)
+	lb.SetVarRoot(s.rootPath)
 	s.lb = lb
 	lb.SetDecompressor(func() (controlclient.Decompressor, error) {
 		return smallzstd.NewDecoder(nil)

tstest/integration/integration_test.go

@@ -33,6 +33,7 @@
 	"inet.af/netaddr"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/ipn/store"
 	"tailscale.com/safesocket"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
@@ -586,7 +587,7 @@ func (n *testNode) diskPrefs() *ipn.Prefs {
 	if _, err := ioutil.ReadFile(n.stateFile); err != nil {
 		t.Fatalf("reading prefs: %v", err)
 	}
-	fs, err := ipn.NewFileStore(n.stateFile)
+	fs, err := store.NewFileStore(nil, n.stateFile)
 	if err != nil {
 		t.Fatalf("reading prefs, NewFileStore: %v", err)
 	}

tstest/integration/tailscaled_deps_test_darwin.go

@@ -19,6 +19,7 @@
 	_ "tailscale.com/envknob"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/ipn/store"
 	_ "tailscale.com/logpolicy"
 	_ "tailscale.com/logtail"
 	_ "tailscale.com/net/dns"

tstest/integration/tailscaled_deps_test_freebsd.go

@@ -19,6 +19,7 @@
 	_ "tailscale.com/envknob"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/ipn/store"
 	_ "tailscale.com/logpolicy"
 	_ "tailscale.com/logtail"
 	_ "tailscale.com/net/dns"

tstest/integration/tailscaled_deps_test_linux.go

@@ -19,6 +19,7 @@
 	_ "tailscale.com/envknob"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/ipn/store"
 	_ "tailscale.com/logpolicy"
 	_ "tailscale.com/logtail"
 	_ "tailscale.com/net/dns"

tstest/integration/tailscaled_deps_test_openbsd.go

@@ -19,6 +19,7 @@
 	_ "tailscale.com/envknob"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/ipn/store"
 	_ "tailscale.com/logpolicy"
 	_ "tailscale.com/logtail"
 	_ "tailscale.com/net/dns"

tstest/integration/tailscaled_deps_test_windows.go

@@ -22,6 +22,7 @@
 	_ "tailscale.com/envknob"
 	_ "tailscale.com/ipn"
 	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/ipn/store"
 	_ "tailscale.com/logpolicy"
 	_ "tailscale.com/logtail"
 	_ "tailscale.com/logtail/backoff"