ipn/store: add common package for instantiating ipn.StateStores

Also move KubeStore and MemStore into their own package.

RELNOTE: tsnet now supports providing a custom ipn.StateStore.

Signed-off-by: Maisem Ali <maisem@tailscale.com>

ipn/ipnlocal/local_test.go

@@ -14,6 +14,7 @@ import (
 
 	"inet.af/netaddr"
 	"tailscale.com/ipn"
+	"tailscale.com/ipn/store/mem"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/tailcfg"
@@ -457,7 +458,7 @@ func TestLazyMachineKeyGeneration(t *testing.T) {
 	panicOnMachineKeyGeneration = true
 
 	var logf logger.Logf = logger.Discard
-	store := new(ipn.MemoryStore)
+	store := new(mem.Store)
 	eng, err := wgengine.NewFakeUserspaceEngine(logf, 0)
 	if err != nil {
 		t.Fatalf("NewFakeUserspaceEngine: %v", err)

ipn/ipnlocal/loglines_test.go

@@ -11,6 +11,7 @@ import (
 
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/ipn/store/mem"
 	"tailscale.com/logtail"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
@@ -47,7 +48,7 @@ func TestLocalLogLines(t *testing.T) {
 	idA := logid(0xaa)
 
 	// set up a LocalBackend, super bare bones. No functional data.
-	store := &ipn.MemoryStore{}
+	store := new(mem.Store)
 	e, err := wgengine.NewFakeUserspaceEngine(logf, 0)
 	if err != nil {
 		t.Fatal(err)

ipn/ipnlocal/state_test.go

@@ -14,6 +14,7 @@ import (
 
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn"
+	"tailscale.com/ipn/store/mem"
 	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/empty"
@@ -922,7 +923,7 @@ func TestStateMachine(t *testing.T) {
 }
 
 type testStateStorage struct {
-	mem     ipn.MemoryStore
+	mem     mem.Store
 	written syncs.AtomicBool
 }
 
@@ -954,7 +955,7 @@ func TestWGEngineStatusRace(t *testing.T) {
 	eng, err := wgengine.NewFakeUserspaceEngine(logf, 0)
 	c.Assert(err, qt.IsNil)
 	t.Cleanup(eng.Close)
-	b, err := NewLocalBackend(logf, "logid", new(ipn.MemoryStore), nil, eng, 0)
+	b, err := NewLocalBackend(logf, "logid", new(mem.Store), nil, eng, 0)
 	c.Assert(err, qt.IsNil)
 
 	cc := newMockControl(t)

ipn/ipnserver/server.go

@@ -36,12 +36,10 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnlocal"
 	"tailscale.com/ipn/localapi"
-	"tailscale.com/ipn/store/aws"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/net/netstat"
 	"tailscale.com/net/netutil"
 	"tailscale.com/net/tsdial"
-	"tailscale.com/paths"
 	"tailscale.com/safesocket"
 	"tailscale.com/smallzstd"
 	"tailscale.com/types/logger"
@@ -657,75 +655,6 @@ func (s *Server) writeToClients(n ipn.Notify) {
 	}
 }
 
-// tryWindowsAppDataMigration attempts to copy the Windows state file
-// from its old location to the new location. (Issue 2856)
-//
-// Tailscale 1.14 and before stored state under %LocalAppData%
-// (usually "C:\WINDOWS\system32\config\systemprofile\AppData\Local"
-// when tailscaled.exe is running as a non-user system service).
-// However it is frequently cleared for almost any reason: Windows
-// updates, System Restore, even various System Cleaner utilities.
-//
-// Returns a string of the path to use for the state file.
-// This will be a fallback %LocalAppData% path if migration fails,
-// a %ProgramData% path otherwise.
-func tryWindowsAppDataMigration(logf logger.Logf, path string) string {
-	if path != paths.DefaultTailscaledStateFile() {
-		// If they're specifying a non-default path, just trust that they know
-		// what they are doing.
-		return path
-	}
-	oldFile := paths.LegacyStateFilePath()
-	return paths.TryConfigFileMigration(logf, oldFile, path)
-}
-
-// StateStore returns a StateStore from path.
-//
-// The path should be an absolute path to a file.
-//
-// Special cases:
-//
-//   * empty string means to use an in-memory store
-//   * if the string begins with "mem:", the suffix
-//     is ignored and an in-memory store is used.
-//   * if the string begins with "kube:", the suffix
-//     is a Kubernetes secret name
-//   * if the string begins with "arn:", the value is
-//     an AWS ARN for an SSM.
-func StateStore(path string, logf logger.Logf) (ipn.StateStore, error) {
-	if path == "" {
-		return &ipn.MemoryStore{}, nil
-	}
-	const memPrefix = "mem:"
-	const kubePrefix = "kube:"
-	const arnPrefix = "arn:"
-	switch {
-	case strings.HasPrefix(path, memPrefix):
-		return &ipn.MemoryStore{}, nil
-	case strings.HasPrefix(path, kubePrefix):
-		secretName := strings.TrimPrefix(path, kubePrefix)
-		store, err := ipn.NewKubeStore(secretName)
-		if err != nil {
-			return nil, fmt.Errorf("ipn.NewKubeStore(%q): %v", secretName, err)
-		}
-		return store, nil
-	case strings.HasPrefix(path, arnPrefix):
-		store, err := aws.NewStore(path)
-		if err != nil {
-			return nil, fmt.Errorf("aws.NewStore(%q): %v", path, err)
-		}
-		return store, nil
-	}
-	if runtime.GOOS == "windows" {
-		path = tryWindowsAppDataMigration(logf, path)
-	}
-	store, err := ipn.NewFileStore(path)
-	if err != nil {
-		return nil, fmt.Errorf("ipn.NewFileStore(%q): %v", path, err)
-	}
-	return store, nil
-}
-
 // Run runs a Tailscale backend service.
 // The getEngine func is called repeatedly, once per connection, until it returns an engine successfully.
 //

ipn/ipnserver/server_test.go

@@ -13,6 +13,7 @@ import (
 
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnserver"
+	"tailscale.com/ipn/store/mem"
 	"tailscale.com/net/tsdial"
 	"tailscale.com/safesocket"
 	"tailscale.com/wgengine"
@@ -66,7 +67,7 @@ func TestRunMultipleAccepts(t *testing.T) {
 
 	opts := ipnserver.Options{}
 	t.Logf("pre-Run")
-	store := new(ipn.MemoryStore)
+	store := new(mem.Store)
 
 	ln, _, err := safesocket.Listen(socketPath, 0)
 	if err != nil {

ipn/store.go

@@ -5,21 +5,7 @@
 package ipn
 
 import (
-	"bytes"
-	"context"
-	"encoding/json"
 	"errors"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"path/filepath"
-	"sync"
-	"time"
-
-	"tailscale.com/atomicfile"
-	"tailscale.com/kube"
-	"tailscale.com/paths"
 )
 
 // ErrStateNotExist is returned by StateStore.ReadState when the
@@ -58,203 +44,3 @@ type StateStore interface {
 	// WriteState saves bs as the state associated with ID.
 	WriteState(id StateKey, bs []byte) error
 }
-
-// KubeStore is a StateStore that uses a Kubernetes Secret for persistence.
-type KubeStore struct {
-	client     *kube.Client
-	secretName string
-}
-
-// NewKubeStore returns a new KubeStore that persists to the named secret.
-func NewKubeStore(secretName string) (*KubeStore, error) {
-	c, err := kube.New()
-	if err != nil {
-		return nil, err
-	}
-	return &KubeStore{
-		client:     c,
-		secretName: secretName,
-	}, nil
-}
-
-func (s *KubeStore) String() string { return "KubeStore" }
-
-// ReadState implements the StateStore interface.
-func (s *KubeStore) ReadState(id StateKey) ([]byte, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	secret, err := s.client.GetSecret(ctx, s.secretName)
-	if err != nil {
-		if st, ok := err.(*kube.Status); ok && st.Code == 404 {
-			return nil, ErrStateNotExist
-		}
-		return nil, err
-	}
-	b, ok := secret.Data[string(id)]
-	if !ok {
-		return nil, ErrStateNotExist
-	}
-	return b, nil
-}
-
-// WriteState implements the StateStore interface.
-func (s *KubeStore) WriteState(id StateKey, bs []byte) error {
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	secret, err := s.client.GetSecret(ctx, s.secretName)
-	if err != nil {
-		if st, ok := err.(*kube.Status); ok && st.Code == 404 {
-			return s.client.CreateSecret(ctx, &kube.Secret{
-				TypeMeta: kube.TypeMeta{
-					APIVersion: "v1",
-					Kind:       "Secret",
-				},
-				ObjectMeta: kube.ObjectMeta{
-					Name: s.secretName,
-				},
-				Data: map[string][]byte{
-					string(id): bs,
-				},
-			})
-		}
-		return err
-	}
-	secret.Data[string(id)] = bs
-	if err := s.client.UpdateSecret(ctx, secret); err != nil {
-		return err
-	}
-	return err
-}
-
-// MemoryStore is a store that keeps state in memory only.
-type MemoryStore struct {
-	mu    sync.Mutex
-	cache map[StateKey][]byte
-}
-
-func (s *MemoryStore) String() string { return "MemoryStore" }
-
-// ReadState implements the StateStore interface.
-func (s *MemoryStore) ReadState(id StateKey) ([]byte, error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	bs, ok := s.cache[id]
-	if !ok {
-		return nil, ErrStateNotExist
-	}
-	return bs, nil
-}
-
-// WriteState implements the StateStore interface.
-func (s *MemoryStore) WriteState(id StateKey, bs []byte) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if s.cache == nil {
-		s.cache = map[StateKey][]byte{}
-	}
-	s.cache[id] = append([]byte(nil), bs...)
-	return nil
-}
-
-// LoadFromJSON attempts to unmarshal json content into the
-// in-memory cache.
-func (s *MemoryStore) LoadFromJSON(data []byte) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	return json.Unmarshal(data, &s.cache)
-}
-
-// ExportToJSON exports the content of the cache to
-// JSON formatted []byte.
-func (s *MemoryStore) ExportToJSON() ([]byte, error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if len(s.cache) == 0 {
-		// Avoid "null" serialization.
-		return []byte("{}"), nil
-	}
-	return json.MarshalIndent(s.cache, "", "  ")
-}
-
-// FileStore is a StateStore that uses a JSON file for persistence.
-type FileStore struct {
-	path string
-
-	mu    sync.RWMutex
-	cache map[StateKey][]byte
-}
-
-// Path returns the path that NewFileStore was called with.
-func (s *FileStore) Path() string { return s.path }
-
-func (s *FileStore) String() string { return fmt.Sprintf("FileStore(%q)", s.path) }
-
-// NewFileStore returns a new file store that persists to path.
-func NewFileStore(path string) (*FileStore, error) {
-	// We unconditionally call this to ensure that our perms are correct
-	if err := paths.MkStateDir(filepath.Dir(path)); err != nil {
-		return nil, fmt.Errorf("creating state directory: %w", err)
-	}
-
-	bs, err := ioutil.ReadFile(path)
-
-	// Treat an empty file as a missing file.
-	// (https://github.com/tailscale/tailscale/issues/895#issuecomment-723255589)
-	if err == nil && len(bs) == 0 {
-		log.Printf("ipn.NewFileStore(%q): file empty; treating it like a missing file [warning]", path)
-		err = os.ErrNotExist
-	}
-
-	if err != nil {
-		if os.IsNotExist(err) {
-			// Write out an initial file, to verify that we can write
-			// to the path.
-			if err = atomicfile.WriteFile(path, []byte("{}"), 0600); err != nil {
-				return nil, err
-			}
-			return &FileStore{
-				path:  path,
-				cache: map[StateKey][]byte{},
-			}, nil
-		}
-		return nil, err
-	}
-
-	ret := &FileStore{
-		path:  path,
-		cache: map[StateKey][]byte{},
-	}
-	if err := json.Unmarshal(bs, &ret.cache); err != nil {
-		return nil, err
-	}
-
-	return ret, nil
-}
-
-// ReadState implements the StateStore interface.
-func (s *FileStore) ReadState(id StateKey) ([]byte, error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-	bs, ok := s.cache[id]
-	if !ok {
-		return nil, ErrStateNotExist
-	}
-	return bs, nil
-}
-
-// WriteState implements the StateStore interface.
-func (s *FileStore) WriteState(id StateKey, bs []byte) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if bytes.Equal(s.cache[id], bs) {
-		return nil
-	}
-	s.cache[id] = append([]byte(nil), bs...)
-	bs, err := json.MarshalIndent(s.cache, "", "  ")
-	if err != nil {
-		return err
-	}
-	return atomicfile.WriteFile(s.path, bs, 0600)
-}

ipn/store/awsstore/store_aws.go

@@ -5,8 +5,8 @@
 //go:build linux
 // +build linux
 
-// Package aws contains an AWS SSM StateStore implementation.
-package aws
+// Package awsstore contains an ipn.StateStore implementation using AWS SSM.
+package awsstore
 
 import (
 	"context"
@@ -20,6 +20,8 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/ssm"
 	ssmTypes "github.com/aws/aws-sdk-go-v2/service/ssm/types"
 	"tailscale.com/ipn"
+	"tailscale.com/ipn/store/mem"
+	"tailscale.com/types/logger"
 )
 
 const (
@@ -46,12 +48,12 @@ type awsStore struct {
 	ssmClient awsSSMClient
 	ssmARN    arn.ARN
 
-	memory ipn.MemoryStore
+	memory mem.Store
 }
 
-// NewStore returns a new ipn.StateStore using the AWS SSM storage
+// New returns a new ipn.StateStore using the AWS SSM storage
 // location given by ssmARN.
-func NewStore(ssmARN string) (ipn.StateStore, error) {
+func New(_ logger.Logf, ssmARN string) (ipn.StateStore, error) {
 	return newStore(ssmARN, nil)
 }
 

ipn/store/awsstore/store_aws_stub.go

@@ -5,7 +5,7 @@
 //go:build !linux
 // +build !linux
 
-package aws
+package awsstore
 
 import (
 	"fmt"

ipn/store/awsstore/store_aws_test.go

@@ -5,7 +5,7 @@
 //go:build linux
 // +build linux
 
-package aws
+package awsstore
 
 import (
 	"context"

ipn/store/kubestore/store_kube.go

@@ -0,0 +1,85 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package kubestore contains an ipn.StateStore implementation using Kubernetes Secrets.
+
+package kubestore
+
+import (
+	"context"
+	"time"
+
+	"tailscale.com/ipn"
+	"tailscale.com/kube"
+	"tailscale.com/types/logger"
+)
+
+// Store is an ipn.StateStore that uses a Kubernetes Secret for persistence.
+type Store struct {
+	client     *kube.Client
+	secretName string
+}
+
+// New returns a new Store that persists to the named secret.
+func New(_ logger.Logf, secretName string) (*Store, error) {
+	c, err := kube.New()
+	if err != nil {
+		return nil, err
+	}
+	return &Store{
+		client:     c,
+		secretName: secretName,
+	}, nil
+}
+
+func (s *Store) String() string { return "kube.Store" }
+
+// ReadState implements the StateStore interface.
+func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	secret, err := s.client.GetSecret(ctx, s.secretName)
+	if err != nil {
+		if st, ok := err.(*kube.Status); ok && st.Code == 404 {
+			return nil, ipn.ErrStateNotExist
+		}
+		return nil, err
+	}
+	b, ok := secret.Data[string(id)]
+	if !ok {
+		return nil, ipn.ErrStateNotExist
+	}
+	return b, nil
+}
+
+// WriteState implements the StateStore interface.
+func (s *Store) WriteState(id ipn.StateKey, bs []byte) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	secret, err := s.client.GetSecret(ctx, s.secretName)
+	if err != nil {
+		if st, ok := err.(*kube.Status); ok && st.Code == 404 {
+			return s.client.CreateSecret(ctx, &kube.Secret{
+				TypeMeta: kube.TypeMeta{
+					APIVersion: "v1",
+					Kind:       "Secret",
+				},
+				ObjectMeta: kube.ObjectMeta{
+					Name: s.secretName,
+				},
+				Data: map[string][]byte{
+					string(id): bs,
+				},
+			})
+		}
+		return err
+	}
+	secret.Data[string(id)] = bs
+	if err := s.client.UpdateSecret(ctx, secret); err != nil {
+		return err
+	}
+	return err
+}

ipn/store/mem/store_mem.go

@@ -0,0 +1,69 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package mem provides an in-memory ipn.StateStore implementation.
+package mem
+
+import (
+	"encoding/json"
+	"sync"
+
+	"tailscale.com/ipn"
+	"tailscale.com/types/logger"
+)
+
+// New returns a new Store.
+func New(logger.Logf, string) (ipn.StateStore, error) {
+	return new(Store), nil
+}
+
+// Store is an ipn.StateStore that keeps state in memory only.
+type Store struct {
+	mu    sync.Mutex
+	cache map[ipn.StateKey][]byte
+}
+
+func (s *Store) String() string { return "mem.Store" }
+
+// ReadState implements the StateStore interface.
+func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	bs, ok := s.cache[id]
+	if !ok {
+		return nil, ipn.ErrStateNotExist
+	}
+	return bs, nil
+}
+
+// WriteState implements the StateStore interface.
+func (s *Store) WriteState(id ipn.StateKey, bs []byte) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.cache == nil {
+		s.cache = map[ipn.StateKey][]byte{}
+	}
+	s.cache[id] = append([]byte(nil), bs...)
+	return nil
+}
+
+// LoadFromJSON attempts to unmarshal json content into the
+// in-memory cache.
+func (s *Store) LoadFromJSON(data []byte) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return json.Unmarshal(data, &s.cache)
+}
+
+// ExportToJSON exports the content of the cache to
+// JSON formatted []byte.
+func (s *Store) ExportToJSON() ([]byte, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if len(s.cache) == 0 {
+		// Avoid "null" serialization.
+		return []byte("{}"), nil
+	}
+	return json.MarshalIndent(s.cache, "", "  ")
+}

ipn/store/stores.go

@@ -0,0 +1,193 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package store provides various implementation of ipn.StateStore.
+package store
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+
+	"tailscale.com/atomicfile"
+	"tailscale.com/ipn"
+	"tailscale.com/ipn/store/mem"
+	"tailscale.com/paths"
+	"tailscale.com/types/logger"
+)
+
+// Provider returns a StateStore for the provided path.
+// The arg is of the form "prefix:rest", where prefix was previously registered with Register.
+type Provider func(logf logger.Logf, arg string) (ipn.StateStore, error)
+
+var regOnce sync.Once
+
+var registerAvailableExternalStores func()
+
+func registerDefaultStores() {
+	Register("mem:", mem.New)
+
+	if registerAvailableExternalStores != nil {
+		registerAvailableExternalStores()
+	}
+}
+
+var knownStores map[string]Provider
+
+// New returns a StateStore based on the provided arg
+// and registered stores.
+// The arg is of the form "prefix:rest", where prefix was previously
+// registered with Register.
+//
+// By default the following stores are registered:
+//
+//   * if the string begins with "mem:", the suffix
+//     is ignored and an in-memory store is used.
+//   * (Linux-only) if the string begins with "arn:",
+//     the suffix an AWS ARN for an SSM.
+//   * (Linux-only) if the string begins with "kube:",
+//     the suffix is a Kubernetes secret name
+//   * In all other cases, the path is treated as a filepath.
+func New(logf logger.Logf, path string) (ipn.StateStore, error) {
+	regOnce.Do(registerDefaultStores)
+	for prefix, sf := range knownStores {
+		if strings.HasPrefix(path, prefix) {
+			// We can't strip the prefix here as some NewStoreFunc (like arn:)
+			// expect the prefix.
+			return sf(logf, path)
+		}
+	}
+	if runtime.GOOS == "windows" {
+		path = TryWindowsAppDataMigration(logf, path)
+	}
+	return NewFileStore(logf, path)
+}
+
+// Register registers a prefix to be used for
+// NewStore. It panics if the prefix is empty, or if the
+// prefix is already registered.
+// The provided fn is called with the path passed to NewStore;
+// the prefix is not stripped.
+func Register(prefix string, fn Provider) {
+	if len(prefix) == 0 {
+		panic("prefix is empty")
+	}
+	if _, ok := knownStores[prefix]; ok {
+		panic(fmt.Sprintf("%q already registered", prefix))
+	}
+	if knownStores == nil {
+		knownStores = make(map[string]Provider)
+	}
+	knownStores[prefix] = fn
+}
+
+// TryWindowsAppDataMigration attempts to copy the Windows state file
+// from its old location to the new location. (Issue 2856)
+//
+// Tailscale 1.14 and before stored state under %LocalAppData%
+// (usually "C:\WINDOWS\system32\config\systemprofile\AppData\Local"
+// when tailscaled.exe is running as a non-user system service).
+// However it is frequently cleared for almost any reason: Windows
+// updates, System Restore, even various System Cleaner utilities.
+//
+// Returns a string of the path to use for the state file.
+// This will be a fallback %LocalAppData% path if migration fails,
+// a %ProgramData% path otherwise.
+func TryWindowsAppDataMigration(logf logger.Logf, path string) string {
+	if path != paths.DefaultTailscaledStateFile() {
+		// If they're specifying a non-default path, just trust that they know
+		// what they are doing.
+		return path
+	}
+	oldFile := paths.LegacyStateFilePath()
+	return paths.TryConfigFileMigration(logf, oldFile, path)
+}
+
+// FileStore is a StateStore that uses a JSON file for persistence.
+type FileStore struct {
+	path string
+
+	mu    sync.RWMutex
+	cache map[ipn.StateKey][]byte
+}
+
+// Path returns the path that NewFileStore was called with.
+func (s *FileStore) Path() string { return s.path }
+
+func (s *FileStore) String() string { return fmt.Sprintf("FileStore(%q)", s.path) }
+
+// NewFileStore returns a new file store that persists to path.
+func NewFileStore(_ logger.Logf, path string) (ipn.StateStore, error) {
+	// We unconditionally call this to ensure that our perms are correct
+	if err := paths.MkStateDir(filepath.Dir(path)); err != nil {
+		return nil, fmt.Errorf("creating state directory: %w", err)
+	}
+
+	bs, err := ioutil.ReadFile(path)
+
+	// Treat an empty file as a missing file.
+	// (https://github.com/tailscale/tailscale/issues/895#issuecomment-723255589)
+	if err == nil && len(bs) == 0 {
+		log.Printf("ipn.NewFileStore(%q): file empty; treating it like a missing file [warning]", path)
+		err = os.ErrNotExist
+	}
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Write out an initial file, to verify that we can write
+			// to the path.
+			if err = atomicfile.WriteFile(path, []byte("{}"), 0600); err != nil {
+				return nil, err
+			}
+			return &FileStore{
+				path:  path,
+				cache: map[ipn.StateKey][]byte{},
+			}, nil
+		}
+		return nil, err
+	}
+
+	ret := &FileStore{
+		path:  path,
+		cache: map[ipn.StateKey][]byte{},
+	}
+	if err := json.Unmarshal(bs, &ret.cache); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+// ReadState implements the StateStore interface.
+func (s *FileStore) ReadState(id ipn.StateKey) ([]byte, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	bs, ok := s.cache[id]
+	if !ok {
+		return nil, ipn.ErrStateNotExist
+	}
+	return bs, nil
+}
+
+// WriteState implements the StateStore interface.
+func (s *FileStore) WriteState(id ipn.StateKey, bs []byte) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if bytes.Equal(s.cache[id], bs) {
+		return nil
+	}
+	s.cache[id] = append([]byte(nil), bs...)
+	bs, err := json.MarshalIndent(s.cache, "", "  ")
+	if err != nil {
+		return err
+	}
+	return atomicfile.WriteFile(s.path, bs, 0600)
+}

ipn/store/stores_linux.go

@@ -0,0 +1,26 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package store
+
+import (
+	"strings"
+
+	"tailscale.com/ipn"
+	"tailscale.com/ipn/store/awsstore"
+	"tailscale.com/ipn/store/kubestore"
+	"tailscale.com/types/logger"
+)
+
+func init() {
+	registerAvailableExternalStores = registerExternalStores
+}
+
+func registerExternalStores() {
+	Register("kube:", func(logf logger.Logf, path string) (ipn.StateStore, error) {
+		secretName := strings.TrimPrefix(path, "kube:")
+		return kubestore.New(logf, secretName)
+	})
+	Register("arn:", awsstore.New)
+}

ipn/store/stores_test.go

@@ -0,0 +1,191 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package store
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"tailscale.com/ipn"
+	"tailscale.com/ipn/store/mem"
+	"tailscale.com/tstest"
+	"tailscale.com/types/logger"
+)
+
+func TestNewStore(t *testing.T) {
+	regOnce.Do(registerDefaultStores)
+	t.Cleanup(func() {
+		knownStores = map[string]Provider{}
+		registerDefaultStores()
+	})
+	knownStores = map[string]Provider{}
+
+	type store1 struct {
+		ipn.StateStore
+		path string
+	}
+
+	type store2 struct {
+		ipn.StateStore
+		path string
+	}
+
+	Register("arn:", func(_ logger.Logf, path string) (ipn.StateStore, error) {
+		return &store1{new(mem.Store), path}, nil
+	})
+	Register("kube:", func(_ logger.Logf, path string) (ipn.StateStore, error) {
+		return &store2{new(mem.Store), path}, nil
+	})
+	Register("mem:", func(_ logger.Logf, path string) (ipn.StateStore, error) {
+		return new(mem.Store), nil
+	})
+
+	path := "mem:abcd"
+	if s, err := New(t.Logf, path); err != nil {
+		t.Fatalf("%q: %v", path, err)
+	} else if _, ok := s.(*mem.Store); !ok {
+		t.Fatalf("%q: got: %T, want: %T", path, s, new(mem.Store))
+	}
+
+	path = "arn:foo"
+	if s, err := New(t.Logf, path); err != nil {
+		t.Fatalf("%q: %v", path, err)
+	} else if _, ok := s.(*store1); !ok {
+		t.Fatalf("%q: got: %T, want: %T", path, s, new(store1))
+	}
+
+	path = "kube:abcd"
+	if s, err := New(t.Logf, path); err != nil {
+		t.Fatalf("%q: %v", path, err)
+	} else if _, ok := s.(*store2); !ok {
+		t.Fatalf("%q: got: %T, want: %T", path, s, new(store2))
+	}
+	f, err := os.CreateTemp("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		os.Remove(f.Name())
+	})
+
+	path = f.Name()
+	if s, err := New(t.Logf, path); err != nil {
+		t.Fatalf("%q: %v", path, err)
+	} else if _, ok := s.(*FileStore); !ok {
+		t.Fatalf("%q: got: %T, want: %T", path, s, new(FileStore))
+	}
+}
+
+func testStoreSemantics(t *testing.T, store ipn.StateStore) {
+	t.Helper()
+
+	tests := []struct {
+		// if true, data is data to write. If false, data is expected
+		// output of read.
+		write bool
+		id    ipn.StateKey
+		data  string
+		// If write=false, true if we expect a not-exist error.
+		notExists bool
+	}{
+		{
+			id:        "foo",
+			notExists: true,
+		},
+		{
+			write: true,
+			id:    "foo",
+			data:  "bar",
+		},
+		{
+			id:   "foo",
+			data: "bar",
+		},
+		{
+			id:        "baz",
+			notExists: true,
+		},
+		{
+			write: true,
+			id:    "baz",
+			data:  "quux",
+		},
+		{
+			id:   "foo",
+			data: "bar",
+		},
+		{
+			id:   "baz",
+			data: "quux",
+		},
+	}
+
+	for _, test := range tests {
+		if test.write {
+			if err := store.WriteState(test.id, []byte(test.data)); err != nil {
+				t.Errorf("writing %q to %q: %v", test.data, test.id, err)
+			}
+		} else {
+			bs, err := store.ReadState(test.id)
+			if err != nil {
+				if test.notExists && err == ipn.ErrStateNotExist {
+					continue
+				}
+				t.Errorf("reading %q: %v", test.id, err)
+				continue
+			}
+			if string(bs) != test.data {
+				t.Errorf("reading %q: got %q, want %q", test.id, string(bs), test.data)
+			}
+		}
+	}
+}
+
+func TestMemoryStore(t *testing.T) {
+	tstest.PanicOnLog()
+
+	store := new(mem.Store)
+	testStoreSemantics(t, store)
+}
+
+func TestFileStore(t *testing.T) {
+	tstest.PanicOnLog()
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test-file-store.conf")
+
+	store, err := NewFileStore(nil, path)
+	if err != nil {
+		t.Fatalf("creating file store failed: %v", err)
+	}
+
+	testStoreSemantics(t, store)
+
+	// Build a brand new file store and check that both IDs written
+	// above are still there.
+	store, err = NewFileStore(nil, path)
+	if err != nil {
+		t.Fatalf("creating second file store failed: %v", err)
+	}
+
+	expected := map[ipn.StateKey]string{
+		"foo": "bar",
+		"baz": "quux",
+	}
+	for key, want := range expected {
+		bs, err := store.ReadState(key)
+		if err != nil {
+			t.Errorf("reading %q (2nd store): %v", key, err)
+			continue
+		}
+		if string(bs) != want {
+			t.Errorf("reading %q (2nd store): got %q, want %q", key, bs, want)
+		}
+	}
+}

ipn/store_test.go

@@ -1,120 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package ipn
-
-import (
-	"path/filepath"
-	"testing"
-
-	"tailscale.com/tstest"
-)
-
-func testStoreSemantics(t *testing.T, store StateStore) {
-	t.Helper()
-
-	tests := []struct {
-		// if true, data is data to write. If false, data is expected
-		// output of read.
-		write bool
-		id    StateKey
-		data  string
-		// If write=false, true if we expect a not-exist error.
-		notExists bool
-	}{
-		{
-			id:        "foo",
-			notExists: true,
-		},
-		{
-			write: true,
-			id:    "foo",
-			data:  "bar",
-		},
-		{
-			id:   "foo",
-			data: "bar",
-		},
-		{
-			id:        "baz",
-			notExists: true,
-		},
-		{
-			write: true,
-			id:    "baz",
-			data:  "quux",
-		},
-		{
-			id:   "foo",
-			data: "bar",
-		},
-		{
-			id:   "baz",
-			data: "quux",
-		},
-	}
-
-	for _, test := range tests {
-		if test.write {
-			if err := store.WriteState(test.id, []byte(test.data)); err != nil {
-				t.Errorf("writing %q to %q: %v", test.data, test.id, err)
-			}
-		} else {
-			bs, err := store.ReadState(test.id)
-			if err != nil {
-				if test.notExists && err == ErrStateNotExist {
-					continue
-				}
-				t.Errorf("reading %q: %v", test.id, err)
-				continue
-			}
-			if string(bs) != test.data {
-				t.Errorf("reading %q: got %q, want %q", test.id, string(bs), test.data)
-			}
-		}
-	}
-}
-
-func TestMemoryStore(t *testing.T) {
-	tstest.PanicOnLog()
-
-	store := &MemoryStore{}
-	testStoreSemantics(t, store)
-}
-
-func TestFileStore(t *testing.T) {
-	tstest.PanicOnLog()
-
-	dir := t.TempDir()
-	path := filepath.Join(dir, "test-file-store.conf")
-
-	store, err := NewFileStore(path)
-	if err != nil {
-		t.Fatalf("creating file store failed: %v", err)
-	}
-
-	testStoreSemantics(t, store)
-
-	// Build a brand new file store and check that both IDs written
-	// above are still there.
-	store, err = NewFileStore(path)
-	if err != nil {
-		t.Fatalf("creating second file store failed: %v", err)
-	}
-
-	expected := map[StateKey]string{
-		"foo": "bar",
-		"baz": "quux",
-	}
-	for key, want := range expected {
-		bs, err := store.ReadState(key)
-		if err != nil {
-			t.Errorf("reading %q (2nd store): %v", key, err)
-			continue
-		}
-		if string(bs) != want {
-			t.Errorf("reading %q (2nd store): got %q, want %q", key, bs, want)
-		}
-	}
-}