2023-01-27 21:37:20 +00:00
|
|
|
// Copyright (c) Tailscale Inc & AUTHORS
|
|
|
|
// SPDX-License-Identifier: BSD-3-Clause
|
2022-02-28 21:08:45 +00:00
|
|
|
|
|
|
|
// Package kubestore contains an ipn.StateStore implementation using Kubernetes Secrets.
|
|
|
|
package kubestore
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2024-04-29 16:03:48 +00:00
|
|
|
"fmt"
|
2023-03-29 20:51:53 +00:00
|
|
|
"net"
|
2024-05-31 18:39:38 +00:00
|
|
|
"os"
|
2023-03-23 18:35:09 +00:00
|
|
|
"strings"
|
2022-02-28 21:08:45 +00:00
|
|
|
"time"
|
|
|
|
|
|
|
|
"tailscale.com/ipn"
|
2024-09-08 18:06:07 +00:00
|
|
|
kubeapi "tailscale.com/kube/api"
|
|
|
|
kubeclient "tailscale.com/kube/client"
|
2022-02-28 21:08:45 +00:00
|
|
|
"tailscale.com/types/logger"
|
|
|
|
)
|
|
|
|
|
|
|
|
// Store is an ipn.StateStore that uses a Kubernetes Secret for persistence.
|
|
|
|
type Store struct {
|
2024-09-08 18:06:07 +00:00
|
|
|
client kubeclient.Client
|
cmd/k8s-operator,ipn/store/kubestore: patch secrets instead of updating
We would call Update on the secret, but that was racey and would occasionaly
fail. Instead use patch whenever we can.
Fixes errors like
```
boot: 2023/08/29 01:03:53 failed to set serve config: sending serve config: updating config: writing ServeConfig to StateStore: Operation cannot be fulfilled on secrets "ts-webdav-kfrzv-0": the object has been modified; please apply your changes to the latest version and try again
{"level":"error","ts":"2023-08-29T01:03:48Z","msg":"Reconciler error","controller":"ingress","controllerGroup":"networking.k8s.io","controllerKind":"Ingress","Ingress":{"name":"webdav","namespace":"default"},"namespace":"default","name":"webdav","reconcileID":"96f5cfed-7782-4834-9b75-b0950fd563ed","error":"failed to provision: failed to create or get API key secret: Operation cannot be fulfilled on secrets \"ts-webdav-kfrzv-0\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:226"}
```
Updates #502
Updates #7895
Signed-off-by: Maisem Ali <maisem@tailscale.com>
2023-08-29 19:43:22 +00:00
|
|
|
canPatch bool
|
2022-02-28 21:08:45 +00:00
|
|
|
secretName string
|
|
|
|
}
|
|
|
|
|
|
|
|
// New returns a new Store that persists to the named secret.
|
|
|
|
func New(_ logger.Logf, secretName string) (*Store, error) {
|
2024-09-08 18:06:07 +00:00
|
|
|
c, err := kubeclient.New()
|
2022-02-28 21:08:45 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2024-05-31 18:39:38 +00:00
|
|
|
if os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" {
|
|
|
|
// Derive the API server address from the environment variables
|
|
|
|
c.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
|
|
|
|
}
|
2024-04-29 16:03:48 +00:00
|
|
|
canPatch, _, err := c.CheckSecretPermissions(context.Background(), secretName)
|
cmd/k8s-operator,ipn/store/kubestore: patch secrets instead of updating
We would call Update on the secret, but that was racey and would occasionaly
fail. Instead use patch whenever we can.
Fixes errors like
```
boot: 2023/08/29 01:03:53 failed to set serve config: sending serve config: updating config: writing ServeConfig to StateStore: Operation cannot be fulfilled on secrets "ts-webdav-kfrzv-0": the object has been modified; please apply your changes to the latest version and try again
{"level":"error","ts":"2023-08-29T01:03:48Z","msg":"Reconciler error","controller":"ingress","controllerGroup":"networking.k8s.io","controllerKind":"Ingress","Ingress":{"name":"webdav","namespace":"default"},"namespace":"default","name":"webdav","reconcileID":"96f5cfed-7782-4834-9b75-b0950fd563ed","error":"failed to provision: failed to create or get API key secret: Operation cannot be fulfilled on secrets \"ts-webdav-kfrzv-0\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:226"}
```
Updates #502
Updates #7895
Signed-off-by: Maisem Ali <maisem@tailscale.com>
2023-08-29 19:43:22 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2022-02-28 21:08:45 +00:00
|
|
|
return &Store{
|
|
|
|
client: c,
|
cmd/k8s-operator,ipn/store/kubestore: patch secrets instead of updating
We would call Update on the secret, but that was racey and would occasionaly
fail. Instead use patch whenever we can.
Fixes errors like
```
boot: 2023/08/29 01:03:53 failed to set serve config: sending serve config: updating config: writing ServeConfig to StateStore: Operation cannot be fulfilled on secrets "ts-webdav-kfrzv-0": the object has been modified; please apply your changes to the latest version and try again
{"level":"error","ts":"2023-08-29T01:03:48Z","msg":"Reconciler error","controller":"ingress","controllerGroup":"networking.k8s.io","controllerKind":"Ingress","Ingress":{"name":"webdav","namespace":"default"},"namespace":"default","name":"webdav","reconcileID":"96f5cfed-7782-4834-9b75-b0950fd563ed","error":"failed to provision: failed to create or get API key secret: Operation cannot be fulfilled on secrets \"ts-webdav-kfrzv-0\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:226"}
```
Updates #502
Updates #7895
Signed-off-by: Maisem Ali <maisem@tailscale.com>
2023-08-29 19:43:22 +00:00
|
|
|
canPatch: canPatch,
|
2022-02-28 21:08:45 +00:00
|
|
|
secretName: secretName,
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
2023-03-29 20:51:53 +00:00
|
|
|
func (s *Store) SetDialer(d func(ctx context.Context, network, address string) (net.Conn, error)) {
|
|
|
|
s.client.SetDialer(d)
|
|
|
|
}
|
|
|
|
|
2022-02-28 21:08:45 +00:00
|
|
|
func (s *Store) String() string { return "kube.Store" }
|
|
|
|
|
|
|
|
// ReadState implements the StateStore interface.
|
|
|
|
func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) {
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
|
|
defer cancel()
|
|
|
|
|
|
|
|
secret, err := s.client.GetSecret(ctx, s.secretName)
|
|
|
|
if err != nil {
|
2024-09-08 18:06:07 +00:00
|
|
|
if st, ok := err.(*kubeapi.Status); ok && st.Code == 404 {
|
2022-02-28 21:08:45 +00:00
|
|
|
return nil, ipn.ErrStateNotExist
|
|
|
|
}
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-03-23 18:35:09 +00:00
|
|
|
b, ok := secret.Data[sanitizeKey(id)]
|
2022-02-28 21:08:45 +00:00
|
|
|
if !ok {
|
|
|
|
return nil, ipn.ErrStateNotExist
|
|
|
|
}
|
|
|
|
return b, nil
|
|
|
|
}
|
|
|
|
|
2023-03-23 18:35:09 +00:00
|
|
|
func sanitizeKey(k ipn.StateKey) string {
|
|
|
|
// The only valid characters in a Kubernetes secret key are alphanumeric, -,
|
|
|
|
// _, and .
|
|
|
|
return strings.Map(func(r rune) rune {
|
|
|
|
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.' {
|
|
|
|
return r
|
|
|
|
}
|
|
|
|
return '_'
|
|
|
|
}, string(k))
|
|
|
|
}
|
|
|
|
|
2022-02-28 21:08:45 +00:00
|
|
|
// WriteState implements the StateStore interface.
|
|
|
|
func (s *Store) WriteState(id ipn.StateKey, bs []byte) error {
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
|
|
defer cancel()
|
|
|
|
|
|
|
|
secret, err := s.client.GetSecret(ctx, s.secretName)
|
|
|
|
if err != nil {
|
2024-09-08 18:06:07 +00:00
|
|
|
if kubeclient.IsNotFoundErr(err) {
|
|
|
|
return s.client.CreateSecret(ctx, &kubeapi.Secret{
|
|
|
|
TypeMeta: kubeapi.TypeMeta{
|
2022-02-28 21:08:45 +00:00
|
|
|
APIVersion: "v1",
|
|
|
|
Kind: "Secret",
|
|
|
|
},
|
2024-09-08 18:06:07 +00:00
|
|
|
ObjectMeta: kubeapi.ObjectMeta{
|
2022-02-28 21:08:45 +00:00
|
|
|
Name: s.secretName,
|
|
|
|
},
|
|
|
|
Data: map[string][]byte{
|
2023-03-23 18:35:09 +00:00
|
|
|
sanitizeKey(id): bs,
|
2022-02-28 21:08:45 +00:00
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
cmd/k8s-operator,ipn/store/kubestore: patch secrets instead of updating
We would call Update on the secret, but that was racey and would occasionaly
fail. Instead use patch whenever we can.
Fixes errors like
```
boot: 2023/08/29 01:03:53 failed to set serve config: sending serve config: updating config: writing ServeConfig to StateStore: Operation cannot be fulfilled on secrets "ts-webdav-kfrzv-0": the object has been modified; please apply your changes to the latest version and try again
{"level":"error","ts":"2023-08-29T01:03:48Z","msg":"Reconciler error","controller":"ingress","controllerGroup":"networking.k8s.io","controllerKind":"Ingress","Ingress":{"name":"webdav","namespace":"default"},"namespace":"default","name":"webdav","reconcileID":"96f5cfed-7782-4834-9b75-b0950fd563ed","error":"failed to provision: failed to create or get API key secret: Operation cannot be fulfilled on secrets \"ts-webdav-kfrzv-0\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:226"}
```
Updates #502
Updates #7895
Signed-off-by: Maisem Ali <maisem@tailscale.com>
2023-08-29 19:43:22 +00:00
|
|
|
if s.canPatch {
|
2024-04-29 16:03:48 +00:00
|
|
|
if len(secret.Data) == 0 { // if user has pre-created a blank Secret
|
2024-09-08 18:06:07 +00:00
|
|
|
m := []kubeclient.JSONPatch{
|
2024-04-29 16:03:48 +00:00
|
|
|
{
|
|
|
|
Op: "add",
|
|
|
|
Path: "/data",
|
|
|
|
Value: map[string][]byte{sanitizeKey(id): bs},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
if err := s.client.JSONPatchSecret(ctx, s.secretName, m); err != nil {
|
|
|
|
return fmt.Errorf("error patching Secret %s with a /data field: %v", s.secretName, err)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
2024-09-08 18:06:07 +00:00
|
|
|
m := []kubeclient.JSONPatch{
|
cmd/k8s-operator,ipn/store/kubestore: patch secrets instead of updating
We would call Update on the secret, but that was racey and would occasionaly
fail. Instead use patch whenever we can.
Fixes errors like
```
boot: 2023/08/29 01:03:53 failed to set serve config: sending serve config: updating config: writing ServeConfig to StateStore: Operation cannot be fulfilled on secrets "ts-webdav-kfrzv-0": the object has been modified; please apply your changes to the latest version and try again
{"level":"error","ts":"2023-08-29T01:03:48Z","msg":"Reconciler error","controller":"ingress","controllerGroup":"networking.k8s.io","controllerKind":"Ingress","Ingress":{"name":"webdav","namespace":"default"},"namespace":"default","name":"webdav","reconcileID":"96f5cfed-7782-4834-9b75-b0950fd563ed","error":"failed to provision: failed to create or get API key secret: Operation cannot be fulfilled on secrets \"ts-webdav-kfrzv-0\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:226"}
```
Updates #502
Updates #7895
Signed-off-by: Maisem Ali <maisem@tailscale.com>
2023-08-29 19:43:22 +00:00
|
|
|
{
|
|
|
|
Op: "add",
|
|
|
|
Path: "/data/" + sanitizeKey(id),
|
|
|
|
Value: bs,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
if err := s.client.JSONPatchSecret(ctx, s.secretName, m); err != nil {
|
2024-04-29 16:03:48 +00:00
|
|
|
return fmt.Errorf("error patching Secret %s with /data/%s field", s.secretName, sanitizeKey(id))
|
cmd/k8s-operator,ipn/store/kubestore: patch secrets instead of updating
We would call Update on the secret, but that was racey and would occasionaly
fail. Instead use patch whenever we can.
Fixes errors like
```
boot: 2023/08/29 01:03:53 failed to set serve config: sending serve config: updating config: writing ServeConfig to StateStore: Operation cannot be fulfilled on secrets "ts-webdav-kfrzv-0": the object has been modified; please apply your changes to the latest version and try again
{"level":"error","ts":"2023-08-29T01:03:48Z","msg":"Reconciler error","controller":"ingress","controllerGroup":"networking.k8s.io","controllerKind":"Ingress","Ingress":{"name":"webdav","namespace":"default"},"namespace":"default","name":"webdav","reconcileID":"96f5cfed-7782-4834-9b75-b0950fd563ed","error":"failed to provision: failed to create or get API key secret: Operation cannot be fulfilled on secrets \"ts-webdav-kfrzv-0\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\tsigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:226"}
```
Updates #502
Updates #7895
Signed-off-by: Maisem Ali <maisem@tailscale.com>
2023-08-29 19:43:22 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
2023-03-23 18:35:09 +00:00
|
|
|
secret.Data[sanitizeKey(id)] = bs
|
2022-02-28 21:08:45 +00:00
|
|
|
if err := s.client.UpdateSecret(ctx, secret); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|