diff --git a/cmd/k8s-operator/e2e/ingress_test.go b/cmd/k8s-operator/e2e/ingress_test.go new file mode 100644 index 000000000..373dd2c7d --- /dev/null +++ b/cmd/k8s-operator/e2e/ingress_test.go @@ -0,0 +1,108 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package e2e + +import ( + "context" + "fmt" + "net/http" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + kube "tailscale.com/k8s-operator" + "tailscale.com/tstest" +) + +// See [TestMain] for test requirements. +func TestIngress(t *testing.T) { + if tsClient == nil { + t.Skip("TestIngress requires credentials for a tailscale client") + } + + ctx := context.Background() + cfg := config.GetConfigOrDie() + cl, err := client.New(cfg, client.Options{}) + if err != nil { + t.Fatal(err) + } + // Apply nginx + createAndCleanup(t, ctx, cl, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx", + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/name": "nginx", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }) + // Apply service to expose it as ingress + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress", + Namespace: "default", + Annotations: map[string]string{ + "tailscale.com/expose": "true", + }, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app.kubernetes.io/name": "nginx", + }, + Ports: []corev1.ServicePort{ + { + Name: "http", + Protocol: "TCP", + Port: 80, + }, + }, + }, + } + createAndCleanup(t, ctx, cl, svc) + + // TODO: instead of timing out only when test times out, cancel context after 60s or so. + if err := wait.PollUntilContextCancel(ctx, time.Millisecond*100, true, func(ctx context.Context) (done bool, err error) { + maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta("default", "test-ingress")} + if err := get(ctx, cl, maybeReadySvc); err != nil { + return false, err + } + isReady := kube.SvcIsReady(maybeReadySvc) + if isReady { + t.Log("Service is ready") + } + return isReady, nil + }); err != nil { + t.Fatalf("error waiting for the Service to become Ready: %v", err) + } + + var resp *http.Response + if err := tstest.WaitFor(time.Second*60, func() error { + // TODO(tomhjp): Get the tailnet DNS name from the associated secret instead. + // If we are not the first tailnet node with the requested name, we'll get + // a -N suffix. + resp, err = tsClient.HTTPClient.Get(fmt.Sprintf("http://%s-%s:80", svc.Namespace, svc.Name)) + if err != nil { + return err + } + return nil + }); err != nil { + t.Fatalf("error trying to reach service: %v", err) + } + + if resp.StatusCode != http.StatusOK { + t.Fatalf("unexpected status: %v; response body s", resp.StatusCode) + } +} diff --git a/cmd/k8s-operator/e2e/main_test.go b/cmd/k8s-operator/e2e/main_test.go new file mode 100644 index 000000000..ae23c939c --- /dev/null +++ b/cmd/k8s-operator/e2e/main_test.go @@ -0,0 +1,194 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package e2e + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "slices" + "strings" + "testing" + + "github.com/go-logr/zapr" + "github.com/tailscale/hujson" + "go.uber.org/zap/zapcore" + "golang.org/x/oauth2/clientcredentials" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + kzap "sigs.k8s.io/controller-runtime/pkg/log/zap" + "tailscale.com/client/tailscale" +) + +const ( + e2eManagedComment = "// This is managed by the k8s-operator e2e tests" +) + +var ( + tsClient *tailscale.Client + testGrants = map[string]string{ + "test-proxy": `{ + "src": ["tag:e2e-test-proxy"], + "dst": ["tag:k8s-operator"], + "app": { + "tailscale.com/cap/kubernetes": [{ + "impersonate": { + "groups": ["ts:e2e-test-proxy"], + }, + }], + }, + }`, + } +) + +// This test suite is currently not run in CI. +// It requires some setup not handled by this code: +// - Kubernetes cluster with tailscale operator installed +// - Current kubeconfig context set to connect to that cluster (directly, no operator proxy) +// - Operator installed with --set apiServerProxyConfig.mode="true" +// - ACLs that define tag:e2e-test-proxy tag. TODO(tomhjp): Can maybe replace this prereq onwards with an API key +// - OAuth client ID and secret in TS_API_CLIENT_ID and TS_API_CLIENT_SECRET env +// - OAuth client must have auth_keys and policy_file write for tag:e2e-test-proxy tag +func TestMain(m *testing.M) { + code, err := runTests(m) + if err != nil { + log.Fatal(err) + } + os.Exit(code) +} + +func runTests(m *testing.M) (int, error) { + zlog := kzap.NewRaw([]kzap.Opts{kzap.UseDevMode(true), kzap.Level(zapcore.DebugLevel)}...).Sugar() + logf.SetLogger(zapr.NewLogger(zlog.Desugar())) + tailscale.I_Acknowledge_This_API_Is_Unstable = true + + if clientID := os.Getenv("TS_API_CLIENT_ID"); clientID != "" { + cleanup, err := setupClientAndACLs() + if err != nil { + return 0, err + } + defer func() { + err = errors.Join(err, cleanup()) + }() + } + + return m.Run(), nil +} + +func setupClientAndACLs() (cleanup func() error, _ error) { + ctx := context.Background() + credentials := clientcredentials.Config{ + ClientID: os.Getenv("TS_API_CLIENT_ID"), + ClientSecret: os.Getenv("TS_API_CLIENT_SECRET"), + TokenURL: "https://login.tailscale.com/api/v2/oauth/token", + Scopes: []string{"auth_keys", "policy_file"}, + } + tsClient = tailscale.NewClient("-", nil) + tsClient.HTTPClient = credentials.Client(ctx) + + if err := patchACLs(ctx, tsClient, func(acls *hujson.Value) { + for test, grant := range testGrants { + deleteTestGrants(test, acls) + addTestGrant(test, grant, acls) + } + }); err != nil { + return nil, err + } + + return func() error { + return patchACLs(ctx, tsClient, func(acls *hujson.Value) { + for test := range testGrants { + deleteTestGrants(test, acls) + } + }) + }, nil +} + +func patchACLs(ctx context.Context, tsClient *tailscale.Client, patchFn func(*hujson.Value)) error { + acls, err := tsClient.ACLHuJSON(ctx) + if err != nil { + return err + } + hj, err := hujson.Parse([]byte(acls.ACL)) + if err != nil { + return err + } + + patchFn(&hj) + + hj.Format() + acls.ACL = hj.String() + if _, err := tsClient.SetACLHuJSON(ctx, *acls, true); err != nil { + return err + } + + return nil +} + +func addTestGrant(test, grant string, acls *hujson.Value) error { + v, err := hujson.Parse([]byte(grant)) + if err != nil { + return err + } + + // Add the managed comment to the first line of the grant object contents. + v.Value.(*hujson.Object).Members[0].Name.BeforeExtra = hujson.Extra(fmt.Sprintf("%s: %s\n", e2eManagedComment, test)) + + if err := acls.Patch([]byte(fmt.Sprintf(`[{"op": "add", "path": "/grants/-", "value": %s}]`, v.String()))); err != nil { + return err + } + + return nil +} + +func deleteTestGrants(test string, acls *hujson.Value) error { + grants := acls.Find("/grants") + + var patches []string + for i, g := range grants.Value.(*hujson.Array).Elements { + members := g.Value.(*hujson.Object).Members + if len(members) == 0 { + continue + } + comment := strings.TrimSpace(string(members[0].Name.BeforeExtra)) + if name, found := strings.CutPrefix(comment, e2eManagedComment+": "); found && name == test { + patches = append(patches, fmt.Sprintf(`{"op": "remove", "path": "/grants/%d"}`, i)) + } + } + + // Remove in reverse order so we don't affect the found indices as we mutate. + slices.Reverse(patches) + + if err := acls.Patch([]byte(fmt.Sprintf("[%s]", strings.Join(patches, ",")))); err != nil { + return err + } + + return nil +} + +func objectMeta(namespace, name string) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + } +} + +func createAndCleanup(t *testing.T, ctx context.Context, cl client.Client, obj client.Object) { + t.Helper() + if err := cl.Create(ctx, obj); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := cl.Delete(ctx, obj); err != nil { + t.Errorf("error cleaning up %s %s/%s: %s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName(), err) + } + }) +} + +func get(ctx context.Context, cl client.Client, obj client.Object) error { + return cl.Get(ctx, client.ObjectKeyFromObject(obj), obj) +} diff --git a/cmd/k8s-operator/e2e/proxy_test.go b/cmd/k8s-operator/e2e/proxy_test.go new file mode 100644 index 000000000..eac983e88 --- /dev/null +++ b/cmd/k8s-operator/e2e/proxy_test.go @@ -0,0 +1,156 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "tailscale.com/client/tailscale" + "tailscale.com/tsnet" + "tailscale.com/tstest" +) + +// See [TestMain] for test requirements. +func TestProxy(t *testing.T) { + if tsClient == nil { + t.Skip("TestProxy requires credentials for a tailscale client") + } + + ctx := context.Background() + cfg := config.GetConfigOrDie() + cl, err := client.New(cfg, client.Options{}) + if err != nil { + t.Fatal(err) + } + + // Create role and role binding to allow a group we'll impersonate to do stuff. + createAndCleanup(t, ctx, cl, &rbacv1.Role{ + ObjectMeta: objectMeta("tailscale", "read-secrets"), + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Verbs: []string{"get"}, + Resources: []string{"secrets"}, + }}, + }) + createAndCleanup(t, ctx, cl, &rbacv1.RoleBinding{ + ObjectMeta: objectMeta("tailscale", "read-secrets"), + Subjects: []rbacv1.Subject{{ + Kind: "Group", + Name: "ts:e2e-test-proxy", + }}, + RoleRef: rbacv1.RoleRef{ + Kind: "Role", + Name: "read-secrets", + }, + }) + + // Get operator host name from kube secret. + operatorSecret := corev1.Secret{ + ObjectMeta: objectMeta("tailscale", "operator"), + } + if err := get(ctx, cl, &operatorSecret); err != nil { + t.Fatal(err) + } + + // Connect to tailnet with test-specific tag so we can use the + // [testGrants] ACLs when connecting to the API server proxy + ts := tsnetServerWithTag(t, ctx, "tag:e2e-test-proxy") + proxyCfg := &rest.Config{ + Host: fmt.Sprintf("https://%s:443", hostNameFromOperatorSecret(t, operatorSecret)), + Dial: ts.Dial, + } + proxyCl, err := client.New(proxyCfg, client.Options{}) + if err != nil { + t.Fatal(err) + } + + // Expect success. + allowedSecret := corev1.Secret{ + ObjectMeta: objectMeta("tailscale", "operator"), + } + // Wait for up to a minute the first time we use the proxy, to give it time + // to provision the TLS certs. + if err := tstest.WaitFor(time.Second*60, func() error { + return get(ctx, proxyCl, &allowedSecret) + }); err != nil { + t.Fatal(err) + } + + // Expect forbidden. + forbiddenSecret := corev1.Secret{ + ObjectMeta: objectMeta("default", "operator"), + } + if err := get(ctx, proxyCl, &forbiddenSecret); err == nil || !apierrors.IsForbidden(err) { + t.Fatalf("expected forbidden error fetching secret from default namespace: %s", err) + } +} + +func tsnetServerWithTag(t *testing.T, ctx context.Context, tag string) *tsnet.Server { + caps := tailscale.KeyCapabilities{ + Devices: tailscale.KeyDeviceCapabilities{ + Create: tailscale.KeyDeviceCreateCapabilities{ + Reusable: false, + Preauthorized: true, + Ephemeral: true, + Tags: []string{tag}, + }, + }, + } + + authKey, authKeyMeta, err := tsClient.CreateKey(ctx, caps) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := tsClient.DeleteKey(ctx, authKeyMeta.ID); err != nil { + t.Errorf("error deleting auth key: %s", err) + } + }) + + ts := &tsnet.Server{ + Hostname: "test-proxy", + Ephemeral: true, + Dir: t.TempDir(), + AuthKey: authKey, + } + _, err = ts.Up(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := ts.Close(); err != nil { + t.Errorf("error shutting down tsnet.Server: %s", err) + } + }) + + return ts +} + +func hostNameFromOperatorSecret(t *testing.T, s corev1.Secret) string { + profiles := map[string]any{} + if err := json.Unmarshal(s.Data["_profiles"], &profiles); err != nil { + t.Fatal(err) + } + key, ok := strings.CutPrefix(string(s.Data["_current-profile"]), "profile-") + if !ok { + t.Fatal(string(s.Data["_current-profile"])) + } + profile, ok := profiles[key] + if !ok { + t.Fatal(profiles) + } + + return ((profile.(map[string]any))["Name"]).(string) +} diff --git a/k8s-operator/conditions.go b/k8s-operator/conditions.go index ace0fb7e3..1ecedfc07 100644 --- a/k8s-operator/conditions.go +++ b/k8s-operator/conditions.go @@ -167,3 +167,14 @@ func DNSCfgIsReady(cfg *tsapi.DNSConfig) bool { cond := cfg.Status.Conditions[idx] return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == cfg.Generation } + +func SvcIsReady(svc *corev1.Service) bool { + idx := xslices.IndexFunc(svc.Status.Conditions, func(cond metav1.Condition) bool { + return cond.Type == string(tsapi.ProxyReady) + }) + if idx == -1 { + return false + } + cond := svc.Status.Conditions[idx] + return cond.Status == metav1.ConditionTrue +}