TheArchive/tailscale
1
0
Fork 0
mirror of https://github.com/tailscale/tailscale.git synced 2024-11-25 19:15:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

cmd/k8s-operator,k8s-operator: proxy configuration mechanism via a new ProxyClass custom resource (#11074)

Browse Source 
* cmd/k8s-operator,k8s-operator: introduce proxy configuration mechanism via ProxyClass custom resource.

ProxyClass custom resource can be used to specify customizations
for the proxy resources created by the operator.

Add a reconciler that validates ProxyClass resources
and sets a Ready condition to True or False with a corresponding reason and message.
This is required because some fields (labels and annotations)
require complex validations that cannot be performed at custom resource apply time.
Reconcilers that use the ProxyClass to configure proxy resources are expected to
verify that the ProxyClass is Ready and not proceed with resource creation
if configuration from a ProxyClass that is not yet Ready is required.

If a tailscale ingress/egress Service is annotated with a tailscale.com/proxy-class annotation, look up the corresponding ProxyClass and, if it is Ready, apply the configuration from the ProxyClass to the proxy's StatefulSet.

If a tailscale Ingress has a tailscale.com/proxy-class annotation
and the referenced ProxyClass custom resource is available and Ready,
apply configuration from the ProxyClass to the proxy resources
that will be created for the Ingress.

Add a new .proxyClass field to the Connector spec.
If connector.spec.proxyClass is set to a ProxyClass that is available and Ready,
apply configuration from the ProxyClass to the proxy resources created for the Connector.

Ensure that when Helm chart is packaged, the ProxyClass yaml is added to chart templates. Ensure that static manifest generator adds ProxyClass yaml to operator.yaml. Regenerate operator.yaml


Signed-off-by: Irbe Krumina <irbe@tailscale.com>
This commit is contained in:
Irbe Krumina 2024-02-13 05:27:54 +00:00 committed by GitHub
parent f7f496025a
commit 5bd19fd3e3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 2584 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
cmd/k8s-operator/connector.go
View File

@ -164,6 +164,17 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
hostname = string(cn.Spec.Hostname)
}
crl := childResourceLabels(cn.Name, a.tsnamespace, "connector")
proxyClass := cn.Spec.ProxyClass
if proxyClass != "" {
if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
return fmt.Errorf("error verifying ProxyClass for Connector: %w", err)
} else if !ready {
logger.Infof("ProxyClass %s specified for the Connector, but is not (yet) Ready, waiting..", proxyClass)
return nil
}
}
sts := &tailscaleSTSConfig{
ParentResourceName: cn.Name,
ParentResourceUID: string(cn.UID),
@ -173,6 +184,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
Connector: &connector{
isExitNode: cn.Spec.ExitNode,
},
ProxyClass: proxyClass,
}
if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 {

120
cmd/k8s-operator/connector_test.go
View File

@ -77,7 +77,7 @@ func TestConnector(t *testing.T) {
confFileHash: "9321660203effb80983eaecc7b5ac5a8c53934926f46e895b9fe295dcfc5a904",
}
expectEqual(t, fc, expectedSecret(t, opts))
expectEqual(t, fc, expectedSTS(opts))
expectEqual(t, fc, expectedSTS(t, fc, opts))
// Add another route to be advertised.
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@ -87,7 +87,7 @@ func TestConnector(t *testing.T) {
opts.confFileHash = "fb6c4daf67425f983985750cd8d6f2beae77e614fcb34176604571f5623d6862"
expectReconciled(t, cr, "", "test")
expectEqual(t, fc, expectedSTS(opts))
expectEqual(t, fc, expectedSTS(t, fc, opts))
// Remove a route.
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@ -96,7 +96,7 @@ func TestConnector(t *testing.T) {
opts.subnetRoutes = "10.44.0.0/20"
opts.confFileHash = "bacba177bcfe3849065cf6fee53d658a9bb4144197ac5b861727d69ea99742bb"
expectReconciled(t, cr, "", "test")
expectEqual(t, fc, expectedSTS(opts))
expectEqual(t, fc, expectedSTS(t, fc, opts))
// Remove the subnet router.
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@ -105,7 +105,7 @@ func TestConnector(t *testing.T) {
opts.subnetRoutes = ""
opts.confFileHash = "7c421a99128eb80e79a285a82702f19f8f720615542a15bd794858a6275d8079"
expectReconciled(t, cr, "", "test")
expectEqual(t, fc, expectedSTS(opts))
expectEqual(t, fc, expectedSTS(t, fc, opts))
// Re-add the subnet router.
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@ -116,7 +116,7 @@ func TestConnector(t *testing.T) {
opts.subnetRoutes = "10.44.0.0/20"
opts.confFileHash = "bacba177bcfe3849065cf6fee53d658a9bb4144197ac5b861727d69ea99742bb"
expectReconciled(t, cr, "", "test")
expectEqual(t, fc, expectedSTS(opts))
expectEqual(t, fc, expectedSTS(t, fc, opts))
// Delete the Connector.
if err = fc.Delete(context.Background(), cn); err != nil {
@ -161,7 +161,7 @@ func TestConnector(t *testing.T) {
confFileHash: "57d922331890c9b1c8c6ae664394cb254334c551d9cd9db14537b5d9da9fb17e",
}
expectEqual(t, fc, expectedSecret(t, opts))
expectEqual(t, fc, expectedSTS(opts))
expectEqual(t, fc, expectedSTS(t, fc, opts))
// Add an exit node.
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@ -170,7 +170,7 @@ func TestConnector(t *testing.T) {
opts.isExitNode = true
opts.confFileHash = "1499b591fd97a50f0330db6ec09979792c49890cf31f5da5bb6a3f50dba1e77a"
expectReconciled(t, cr, "", "test")
expectEqual(t, fc, expectedSTS(opts))
expectEqual(t, fc, expectedSTS(t, fc, opts))
// Delete the Connector.
if err = fc.Delete(context.Background(), cn); err != nil {
@ -183,3 +183,109 @@ func TestConnector(t *testing.T) {
expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
}
func TestConnectorWithProxyClass(t *testing.T) {
// Setup
pc := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"},
Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{
Labels: map[string]string{"foo": "bar"},
Annotations: map[string]string{"bar.io/foo": "some-val"},
Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
}
cn := &tsapi.Connector{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
UID: types.UID("1234-UID"),
},
TypeMeta: metav1.TypeMeta{
Kind: tsapi.ConnectorKind,
APIVersion: "tailscale.io/v1alpha1",
},
Spec: tsapi.ConnectorSpec{
SubnetRouter: &tsapi.SubnetRouter{
AdvertiseRoutes: []tsapi.Route{"10.40.0.0/14"},
},
ExitNode: true,
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pc, cn).
WithStatusSubresource(pc, cn).
Build()
ft := &fakeTSClient{}
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
cl := tstest.NewClock(tstest.ClockOpts{})
cr := &ConnectorReconciler{
Client: fc,
clock: cl,
ssr: &tailscaleSTSReconciler{
Client: fc,
tsClient: ft,
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
}
// 1. Connector is created with no ProxyClass specified, create
// resources with the default configuration.
expectReconciled(t, cr, "", "test")
fullName, shortName := findGenName(t, fc, "", "test", "connector")
opts := configOpts{
stsName: shortName,
secretName: fullName,
parentType: "connector",
hostname: "test-connector",
shouldUseDeclarativeConfig: true,
isExitNode: true,
subnetRoutes: "10.40.0.0/14",
confFileHash: "9321660203effb80983eaecc7b5ac5a8c53934926f46e895b9fe295dcfc5a904",
}
expectEqual(t, fc, expectedSecret(t, opts))
expectEqual(t, fc, expectedSTS(t, fc, opts))
// 2. Update Connector to specify a ProxyClass. ProxyClass is not yet
// ready, so its configuration is NOT applied to the Connector
// resources.
mustUpdate(t, fc, "", "test", func(conn *tsapi.Connector) {
conn.Spec.ProxyClass = "custom-metadata"
})
expectReconciled(t, cr, "", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts))
// 3. ProxyClass is set to Ready by proxy-class reconciler. Connector
// get reconciled and configuration from the ProxyClass is applied to
// its resources.
mustUpdateStatus(t, fc, "", "custom-metadata", func(pc *tsapi.ProxyClass) {
pc.Status = tsapi.ProxyClassStatus{
Conditions: []tsapi.ConnectorCondition{{
Status: metav1.ConditionTrue,
Type: tsapi.ProxyClassready,
ObservedGeneration: pc.Generation,
}}}
})
opts.proxyClass = pc.Name
// We lose the auth key on second reconcile, because in code it's set to
// StringData, but is actually read from Data. This works with a real
// API server, but not with our test setup here.
opts.confFileHash = "1499b591fd97a50f0330db6ec09979792c49890cf31f5da5bb6a3f50dba1e77a"
expectReconciled(t, cr, "", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts))
// 4. Connector.spec.proxyClass field is unset, Connector gets
// reconciled and configuration from the ProxyClass is removed from the
// cluster resources for the Connector.
mustUpdate(t, fc, "", "test", func(conn *tsapi.Connector) {
conn.Spec.ProxyClass = ""
})
opts.proxyClass = ""
expectReconciled(t, cr, "", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts))
}

2
cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml
View File

@ -22,7 +22,7 @@ rules:
resources: ["ingressclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["tailscale.com"]
resources: ["connectors", "connectors/status"]
resources: ["connectors", "connectors/status", "proxyclasses", "proxyclasses/status"]
verbs: ["get", "list", "watch", "update"]
---
apiVersion: rbac.authorization.k8s.io/v1

3
cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml
View File

@ -54,6 +54,9 @@ spec:
description: Hostname is the tailnet hostname that should be assigned to the Connector node. If unset, hostname defaults to <connector name>-connector. Hostname can contain lower case letters, numbers and dashes, it must not start or end with a dash and must be between 2 and 63 characters long.
type: string
pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$
proxyClass:
description: ProxyClass is the name of the ProxyClass custom resource that contains configuration options that should be applied to the resources created for this Connector. If unset, the operator will create resources with the default configuration.
type: string
subnetRouter:
description: SubnetRouter defines subnet routes that the Connector node should expose to tailnet. If unset, none are exposed. https://tailscale.com/kb/1019/subnets/
type: object

494
cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml Normal file
View File

@ -0,0 +1,494 @@
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.13.0
name: proxyclasses.tailscale.com
spec:
group: tailscale.com
names:
kind: ProxyClass
listKind: ProxyClassList
plural: proxyclasses
singular: proxyclass
scope: Cluster
versions:
- additionalPrinterColumns:
- description: Status of the ProxyClass.
jsonPath: .status.conditions[?(@.type == "ProxyClassReady")].reason
name: Status
type: string
name: v1alpha1
schema:
openAPIV3Schema:
type: object
required:
- spec
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
kind:
description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
type: string
metadata:
type: object
spec:
type: object
required:
- statefulSet
properties:
statefulSet:
description: Proxy's StatefulSet spec.
type: object
properties:
annotations:
description: Annotations that will be added to the StatefulSet created for the proxy. Any Annotations specified here will be merged with the default annotations applied to the StatefulSet by the Tailscale Kubernetes operator as well as any other annotations that might have been applied by other actors. Annotations must be valid Kubernetes annotations. https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set
type: object
additionalProperties:
type: string
labels:
description: Labels that will be added to the StatefulSet created for the proxy. Any labels specified here will be merged with the default labels applied to the StatefulSet by the Tailscale Kubernetes operator as well as any other labels that might have been applied by other actors. Label keys and values must be valid Kubernetes label keys and values. https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
type: object
additionalProperties:
type: string
pod:
description: Configuration for the proxy Pod.
type: object
properties:
annotations:
description: Annotations that will be added to the proxy Pod. Any annotations specified here will be merged with the default annotations applied to the Pod by the Tailscale Kubernetes operator. Annotations must be valid Kubernetes annotations. https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set
type: object
additionalProperties:
type: string
imagePullSecrets:
description: Proxy Pod's image pull Secrets. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec
type: array
items:
description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.
type: object
properties:
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
type: string
x-kubernetes-map-type: atomic
labels:
description: Labels that will be added to the proxy Pod. Any labels specified here will be merged with the default labels applied to the Pod by the Tailscale Kubernetes operator. Label keys and values must be valid Kubernetes label keys and values. https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
type: object
additionalProperties:
type: string
nodeName:
description: Proxy Pod's node name. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling
type: string
nodeSelector:
description: Proxy Pod's node selector. By default Tailscale Kubernetes operator does not apply any node selector. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling
type: object
additionalProperties:
type: string
securityContext:
description: Proxy Pod's security context. By default Tailscale Kubernetes operator does not apply any Pod security context. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context-2
type: object
properties:
fsGroup:
description: "A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: \n 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \n If unset, the Kubelet will not modify the ownership and permissions of any volume. Note that this field cannot be set when spec.os.name is windows."
type: integer
format: int64
fsGroupChangePolicy:
description: 'fsGroupChangePolicy defines behavior of changing ownership and permission of the volume before being exposed inside Pod. This field will only apply to volume types which support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. Note that this field cannot be set when spec.os.name is windows.'
type: string
runAsGroup:
description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows.
type: integer
format: int64
runAsNonRoot:
description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.
type: boolean
runAsUser:
description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows.
type: integer
format: int64
seLinuxOptions:
description: The SELinux context to be applied to all containers. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows.
type: object
properties:
level:
description: Level is SELinux level label that applies to the container.
type: string
role:
description: Role is a SELinux role label that applies to the container.
type: string
type:
description: Type is a SELinux type label that applies to the container.
type: string
user:
description: User is a SELinux user label that applies to the container.
type: string
seccompProfile:
description: The seccomp options to use by the containers in this pod. Note that this field cannot be set when spec.os.name is windows.
type: object
required:
- type
properties:
localhostProfile:
description: localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must be set if type is "Localhost". Must NOT be set for any other type.
type: string
type:
description: "type indicates which kind of seccomp profile will be applied. Valid options are: \n Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied."
type: string
supplementalGroups:
description: A list of groups applied to the first process run in each container, in addition to the container's primary GID, the fsGroup (if specified), and group memberships defined in the container image for the uid of the container process. If unspecified, no additional groups are added to any container. Note that group memberships defined in the container image for the uid of the container process are still effective, even if they are not included in this list. Note that this field cannot be set when spec.os.name is windows.
type: array
items:
type: integer
format: int64
sysctls:
description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch. Note that this field cannot be set when spec.os.name is windows.
type: array
items:
description: Sysctl defines a kernel parameter to be set
type: object
required:
- name
- value
properties:
name:
description: Name of a property to set
type: string
value:
description: Value of a property to set
type: string
windowsOptions:
description: The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux.
type: object
properties:
gmsaCredentialSpec:
description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.
type: string
gmsaCredentialSpecName:
description: GMSACredentialSpecName is the name of the GMSA credential spec to use.
type: string
hostProcess:
description: HostProcess determines if a container should be run as a 'Host Process' container. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). In addition, if HostProcess is true then HostNetwork must also be set to true.
type: boolean
runAsUserName:
description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.
type: string
tailscaleContainer:
description: Configuration for the proxy container running tailscale.
type: object
properties:
resources:
description: Container resource requirements. By default Tailscale Kubernetes operator does not apply any resource requirements. The amount of resources required wil depend on the amount of resources the operator needs to parse, usage patterns and cluster size. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources
type: object
properties:
claims:
description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers."
type: array
items:
description: ResourceClaim references one entry in PodSpec.ResourceClaims.
type: object
required:
- name
properties:
name:
description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container.
type: string
x-kubernetes-list-map-keys:
- name
x-kubernetes-list-type: map
limits:
description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
type: object
additionalProperties:
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
anyOf:
- type: integer
- type: string
x-kubernetes-int-or-string: true
requests:
description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
type: object
additionalProperties:
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
anyOf:
- type: integer
- type: string
x-kubernetes-int-or-string: true
securityContext:
description: 'Container security context. Security context specified here will override the security context by the operator. By default the operator: - sets ''privileged: true'' for the init container - set NET_ADMIN capability for tailscale container for proxies that are created for Services or Connector. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context'
type: object
properties:
allowPrivilegeEscalation:
description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN Note that this field cannot be set when spec.os.name is windows.'
type: boolean
capabilities:
description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. Note that this field cannot be set when spec.os.name is windows.
type: object
properties:
add:
description: Added capabilities
type: array
items:
description: Capability represent POSIX capabilities type
type: string
drop:
description: Removed capabilities
type: array
items:
description: Capability represent POSIX capabilities type
type: string
privileged:
description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. Note that this field cannot be set when spec.os.name is windows.
type: boolean
procMount:
description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows.
type: string
readOnlyRootFilesystem:
description: Whether this container has a read-only root filesystem. Default is false. Note that this field cannot be set when spec.os.name is windows.
type: boolean
runAsGroup:
description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows.
type: integer
format: int64
runAsNonRoot:
description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.
type: boolean
runAsUser:
description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows.
type: integer
format: int64
seLinuxOptions:
description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows.
type: object
properties:
level:
description: Level is SELinux level label that applies to the container.
type: string
role:
description: Role is a SELinux role label that applies to the container.
type: string
type:
description: Type is a SELinux type label that applies to the container.
type: string
user:
description: User is a SELinux user label that applies to the container.
type: string
seccompProfile:
description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. Note that this field cannot be set when spec.os.name is windows.
type: object
required:
- type
properties:
localhostProfile:
description: localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must be set if type is "Localhost". Must NOT be set for any other type.
type: string
type:
description: "type indicates which kind of seccomp profile will be applied. Valid options are: \n Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied."
type: string
windowsOptions:
description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux.
type: object
properties:
gmsaCredentialSpec:
description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.
type: string
gmsaCredentialSpecName:
description: GMSACredentialSpecName is the name of the GMSA credential spec to use.
type: string
hostProcess:
description: HostProcess determines if a container should be run as a 'Host Process' container. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). In addition, if HostProcess is true then HostNetwork must also be set to true.
type: boolean
runAsUserName:
description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.
type: string
tailscaleInitContainer:
description: Configuration for the proxy init container that enables forwarding.
type: object
properties:
resources:
description: Container resource requirements. By default Tailscale Kubernetes operator does not apply any resource requirements. The amount of resources required wil depend on the amount of resources the operator needs to parse, usage patterns and cluster size. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources
type: object
properties:
claims:
description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers."
type: array
items:
description: ResourceClaim references one entry in PodSpec.ResourceClaims.
type: object
required:
- name
properties:
name:
description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container.
type: string
x-kubernetes-list-map-keys:
- name
x-kubernetes-list-type: map
limits:
description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
type: object
additionalProperties:
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
anyOf:
- type: integer
- type: string
x-kubernetes-int-or-string: true
requests:
description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
type: object
additionalProperties:
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
anyOf:
- type: integer
- type: string
x-kubernetes-int-or-string: true
securityContext:
description: 'Container security context. Security context specified here will override the security context by the operator. By default the operator: - sets ''privileged: true'' for the init container - set NET_ADMIN capability for tailscale container for proxies that are created for Services or Connector. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context'
type: object
properties:
allowPrivilegeEscalation:
description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN Note that this field cannot be set when spec.os.name is windows.'
type: boolean
capabilities:
description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. Note that this field cannot be set when spec.os.name is windows.
type: object
properties:
add:
description: Added capabilities
type: array
items:
description: Capability represent POSIX capabilities type
type: string
drop:
description: Removed capabilities
type: array
items:
description: Capability represent POSIX capabilities type
type: string
privileged:
description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. Note that this field cannot be set when spec.os.name is windows.
type: boolean
procMount:
description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows.
type: string
readOnlyRootFilesystem:
description: Whether this container has a read-only root filesystem. Default is false. Note that this field cannot be set when spec.os.name is windows.
type: boolean
runAsGroup:
description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows.
type: integer
format: int64
runAsNonRoot:
description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.
type: boolean
runAsUser:
description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows.
type: integer
format: int64
seLinuxOptions:
description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows.
type: object
properties:
level:
description: Level is SELinux level label that applies to the container.
type: string
role:
description: Role is a SELinux role label that applies to the container.
type: string
type:
description: Type is a SELinux type label that applies to the container.
type: string
user:
description: User is a SELinux user label that applies to the container.
type: string
seccompProfile:
description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. Note that this field cannot be set when spec.os.name is windows.
type: object
required:
- type
properties:
localhostProfile:
description: localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must be set if type is "Localhost". Must NOT be set for any other type.
type: string
type:
description: "type indicates which kind of seccomp profile will be applied. Valid options are: \n Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied."
type: string
windowsOptions:
description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux.
type: object
properties:
gmsaCredentialSpec:
description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.
type: string
gmsaCredentialSpecName:
description: GMSACredentialSpecName is the name of the GMSA credential spec to use.
type: string
hostProcess:
description: HostProcess determines if a container should be run as a 'Host Process' container. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). In addition, if HostProcess is true then HostNetwork must also be set to true.
type: boolean
runAsUserName:
description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.
type: string
tolerations:
description: Proxy Pod's tolerations. By default Tailscale Kubernetes operator does not apply any tolerations. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling
type: array
items:
description: The pod this Toleration is attached to tolerates any taint that matches the triple <key,value,effect> using the matching operator <operator>.
type: object
properties:
effect:
description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
type: string
key:
description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.
type: string
operator:
description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.
type: string
tolerationSeconds:
description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.
type: integer
format: int64
value:
description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.
type: string
status:
type: object
properties:
conditions:
description: List of status conditions to indicate the status of the ProxyClass. Known condition types are `ProxyClassReady`.
type: array
items:
description: ConnectorCondition contains condition information for a Connector.
type: object
required:
- status
- type
properties:
lastTransitionTime:
description: LastTransitionTime is the timestamp corresponding to the last status change of this condition.
type: string
format: date-time
message:
description: Message is a human readable description of the details of the last transition, complementing reason.
type: string
observedGeneration:
description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Connector.
type: integer
format: int64
reason:
description: Reason is a brief machine readable explanation for the condition's last transition.
type: string
status:
description: Status of the condition, one of ('True', 'False', 'Unknown').
type: string
type:
description: Type of the condition, known values are (`SubnetRouterReady`).
type: string
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
served: true
storage: true
subresources:
status: {}

15
cmd/k8s-operator/deploy/examples/proxyclass.yaml Normal file
View File

@ -0,0 +1,15 @@
apiVersion: tailscale.com/v1alpha1
kind: ProxyClass
metadata:
name: prod
spec:
statefulSet:
annotations:
platform-component: infra
pod:
labels:
team: eng
nodeSelector:
beta.kubernetes.io/os: "linux"
imagePullSecrets:
- name: "foo"

500
cmd/k8s-operator/deploy/manifests/operator.yaml
View File

@ -79,6 +79,9 @@ spec:
description: Hostname is the tailnet hostname that should be assigned to the Connector node. If unset, hostname defaults to <connector name>-connector. Hostname can contain lower case letters, numbers and dashes, it must not start or end with a dash and must be between 2 and 63 characters long.
pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$
type: string
proxyClass:
description: ProxyClass is the name of the ProxyClass custom resource that contains configuration options that should be applied to the resources created for this Connector. If unset, the operator will create resources with the default configuration.
type: string
subnetRouter:
description: SubnetRouter defines subnet routes that the Connector node should expose to tailnet. If unset, none are exposed. https://tailscale.com/kb/1019/subnets/
properties:
@ -153,6 +156,501 @@ spec:
subresources:
status: {}
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.13.0
name: proxyclasses.tailscale.com
spec:
group: tailscale.com
names:
kind: ProxyClass
listKind: ProxyClassList
plural: proxyclasses
singular: proxyclass
scope: Cluster
versions:
- additionalPrinterColumns:
- description: Status of the ProxyClass.
jsonPath: .status.conditions[?(@.type == "ProxyClassReady")].reason
name: Status
type: string
name: v1alpha1
schema:
openAPIV3Schema:
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
kind:
description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
type: string
metadata:
type: object
spec:
properties:
statefulSet:
description: Proxy's StatefulSet spec.
properties:
annotations:
additionalProperties:
type: string
description: Annotations that will be added to the StatefulSet created for the proxy. Any Annotations specified here will be merged with the default annotations applied to the StatefulSet by the Tailscale Kubernetes operator as well as any other annotations that might have been applied by other actors. Annotations must be valid Kubernetes annotations. https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set
type: object
labels:
additionalProperties:
type: string
description: Labels that will be added to the StatefulSet created for the proxy. Any labels specified here will be merged with the default labels applied to the StatefulSet by the Tailscale Kubernetes operator as well as any other labels that might have been applied by other actors. Label keys and values must be valid Kubernetes label keys and values. https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
type: object
pod:
description: Configuration for the proxy Pod.
properties:
annotations:
additionalProperties:
type: string
description: Annotations that will be added to the proxy Pod. Any annotations specified here will be merged with the default annotations applied to the Pod by the Tailscale Kubernetes operator. Annotations must be valid Kubernetes annotations. https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set
type: object
imagePullSecrets:
description: Proxy Pod's image pull Secrets. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec
items:
description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.
properties:
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
type: string
type: object
x-kubernetes-map-type: atomic
type: array
labels:
additionalProperties:
type: string
description: Labels that will be added to the proxy Pod. Any labels specified here will be merged with the default labels applied to the Pod by the Tailscale Kubernetes operator. Label keys and values must be valid Kubernetes label keys and values. https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
type: object
nodeName:
description: Proxy Pod's node name. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling
type: string
nodeSelector:
additionalProperties:
type: string
description: Proxy Pod's node selector. By default Tailscale Kubernetes operator does not apply any node selector. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling
type: object
securityContext:
description: Proxy Pod's security context. By default Tailscale Kubernetes operator does not apply any Pod security context. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context-2
properties:
fsGroup:
description: "A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: \n 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \n If unset, the Kubelet will not modify the ownership and permissions of any volume. Note that this field cannot be set when spec.os.name is windows."
format: int64
type: integer
fsGroupChangePolicy:
description: 'fsGroupChangePolicy defines behavior of changing ownership and permission of the volume before being exposed inside Pod. This field will only apply to volume types which support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. Note that this field cannot be set when spec.os.name is windows.'
type: string
runAsGroup:
description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows.
format: int64
type: integer
runAsNonRoot:
description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.
type: boolean
runAsUser:
description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows.
format: int64
type: integer
seLinuxOptions:
description: The SELinux context to be applied to all containers. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows.
properties:
level:
description: Level is SELinux level label that applies to the container.
type: string
role:
description: Role is a SELinux role label that applies to the container.
type: string
type:
description: Type is a SELinux type label that applies to the container.
type: string
user:
description: User is a SELinux user label that applies to the container.
type: string
type: object
seccompProfile:
description: The seccomp options to use by the containers in this pod. Note that this field cannot be set when spec.os.name is windows.
properties:
localhostProfile:
description: localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must be set if type is "Localhost". Must NOT be set for any other type.
type: string
type:
description: "type indicates which kind of seccomp profile will be applied. Valid options are: \n Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied."
type: string
required:
- type
type: object
supplementalGroups:
description: A list of groups applied to the first process run in each container, in addition to the container's primary GID, the fsGroup (if specified), and group memberships defined in the container image for the uid of the container process. If unspecified, no additional groups are added to any container. Note that group memberships defined in the container image for the uid of the container process are still effective, even if they are not included in this list. Note that this field cannot be set when spec.os.name is windows.
items:
format: int64
type: integer
type: array
sysctls:
description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch. Note that this field cannot be set when spec.os.name is windows.
items:
description: Sysctl defines a kernel parameter to be set
properties:
name:
description: Name of a property to set
type: string
value:
description: Value of a property to set
type: string
required:
- name
- value
type: object
type: array
windowsOptions:
description: The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux.
properties:
gmsaCredentialSpec:
description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.
type: string
gmsaCredentialSpecName:
description: GMSACredentialSpecName is the name of the GMSA credential spec to use.
type: string
hostProcess:
description: HostProcess determines if a container should be run as a 'Host Process' container. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). In addition, if HostProcess is true then HostNetwork must also be set to true.
type: boolean
runAsUserName:
description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.
type: string
type: object
type: object
tailscaleContainer:
description: Configuration for the proxy container running tailscale.
properties:
resources:
description: Container resource requirements. By default Tailscale Kubernetes operator does not apply any resource requirements. The amount of resources required wil depend on the amount of resources the operator needs to parse, usage patterns and cluster size. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources
properties:
claims:
description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers."
items:
description: ResourceClaim references one entry in PodSpec.ResourceClaims.
properties:
name:
description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container.
type: string
required:
- name
type: object
type: array
x-kubernetes-list-map-keys:
- name
x-kubernetes-list-type: map
limits:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
type: object
requests:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
type: object
type: object
securityContext:
description: 'Container security context. Security context specified here will override the security context by the operator. By default the operator: - sets ''privileged: true'' for the init container - set NET_ADMIN capability for tailscale container for proxies that are created for Services or Connector. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context'
properties:
allowPrivilegeEscalation:
description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN Note that this field cannot be set when spec.os.name is windows.'
type: boolean
capabilities:
description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. Note that this field cannot be set when spec.os.name is windows.
properties:
add:
description: Added capabilities
items:
description: Capability represent POSIX capabilities type
type: string
type: array
drop:
description: Removed capabilities
items:
description: Capability represent POSIX capabilities type
type: string
type: array
type: object
privileged:
description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. Note that this field cannot be set when spec.os.name is windows.
type: boolean
procMount:
description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows.
type: string
readOnlyRootFilesystem:
description: Whether this container has a read-only root filesystem. Default is false. Note that this field cannot be set when spec.os.name is windows.
type: boolean
runAsGroup:
description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows.
format: int64
type: integer
runAsNonRoot:
description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.
type: boolean
runAsUser:
description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows.
format: int64
type: integer
seLinuxOptions:
description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows.
properties:
level:
description: Level is SELinux level label that applies to the container.
type: string
role:
description: Role is a SELinux role label that applies to the container.
type: string
type:
description: Type is a SELinux type label that applies to the container.
type: string
user:
description: User is a SELinux user label that applies to the container.
type: string
type: object
seccompProfile:
description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. Note that this field cannot be set when spec.os.name is windows.
properties:
localhostProfile:
description: localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must be set if type is "Localhost". Must NOT be set for any other type.
type: string
type:
description: "type indicates which kind of seccomp profile will be applied. Valid options are: \n Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied."
type: string
required:
- type
type: object
windowsOptions:
description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux.
properties:
gmsaCredentialSpec:
description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.
type: string
gmsaCredentialSpecName:
description: GMSACredentialSpecName is the name of the GMSA credential spec to use.
type: string
hostProcess:
description: HostProcess determines if a container should be run as a 'Host Process' container. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). In addition, if HostProcess is true then HostNetwork must also be set to true.
type: boolean
runAsUserName:
description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.
type: string
type: object
type: object
type: object
tailscaleInitContainer:
description: Configuration for the proxy init container that enables forwarding.
properties:
resources:
description: Container resource requirements. By default Tailscale Kubernetes operator does not apply any resource requirements. The amount of resources required wil depend on the amount of resources the operator needs to parse, usage patterns and cluster size. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources
properties:
claims:
description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers."
items:
description: ResourceClaim references one entry in PodSpec.ResourceClaims.
properties:
name:
description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container.
type: string
required:
- name
type: object
type: array
x-kubernetes-list-map-keys:
- name
x-kubernetes-list-type: map
limits:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
type: object
requests:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
type: object
type: object
securityContext:
description: 'Container security context. Security context specified here will override the security context by the operator. By default the operator: - sets ''privileged: true'' for the init container - set NET_ADMIN capability for tailscale container for proxies that are created for Services or Connector. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context'
properties:
allowPrivilegeEscalation:
description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN Note that this field cannot be set when spec.os.name is windows.'
type: boolean
capabilities:
description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. Note that this field cannot be set when spec.os.name is windows.
properties:
add:
description: Added capabilities
items:
description: Capability represent POSIX capabilities type
type: string
type: array
drop:
description: Removed capabilities
items:
description: Capability represent POSIX capabilities type
type: string
type: array
type: object
privileged:
description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. Note that this field cannot be set when spec.os.name is windows.
type: boolean
procMount:
description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows.
type: string
readOnlyRootFilesystem:
description: Whether this container has a read-only root filesystem. Default is false. Note that this field cannot be set when spec.os.name is windows.
type: boolean
runAsGroup:
description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows.
format: int64
type: integer
runAsNonRoot:
description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.
type: boolean
runAsUser:
description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows.
format: int64
type: integer
seLinuxOptions:
description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows.
properties:
level:
description: Level is SELinux level label that applies to the container.
type: string
role:
description: Role is a SELinux role label that applies to the container.
type: string
type:
description: Type is a SELinux type label that applies to the container.
type: string
user:
description: User is a SELinux user label that applies to the container.
type: string
type: object
seccompProfile:
description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. Note that this field cannot be set when spec.os.name is windows.
properties:
localhostProfile:
description: localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must be set if type is "Localhost". Must NOT be set for any other type.
type: string
type:
description: "type indicates which kind of seccomp profile will be applied. Valid options are: \n Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied."
type: string
required:
- type
type: object
windowsOptions:
description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux.
properties:
gmsaCredentialSpec:
description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.
type: string
gmsaCredentialSpecName:
description: GMSACredentialSpecName is the name of the GMSA credential spec to use.
type: string
hostProcess:
description: HostProcess determines if a container should be run as a 'Host Process' container. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). In addition, if HostProcess is true then HostNetwork must also be set to true.
type: boolean
runAsUserName:
description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.
type: string
type: object
type: object
type: object
tolerations:
description: Proxy Pod's tolerations. By default Tailscale Kubernetes operator does not apply any tolerations. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling
items:
description: The pod this Toleration is attached to tolerates any taint that matches the triple <key,value,effect> using the matching operator <operator>.
properties:
effect:
description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
type: string
key:
description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.
type: string
operator:
description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.
type: string
tolerationSeconds:
description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.
format: int64
type: integer
value:
description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.
type: string
type: object
type: array
type: object
type: object
required:
- statefulSet
type: object
status:
properties:
conditions:
description: List of status conditions to indicate the status of the ProxyClass. Known condition types are `ProxyClassReady`.
items:
description: ConnectorCondition contains condition information for a Connector.
properties:
lastTransitionTime:
description: LastTransitionTime is the timestamp corresponding to the last status change of this condition.
format: date-time
type: string
message:
description: Message is a human readable description of the details of the last transition, complementing reason.
type: string
observedGeneration:
description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Connector.
format: int64
type: integer
reason:
description: Reason is a brief machine readable explanation for the condition's last transition.
type: string
status:
description: Status of the condition, one of ('True', 'False', 'Unknown').
type: string
type:
description: Type of the condition, known values are (`SubnetRouterReady`).
type: string
required:
- status
- type
type: object
type: array
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
type: object
required:
- spec
type: object
served: true
storage: true
subresources:
status: {}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
@ -186,6 +684,8 @@ rules:
resources:
- connectors
- connectors/status
- proxyclasses
- proxyclasses/status
verbs:
- get
- list

30
cmd/k8s-operator/generate/main.go
View File

@ -20,9 +20,11 @@
const (
operatorDeploymentFilesPath = "cmd/k8s-operator/deploy"
crdPath = operatorDeploymentFilesPath + "/crds/tailscale.com_connectors.yaml"
connectorCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_connectors.yaml"
proxyClassCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_proxyclasses.yaml"
helmTemplatesPath = operatorDeploymentFilesPath + "/chart/templates"
crdTemplatePath = helmTemplatesPath + "/connectors.yaml"
connectorCRDHelmTemplatePath = helmTemplatesPath + "/connector.yaml"
proxyClassCRDHelmTemplatePath = helmTemplatesPath + "/proxyclass.yaml"
helmConditionalStart = "{{ if .Values.installCRDs -}}\n"
helmConditionalEnd = "{{- end -}}"
@ -44,9 +46,9 @@ func main() {
default:
log.Fatalf("unknown option %s, known options are 'staticmanifests', 'helmcrd'", os.Args[1])
}
log.Printf("Inserting CRD into the Helm templates")
log.Printf("Inserting CRDs Helm templates")
if err := generate(repoRoot); err != nil {
log.Fatalf("error adding Connector CRD to Helm templates: %v", err)
log.Fatalf("error adding CRDs to Helm templates: %v", err)
}
defer func() {
if err := cleanup(repoRoot); err != nil {
@ -106,8 +108,11 @@ func main() {
}
}
// generate places tailscale.com CRDs (currently Connector and ProxyClass) into
// the Helm chart templates behind .Values.installCRDs=true condition (true by
// default).
func generate(baseDir string) error {
log.Print("Placing Connector CRD into Helm templates..")
addCRDToHelm := func(crdPath, crdTemplatePath string) error {
chartBytes, err := os.ReadFile(filepath.Join(baseDir, crdPath))
if err != nil {
return fmt.Errorf("error reading CRD contents: %w", err)
@ -128,12 +133,23 @@ func generate(baseDir string) error {
return fmt.Errorf("error writing helm if-statement end: %w", err)
}
return nil
}
if err := addCRDToHelm(connectorCRDPath, connectorCRDHelmTemplatePath); err != nil {
return fmt.Errorf("error adding Connector CRD to Helm templates: %w", err)
}
if err := addCRDToHelm(proxyClassCRDPath, proxyClassCRDHelmTemplatePath); err != nil {
return fmt.Errorf("error adding ProxyClass CRD to Helm templates: %w", err)
}
return nil
}
func cleanup(baseDir string) error {
log.Print("Cleaning up CRD from Helm templates")
if err := os.Remove(filepath.Join(baseDir, crdTemplatePath)); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("error cleaning up CRD template: %w", err)
if err := os.Remove(filepath.Join(baseDir, connectorCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("error cleaning up Connector CRD template: %w", err)
}
if err := os.Remove(filepath.Join(baseDir, proxyClassCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("error cleaning up ProxyClass CRD template: %w", err)
}
return nil
}

14
cmd/k8s-operator/generate/main_test.go
View File

@ -42,7 +42,7 @@ func Test_generate(t *testing.T) {
t.Fatalf("Helm chart linter failed: %v", err)
}
// Test that default Helm install contains the CRD
// Test that default Helm install contains the Connector and ProxyClass CRDs.
installContentsWithCRD := bytes.NewBuffer([]byte{})
helmTemplateWithCRDCmd := exec.Command(helmCLIPath, "template", helmPackagePath)
helmTemplateWithCRDCmd.Stderr = os.Stderr
@ -51,10 +51,13 @@ func Test_generate(t *testing.T) {
t.Fatalf("templating Helm chart with CRDs failed: %v", err)
}
if !strings.Contains(installContentsWithCRD.String(), "name: connectors.tailscale.com") {
t.Errorf("CRD not found in default chart install")
t.Errorf("Connector CRD not found in default chart install")
}
if !strings.Contains(installContentsWithCRD.String(), "name: proxyclasses.tailscale.com") {
t.Errorf("ProxyClass CRD not found in default chart install")
}
// Test that CRD can be excluded from Helm chart install
// Test that CRDs can be excluded from Helm chart install
installContentsWithoutCRD := bytes.NewBuffer([]byte{})
helmTemplateWithoutCRDCmd := exec.Command(helmCLIPath, "template", helmPackagePath, "--set", "installCRDs=false")
helmTemplateWithoutCRDCmd.Stderr = os.Stderr
@ -63,6 +66,9 @@ func Test_generate(t *testing.T) {
t.Fatalf("templating Helm chart without CRDs failed: %v", err)
}
if strings.Contains(installContentsWithoutCRD.String(), "name: connectors.tailscale.com") {
t.Errorf("CRD found in chart install that should not contain a CRD")
t.Errorf("Connector CRD found in chart install that should not contain a CRD")
}
if strings.Contains(installContentsWithoutCRD.String(), "name: connectors.tailscale.com") {
t.Errorf("ProxyClass CRD found in chart install that should not contain a CRD")
}
}

12
cmd/k8s-operator/ingress.go
View File

@ -132,6 +132,17 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
return fmt.Errorf("failed to add finalizer: %w", err)
}
}
proxyClass := proxyClassForObject(ing)
if proxyClass != "" {
if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
return fmt.Errorf("error verifying ProxyClass for Ingress: %w", err)
} else if !ready {
logger.Infof("ProxyClass %s specified for the Ingress, but is not (yet) Ready, waiting..", proxyClass)
return nil
}
}
a.mu.Lock()
a.managedIngresses.Add(ing.UID)
gaugeIngressResources.Set(int64(a.managedIngresses.Len()))
@ -253,6 +264,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
ServeConfig: sc,
Tags: tags,
ChildResourceLabels: crl,
ProxyClass: proxyClass,
}
if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" {

135
cmd/k8s-operator/ingress_test.go
View File

@ -16,6 +16,7 @@
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"tailscale.com/ipn"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/types/ptr"
"tailscale.com/util/mak"
)
@ -101,7 +102,7 @@ func TestTailscaleIngress(t *testing.T) {
expectEqual(t, fc, expectedSecret(t, opts))
expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"))
expectEqual(t, fc, expectedSTSUserspace(opts))
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts))
// 2. Ingress status gets updated with ingress proxy's MagicDNS name
// once that becomes available.
@ -125,7 +126,7 @@ func TestTailscaleIngress(t *testing.T) {
})
opts.shouldEnableForwardingClusterTrafficViaIngress = true
expectReconciled(t, ingR, "default", "test")
expectEqual(t, fc, expectedSTS(opts))
expectEqual(t, fc, expectedSTS(t, fc, opts))
// 4. Resources get cleaned up when Ingress class is unset
mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) {
@ -137,3 +138,133 @@ func TestTailscaleIngress(t *testing.T) {
expectMissing[corev1.Service](t, fc, "operator-ns", shortName)
expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
}
func TestTailscaleIngressWithProxyClass(t *testing.T) {
// Setup
pc := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"},
Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{
Labels: map[string]string{"foo": "bar"},
Annotations: map[string]string{"bar.io/foo": "some-val"},
Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
}
tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pc, tsIngressClass).
WithStatusSubresource(pc).
Build()
ft := &fakeTSClient{}
fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
ingR := &IngressReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
tsClient: ft,
tsnetServer: fakeTsnetServer,
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
}
// 1. Ingress is created with no ProxyClass specified, default proxy
// resources get configured.
ing := &networkingv1.Ingress{
TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: types.UID("1234-UID"),
},
Spec: networkingv1.IngressSpec{
IngressClassName: ptr.To("tailscale"),
DefaultBackend: &networkingv1.IngressBackend{
Service: &networkingv1.IngressServiceBackend{
Name: "test",
Port: networkingv1.ServiceBackendPort{
Number: 8080,
},
},
},
TLS: []networkingv1.IngressTLS{
{Hosts: []string{"default-test"}},
},
},
}
mustCreate(t, fc, ing)
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
},
Spec: corev1.ServiceSpec{
ClusterIP: "1.2.3.4",
Ports: []corev1.ServicePort{{
Port: 8080,
Name: "http"},
},
},
})
expectReconciled(t, ingR, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "ingress")
opts := configOpts{
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "ingress",
hostname: "default-test",
}
serveConfig := &ipn.ServeConfig{
TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}},
}
opts.serveConfig = serveConfig
expectEqual(t, fc, expectedSecret(t, opts))
expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"))
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts))
// 2. Ingress is updated to specify a ProxyClass, ProxyClass is not yet
// ready, so proxy resource configuration does not change.
mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) {
mak.Set(&ing.ObjectMeta.Labels, LabelProxyClass, "custom-metadata")
})
expectReconciled(t, ingR, "default", "test")
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts))
// 3. ProxyClass is set to Ready by proxy-class reconciler. Ingress get
// reconciled and configuration from the ProxyClass is applied to the
// created proxy resources.
mustUpdateStatus(t, fc, "", "custom-metadata", func(pc *tsapi.ProxyClass) {
pc.Status = tsapi.ProxyClassStatus{
Conditions: []tsapi.ConnectorCondition{{
Status: metav1.ConditionTrue,
Type: tsapi.ProxyClassready,
ObservedGeneration: pc.Generation,
}}}
})
expectReconciled(t, ingR, "default", "test")
opts.proxyClass = pc.Name
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts))
// 4. tailscale.com/proxy-class label is removed from the Ingress, the
// Ingress gets reconciled and the custom ProxyClass configuration is
// removed from the proxy resources.
mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) {
delete(ing.ObjectMeta.Labels, LabelProxyClass)
})
expectReconciled(t, ingR, "default", "test")
opts.proxyClass = ""
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts))
}

91
cmd/k8s-operator/operator.go
View File

@ -233,6 +233,9 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
svcFilter := handler.EnqueueRequestsFromMapFunc(serviceHandler)
svcChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("svc"))
// If a ProxyClassChanges, enqueue all Services labeled with that
// ProxyClass's name.
proxyClassFilterForSvc := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForSvc(mgr.GetClient(), startlog))
eventRecorder := mgr.GetEventRecorderFor("tailscale-operator")
ssr := &tailscaleSTSReconciler{
@ -251,6 +254,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
Watches(&corev1.Service{}, svcFilter).
Watches(&appsv1.StatefulSet{}, svcChildFilter).
Watches(&corev1.Secret{}, svcChildFilter).
Watches(&tsapi.ProxyClass{}, proxyClassFilterForSvc).
Complete(&ServiceReconciler{
ssr: ssr,
Client: mgr.GetClient(),
@ -259,15 +263,19 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
recorder: eventRecorder,
})
if err != nil {
startlog.Fatalf("could not create controller: %v", err)
startlog.Fatalf("could not create service reconciler: %v", err)
}
ingressChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("ingress"))
// If a ProxyClassChanges, enqueue all Ingresses labeled with that
// ProxyClass's name.
proxyClassFilterForIngress := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForIngress(mgr.GetClient(), startlog))
err = builder.
ControllerManagedBy(mgr).
For(&networkingv1.Ingress{}).
Watches(&appsv1.StatefulSet{}, ingressChildFilter).
Watches(&corev1.Secret{}, ingressChildFilter).
Watches(&corev1.Service{}, ingressChildFilter).
Watches(&tsapi.ProxyClass{}, proxyClassFilterForIngress).
Complete(&IngressReconciler{
ssr: ssr,
recorder: eventRecorder,
@ -275,14 +283,18 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
logger: zlog.Named("ingress-reconciler"),
})
if err != nil {
startlog.Fatalf("could not create controller: %v", err)
startlog.Fatalf("could not create ingress reconciler: %v", err)
}
connectorFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("connector"))
// If a ProxyClassChanges, enqueue all Connectors that have
// .spec.proxyClass set to the name of this ProxyClass.
proxyClassFilterForConnector := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForConnector(mgr.GetClient(), startlog))
err = builder.ControllerManagedBy(mgr).
For(&tsapi.Connector{}).
Watches(&appsv1.StatefulSet{}, connectorFilter).
Watches(&corev1.Secret{}, connectorFilter).
Watches(&tsapi.ProxyClass{}, proxyClassFilterForConnector).
Complete(&ConnectorReconciler{
ssr: ssr,
recorder: eventRecorder,
@ -293,6 +305,17 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
if err != nil {
startlog.Fatal("could not create connector reconciler: %v", err)
}
err = builder.ControllerManagedBy(mgr).
For(&tsapi.ProxyClass{}).
Complete(&ProxyClassReconciler{
Client: mgr.GetClient(),
recorder: eventRecorder,
logger: zlog.Named("proxyclass-reconciler"),
clock: tstime.DefaultClock{},
})
if err != nil {
startlog.Fatal("could not create proxyclass reconciler: %v", err)
}
startlog.Infof("Startup complete, operator running, version: %s", version.Long())
if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
startlog.Fatalf("could not start manager: %v", err)
@ -321,6 +344,7 @@ func parentFromObjectLabels(o client.Object) types.NamespacedName {
Name: ls[LabelParentName],
}
}
func managedResourceHandlerForType(typ string) handler.MapFunc {
return func(_ context.Context, o client.Object) []reconcile.Request {
if !isManagedByType(o, typ) {
@ -330,14 +354,75 @@ func managedResourceHandlerForType(typ string) handler.MapFunc {
{NamespacedName: parentFromObjectLabels(o)},
}
}
}
// proxyClassHandlerForSvc returns a handler that, for a given ProxyClass,
// returns a list of reconcile requests for all Services labeled with
// tailscale.com/proxy-class: <proxy class name>.
func proxyClassHandlerForSvc(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
svcList := new(corev1.ServiceList)
labels := map[string]string{
LabelProxyClass: o.GetName(),
}
if err := cl.List(ctx, svcList, client.MatchingLabels(labels)); err != nil {
logger.Debugf("error listing Services for ProxyClass: %v", err)
return nil
}
reqs := make([]reconcile.Request, 0)
for _, svc := range svcList.Items {
reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&svc)})
}
return reqs
}
}
// proxyClassHandlerForIngress returns a handler that, for a given ProxyClass,
// returns a list of reconcile requests for all Ingresses labeled with
// tailscale.com/proxy-class: <proxy class name>.
func proxyClassHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
ingList := new(networkingv1.IngressList)
labels := map[string]string{
LabelProxyClass: o.GetName(),
}
if err := cl.List(ctx, ingList, client.MatchingLabels(labels)); err != nil {
logger.Debugf("error listing Ingresses for ProxyClass: %v", err)
return nil
}
reqs := make([]reconcile.Request, 0)
for _, ing := range ingList.Items {
reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)})
}
return reqs
}
}
// proxyClassHandlerForConnector returns a handler that, for a given ProxyClass,
// returns a list of reconcile requests for all Connectors that have
// .spec.proxyClass set.
func proxyClassHandlerForConnector(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
connList := new(tsapi.ConnectorList)
if err := cl.List(ctx, connList); err != nil {
logger.Debugf("error listing Connectors for ProxyClass: %v", err)
return nil
}
reqs := make([]reconcile.Request, 0)
proxyClassName := o.GetName()
for _, conn := range connList.Items {
if conn.Spec.ProxyClass == proxyClassName {
reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&conn)})
}
}
return reqs
}
}
func serviceHandler(_ context.Context, o client.Object) []reconcile.Request {
if isManagedByType(o, "svc") {
// If this is a Service managed by a Service we want to enqueue its parent
return []reconcile.Request{{NamespacedName: parentFromObjectLabels(o)}}
}
if isManagedResource(o) {
// If this is a Servce managed by a resource that is not a Service, we leave it alone

127
cmd/k8s-operator/operator_test.go
View File

@ -15,7 +15,9 @@
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/types/ptr"
"tailscale.com/util/mak"
)
func TestLoadBalancerClass(t *testing.T) {
@ -69,7 +71,7 @@ func TestLoadBalancerClass(t *testing.T) {
expectEqual(t, fc, expectedSecret(t, opts))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(opts))
expectEqual(t, fc, expectedSTS(t, fc, opts))
// Normally the Tailscale proxy pod would come up here and write its info
// into the secret. Simulate that, then verify reconcile again and verify
@ -210,7 +212,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
expectEqual(t, fc, expectedSecret(t, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(o))
expectEqual(t, fc, expectedSTS(t, fc, o))
want := &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
@ -234,7 +236,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
expectEqual(t, fc, want)
expectEqual(t, fc, expectedSecret(t, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(o))
expectEqual(t, fc, expectedSTS(t, fc, o))
// Change the tailscale-target-fqdn annotation which should update the
// StatefulSet
@ -320,7 +322,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
expectEqual(t, fc, expectedSecret(t, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(o))
expectEqual(t, fc, expectedSTS(t, fc, o))
want := &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
@ -344,7 +346,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
expectEqual(t, fc, want)
expectEqual(t, fc, expectedSecret(t, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(o))
expectEqual(t, fc, expectedSTS(t, fc, o))
// Change the tailscale-target-ip annotation which should update the
// StatefulSet
@ -427,7 +429,7 @@ func TestAnnotations(t *testing.T) {
expectEqual(t, fc, expectedSecret(t, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(o))
expectEqual(t, fc, expectedSTS(t, fc, o))
want := &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
@ -535,7 +537,7 @@ func TestAnnotationIntoLB(t *testing.T) {
expectEqual(t, fc, expectedSecret(t, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(o))
expectEqual(t, fc, expectedSTS(t, fc, o))
// Normally the Tailscale proxy pod would come up here and write its info
// into the secret. Simulate that, since it would have normally happened at
@ -580,7 +582,7 @@ func TestAnnotationIntoLB(t *testing.T) {
expectReconciled(t, sr, "default", "test")
// None of the proxy machinery should have changed...
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(o))
expectEqual(t, fc, expectedSTS(t, fc, o))
// ... but the service should have a LoadBalancer status.
want = &corev1.Service{
@ -666,7 +668,7 @@ func TestLBIntoAnnotation(t *testing.T) {
expectEqual(t, fc, expectedSecret(t, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(o))
expectEqual(t, fc, expectedSTS(t, fc, o))
// Normally the Tailscale proxy pod would come up here and write its info
// into the secret. Simulate that, then verify reconcile again and verify
@ -729,7 +731,7 @@ func TestLBIntoAnnotation(t *testing.T) {
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(o))
expectEqual(t, fc, expectedSTS(t, fc, o))
want = &corev1.Service{
TypeMeta: metav1.TypeMeta{
@ -807,7 +809,7 @@ func TestCustomHostname(t *testing.T) {
expectEqual(t, fc, expectedSecret(t, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(o))
expectEqual(t, fc, expectedSTS(t, fc, o))
want := &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
@ -920,7 +922,104 @@ func TestCustomPriorityClassName(t *testing.T) {
clusterTargetIP: "10.20.30.40",
}
expectEqual(t, fc, expectedSTS(o))
expectEqual(t, fc, expectedSTS(t, fc, o))
}
func TestProxyClassForService(t *testing.T) {
// Setup
pc := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"},
Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{
Labels: map[string]string{"foo": "bar"},
Annotations: map[string]string{"bar.io/foo": "some-val"},
Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pc).
WithStatusSubresource(pc).
Build()
ft := &fakeTSClient{}
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
tsClient: ft,
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
}
// 1. A new tailscale LoadBalancer Service is created without any
// ProxyClass. Resources get created for it as usual.
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: types.UID("1234-UID"),
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
LoadBalancerClass: ptr.To("tailscale"),
},
})
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
opts := configOpts{
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "svc",
hostname: "default-test",
clusterTargetIP: "10.20.30.40",
}
expectEqual(t, fc, expectedSecret(t, opts))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, opts))
// 2. The Service gets updated with tailscale.com/proxy-class label
// pointing at the 'custom-metadata' ProxyClass. The ProxyClass is not
// yet ready, so no changes are actually applied to the proxy resources.
mustUpdate(t, fc, "default", "test", func(svc *corev1.Service) {
mak.Set(&svc.Labels, LabelProxyClass, "custom-metadata")
})
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts))
// 3. ProxyClass is set to Ready, the Service gets reconciled by the
// services-reconciler and the customization from the ProxyClass is
// applied to the proxy resources.
mustUpdateStatus(t, fc, "", "custom-metadata", func(pc *tsapi.ProxyClass) {
pc.Status = tsapi.ProxyClassStatus{
Conditions: []tsapi.ConnectorCondition{{
Status: metav1.ConditionTrue,
Type: tsapi.ProxyClassready,
ObservedGeneration: pc.Generation,
}}}
})
opts.proxyClass = pc.Name
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts))
// 4. tailscale.com/proxy-class label is removed from the Service, the
// configuration from the ProxyClass is removed from the cluster
// resources.
mustUpdate(t, fc, "default", "test", func(svc *corev1.Service) {
delete(svc.Labels, LabelProxyClass)
})
opts.proxyClass = ""
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts))
}
func TestDefaultLoadBalancer(t *testing.T) {
@ -973,7 +1072,7 @@ func TestDefaultLoadBalancer(t *testing.T) {
hostname: "default-test",
clusterTargetIP: "10.20.30.40",
}
expectEqual(t, fc, expectedSTS(o))
expectEqual(t, fc, expectedSTS(t, fc, o))
}
func TestProxyFirewallMode(t *testing.T) {
@ -1026,7 +1125,7 @@ func TestProxyFirewallMode(t *testing.T) {
firewallMode: "nftables",
clusterTargetIP: "10.20.30.40",
}
expectEqual(t, fc, expectedSTS(o))
expectEqual(t, fc, expectedSTS(t, fc, o))
}

108
cmd/k8s-operator/proxyclass.go Normal file
View File

@ -0,0 +1,108 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
// tailscale-operator provides a way to expose services running in a Kubernetes
// cluster to your Tailnet.
package main
import (
"context"
"fmt"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
apivalidation "k8s.io/apimachinery/pkg/api/validation"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
metavalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/tstime"
)
const (
reasonProxyClassInvalid = "ProxyClassInvalid"
reasonProxyClassValid = "ProxyClassValid"
messageProxyClassInvalid = "ProxyClass is not valid: %v"
)
type ProxyClassReconciler struct {
client.Client
recorder record.EventRecorder
logger *zap.SugaredLogger
clock tstime.Clock
}
func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
logger := pcr.logger.With("ProxyClass", req.Name)
logger.Debugf("starting reconcile")
defer logger.Debugf("reconcile finished")
pc := new(tsapi.ProxyClass)
err = pcr.Get(ctx, req.NamespacedName, pc)
if apierrors.IsNotFound(err) {
logger.Debugf("ProxyClass not found, assuming it was deleted")
return reconcile.Result{}, nil
} else if err != nil {
return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com ProxyClass: %w", err)
}
if !pc.DeletionTimestamp.IsZero() {
logger.Debugf("ProxyClass is being deleted, do nothing")
return reconcile.Result{}, nil
}
oldPCStatus := pc.Status.DeepCopy()
if errs := pcr.validate(pc); errs != nil {
msg := fmt.Sprintf(messageProxyClassInvalid, errs.ToAggregate().Error())
pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonProxyClassInvalid, msg)
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, pc.Generation, pcr.clock, logger)
} else {
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, pc.Generation, pcr.clock, logger)
}
if !apiequality.Semantic.DeepEqual(oldPCStatus, pc.Status) {
if err := pcr.Client.Status().Update(ctx, pc); err != nil {
logger.Errorf("error updating ProxyClass status: %v", err)
return reconcile.Result{}, err
}
}
return reconcile.Result{}, nil
}
func (a *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations field.ErrorList) {
if sts := pc.Spec.StatefulSet; sts != nil {
if len(sts.Labels) > 0 {
if errs := metavalidation.ValidateLabels(sts.Labels, field.NewPath(".spec.statefulSet.labels")); errs != nil {
violations = append(violations, errs...)
}
}
if len(sts.Annotations) > 0 {
if errs := apivalidation.ValidateAnnotations(sts.Annotations, field.NewPath(".spec.statefulSet.annotations")); errs != nil {
violations = append(violations, errs...)
}
}
if pod := sts.Pod; pod != nil {
if len(pod.Labels) > 0 {
if errs := metavalidation.ValidateLabels(pod.Labels, field.NewPath(".spec.statefulSet.pod.labels")); errs != nil {
violations = append(violations, errs...)
}
}
if len(pod.Annotations) > 0 {
if errs := apivalidation.ValidateAnnotations(pod.Annotations, field.NewPath(".spec.statefulSet.pod.annotations")); errs != nil {
violations = append(violations, errs...)
}
}
}
}
// We do not validate embedded fields (security context, resource
// requirements etc) as we inherit upstream validation for those fields.
// Invalid values would get rejected by upstream validations at apply
// time.
return violations
}

83
cmd/k8s-operator/proxyclass_test.go Normal file
View File

@ -0,0 +1,83 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
// tailscale-operator provides a way to expose services running in a Kubernetes
// cluster to your Tailnet.
package main
import (
"testing"
"time"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/tstest"
)
func TestProxyClass(t *testing.T) {
pc := &tsapi.ProxyClass{
TypeMeta: metav1.TypeMeta{Kind: "ProxyClass", APIVersion: "tailscale.com/v1alpha1"},
ObjectMeta: metav1.ObjectMeta{
Name: "test",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: types.UID("1234-UID"),
},
Spec: tsapi.ProxyClassSpec{
StatefulSet: &tsapi.StatefulSet{
Labels: map[string]string{"foo": "bar", "xyz1234": "abc567"},
Annotations: map[string]string{"foo.io/bar": "{'key': 'val1232'}"},
Pod: &tsapi.Pod{
Labels: map[string]string{"foo": "bar", "xyz1234": "abc567"},
Annotations: map[string]string{"foo.io/bar": "{'key': 'val1232'}"},
},
},
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pc).
WithStatusSubresource(pc).
Build()
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
cl := tstest.NewClock(tstest.ClockOpts{})
pcr := &ProxyClassReconciler{
Client: fc,
logger: zl.Sugar(),
clock: cl,
recorder: record.NewFakeRecorder(1),
}
expectReconciled(t, pcr, "", "test")
// 1. A valid ProxyClass resource gets its status updated to Ready.
pc.Status.Conditions = append(pc.Status.Conditions, tsapi.ConnectorCondition{
Type: tsapi.ProxyClassready,
Status: metav1.ConditionTrue,
Reason: reasonProxyClassValid,
Message: reasonProxyClassValid,
LastTransitionTime: &metav1.Time{Time: cl.Now().Truncate(time.Second)},
})
expectEqual(t, fc, pc)
// 2. An invalid ProxyClass resource gets its status updated to Invalid.
pc.Spec.StatefulSet.Labels["foo"] = "?!someVal"
mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
proxyClass.Spec.StatefulSet.Labels = pc.Spec.StatefulSet.Labels
})
expectReconciled(t, pcr, "", "test")
msg := `ProxyClass is not valid: .spec.statefulSet.labels: Invalid value: "?!someVal": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue', or 'my_value', or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')`
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
expectEqual(t, fc, pc)
}

150
cmd/k8s-operator/sts.go
View File

@ -14,6 +14,7 @@
"fmt"
"net/http"
"os"
"slices"
"strings"
"go.uber.org/zap"
@ -22,11 +23,14 @@
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/storage/names"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/yaml"
"tailscale.com/client/tailscale"
"tailscale.com/ipn"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/net/netutil"
"tailscale.com/tailcfg"
"tailscale.com/types/opt"
@ -35,11 +39,20 @@
)
const (
// Labels that the operator sets on StatefulSets and Pods. If you add a
// new label here, do also add it to tailscaleManagedLabels var to
// ensure that it does not get overwritten by ProxyClass configuration.
LabelManaged = "tailscale.com/managed"
LabelParentType = "tailscale.com/parent-resource-type"
LabelParentName = "tailscale.com/parent-resource"
LabelParentNamespace = "tailscale.com/parent-resource-ns"
// LabelProxyClass can be set by users on Connectors, tailscale
// Ingresses and Services that define cluster ingress or cluster egress,
// to specify that configuration in this ProxyClass should be applied to
// resources created for the Connector, Ingress or Service.
LabelProxyClass = "tailscale.com/proxy-class"
FinalizerName = "tailscale.com/finalizer"
// Annotations settable by users on services.
@ -68,7 +81,10 @@
AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy = "tailscale.com/experimental-forward-cluster-traffic-via-ingress"
// Annotations set by the operator on pods to trigger restarts when the
// hostname, IP, FQDN or tailscaled config changes.
// hostname, IP, FQDN or tailscaled config changes. If you add a new
// annotation here, also add it to tailscaleManagedAnnotations var to
// ensure that it does not get removed when a ProxyClass configuration
// is applied.
podAnnotationLastSetClusterIP = "tailscale.com/operator-last-set-cluster-ip"
podAnnotationLastSetHostname = "tailscale.com/operator-last-set-hostname"
podAnnotationLastSetTailnetTargetIP = "tailscale.com/operator-last-set-ts-tailnet-target-ip"
@ -81,6 +97,13 @@
tailscaledConfigKey = "tailscaled"
)
var (
// tailscaleManagedLabels are label keys that tailscale operator sets on StatefulSets and Pods.
tailscaleManagedLabels = []string{LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"}
// tailscaleManagedAnnotations are annotation keys that tailscale operator sets on StatefulSets and Pods.
tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetHostname, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN, podAnnotationLastSetConfigFileHash}
)
type tailscaleSTSConfig struct {
ParentResourceName string
ParentResourceUID string
@ -102,6 +125,8 @@ type tailscaleSTSConfig struct {
// Connector specifies a configuration of a Connector instance if that's
// what this StatefulSet should be created for.
Connector *connector
ProxyClass string
}
type connector struct {
@ -398,10 +423,10 @@ func (a *tailscaleSTSReconciler) newAuthKey(ctx context.Context, tags []string)
var userspaceProxyYaml []byte
func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string) (*appsv1.StatefulSet, error) {
var ss appsv1.StatefulSet
ss := new(appsv1.StatefulSet)
if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding
if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil {
return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err)
return nil, fmt.Errorf("failed to unmarshal userspace proxy spec: %v", err)
}
} else {
if err := yaml.Unmarshal(proxyYaml, &ss); err != nil {
@ -415,12 +440,25 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
}
}
}
container := &ss.Spec.Template.Spec.Containers[0]
pod := &ss.Spec.Template
container := &pod.Spec.Containers[0]
proxyClass := new(tsapi.ProxyClass)
if sts.ProxyClass != "" {
if err := a.Get(ctx, types.NamespacedName{Name: sts.ProxyClass}, proxyClass); err != nil {
return nil, fmt.Errorf("failed to get ProxyClass: %w", err)
}
if !tsoperator.ProxyClassIsReady(proxyClass) {
logger.Infof("ProxyClass %s specified for the proxy, but it is not (yet) in a ready state, waiting..")
return nil, nil
}
}
container.Image = a.proxyImage
ss.ObjectMeta = metav1.ObjectMeta{
Name: headlessSvc.Name,
Namespace: a.operatorNamespace,
Labels: sts.ChildResourceLabels,
}
for key, val := range sts.ChildResourceLabels {
mak.Set(&ss.ObjectMeta.Labels, key, val)
}
ss.Spec.ServiceName = headlessSvc.Name
ss.Spec.Selector = &metav1.LabelSelector{
@ -428,9 +466,9 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
"app": sts.ParentResourceUID,
},
}
mak.Set(&ss.Spec.Template.Labels, "app", sts.ParentResourceUID)
mak.Set(&pod.Labels, "app", sts.ParentResourceUID)
for key, val := range sts.ChildResourceLabels {
ss.Spec.Template.Labels[key] = val // sync StatefulSet labels to Pod to make it easier for users to select the Pod
pod.Labels[key] = val // sync StatefulSet labels to Pod to make it easier for users to select the Pod
}
// Generic containerboot configuration options.
@ -455,12 +493,12 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
// it is passed via an environment variable. So we need to restart the
// container when the value changes. We do this by adding an annotation to
// the pod template that contains the last value we set.
mak.Set(&ss.Spec.Template.Annotations, podAnnotationLastSetHostname, sts.Hostname)
mak.Set(&pod.Annotations, podAnnotationLastSetHostname, sts.Hostname)
}
// Configure containeboot to run tailscaled with a configfile read from the state Secret.
if shouldDoTailscaledDeclarativeConfig(sts) {
mak.Set(&ss.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, tsConfigHash)
ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "tailscaledconfig",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
@ -489,7 +527,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
Value: a.tsFirewallMode,
})
}
ss.Spec.Template.Spec.PriorityClassName = a.proxyPriorityClassName
pod.Spec.PriorityClassName = a.proxyPriorityClassName
// Ingress/egress proxy configuration options.
if sts.ClusterTargetIP != "" {
@ -520,7 +558,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
ReadOnly: true,
MountPath: "/etc/tailscaled",
})
ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "serve-config",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
@ -534,7 +572,95 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
})
}
logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName())
return createOrUpdate(ctx, a.Client, a.operatorNamespace, &ss, func(s *appsv1.StatefulSet) { s.Spec = ss.Spec })
if sts.ProxyClass != "" {
logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClass)
ss = applyProxyClassToStatefulSet(proxyClass, ss)
}
updateSS := func(s *appsv1.StatefulSet) {
s.Spec = ss.Spec
s.ObjectMeta.Labels = ss.Labels
s.ObjectMeta.Annotations = ss.Annotations
}
return createOrUpdate(ctx, a.Client, a.operatorNamespace, ss, updateSS)
}
// mergeStatefulSetLabelsOrAnnots returns a map that contains all keys/values
// present in 'custom' map as well as those keys/values from the current map
// whose keys are present in the 'managed' map. The reason why this merge is
// necessary is to ensure that labels/annotations applied from a ProxyClass get removed
// if they are removed from a ProxyClass or if the ProxyClass no longer applies
// to this StatefulSet whilst any tailscale managed labels/annotations remain present.
func mergeStatefulSetLabelsOrAnnots(current, custom map[string]string, managed []string) map[string]string {
if custom == nil {
custom = make(map[string]string)
}
if current == nil {
return custom
}
for key, val := range current {
if slices.Contains(managed, key) {
custom[key] = val
}
}
return custom
}
func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet) *appsv1.StatefulSet {
if pc == nil || ss == nil || pc.Spec.StatefulSet == nil {
return ss
}
// Update StatefulSet metadata.
if wantsSSLabels := pc.Spec.StatefulSet.Labels; len(wantsSSLabels) > 0 {
ss.ObjectMeta.Labels = mergeStatefulSetLabelsOrAnnots(ss.ObjectMeta.Labels, wantsSSLabels, tailscaleManagedLabels)
}
if wantsSSAnnots := pc.Spec.StatefulSet.Annotations; len(wantsSSAnnots) > 0 {
ss.ObjectMeta.Annotations = mergeStatefulSetLabelsOrAnnots(ss.ObjectMeta.Annotations, wantsSSAnnots, tailscaleManagedAnnotations)
}
// Update Pod fields.
if pc.Spec.StatefulSet.Pod == nil {
return ss
}
wantsPod := pc.Spec.StatefulSet.Pod
if wantsPodLabels := wantsPod.Labels; len(wantsPodLabels) > 0 {
ss.Spec.Template.ObjectMeta.Labels = mergeStatefulSetLabelsOrAnnots(ss.Spec.Template.ObjectMeta.Labels, wantsPodLabels, tailscaleManagedLabels)
}
if wantsPodAnnots := wantsPod.Annotations; len(wantsPodAnnots) > 0 {
ss.Spec.Template.ObjectMeta.Annotations = mergeStatefulSetLabelsOrAnnots(ss.Spec.Template.ObjectMeta.Annotations, wantsPodAnnots, tailscaleManagedAnnotations)
}
ss.Spec.Template.Spec.SecurityContext = wantsPod.SecurityContext
ss.Spec.Template.Spec.ImagePullSecrets = wantsPod.ImagePullSecrets
ss.Spec.Template.Spec.NodeName = wantsPod.NodeName
ss.Spec.Template.Spec.NodeSelector = wantsPod.NodeSelector
ss.Spec.Template.Spec.Tolerations = wantsPod.Tolerations
// Update containers.
updateContainer := func(overlay *tsapi.Container, base corev1.Container) corev1.Container {
if overlay == nil {
return base
}
if overlay.SecurityContext != nil {
base.SecurityContext = overlay.SecurityContext
}
base.Resources = overlay.Resources
return base
}
for i, c := range ss.Spec.Template.Spec.Containers {
if c.Name == "tailscale" {
ss.Spec.Template.Spec.Containers[i] = updateContainer(wantsPod.TailscaleContainer, ss.Spec.Template.Spec.Containers[i])
break
}
}
if initContainers := ss.Spec.Template.Spec.InitContainers; len(initContainers) > 0 {
for i, c := range initContainers {
if c.Name == "sysctler" {
ss.Spec.Template.Spec.InitContainers[i] = updateContainer(wantsPod.TailscaleInitContainer, initContainers[i])
break
}
}
}
return ss
}
// tailscaledConfig takes a proxy config, a newly generated auth key if

252
cmd/k8s-operator/sts_test.go
View File

@ -6,10 +6,20 @@
package main
import (
_ "embed"
"fmt"
"reflect"
"regexp"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"sigs.k8s.io/yaml"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/types/ptr"
)
// Test_statefulSetNameBase tests that parent name portion in a StatefulSet name
@ -39,3 +49,245 @@ func Test_statefulSetNameBase(t *testing.T) {
}
}
}
func Test_applyProxyClassToStatefulSet(t *testing.T) {
// Setup
proxyClassAllOpts := &tsapi.ProxyClass{
Spec: tsapi.ProxyClassSpec{
StatefulSet: &tsapi.StatefulSet{
Labels: map[string]string{"foo": "bar"},
Annotations: map[string]string{"foo.io/bar": "foo"},
Pod: &tsapi.Pod{
Labels: map[string]string{"bar": "foo"},
Annotations: map[string]string{"bar.io/foo": "foo"},
SecurityContext: &corev1.PodSecurityContext{
RunAsUser: ptr.To(int64(0)),
},
ImagePullSecrets: []corev1.LocalObjectReference{{Name: "docker-creds"}},
NodeName: "some-node",
NodeSelector: map[string]string{"beta.kubernetes.io/os": "linux"},
Tolerations: []corev1.Toleration{{Key: "", Operator: "Exists"}},
TailscaleContainer: &tsapi.Container{
SecurityContext: &corev1.SecurityContext{
Privileged: ptr.To(true),
},
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1000m"), corev1.ResourceMemory: resource.MustParse("128Mi")},
Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("64Mi")},
},
},
TailscaleInitContainer: &tsapi.Container{
SecurityContext: &corev1.SecurityContext{
Privileged: ptr.To(true),
RunAsUser: ptr.To(int64(0)),
},
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1000m"), corev1.ResourceMemory: resource.MustParse("128Mi")},
Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("64Mi")},
},
},
},
},
},
}
proxyClassJustLabels := &tsapi.ProxyClass{
Spec: tsapi.ProxyClassSpec{
StatefulSet: &tsapi.StatefulSet{
Labels: map[string]string{"foo": "bar"},
Annotations: map[string]string{"foo.io/bar": "foo"},
Pod: &tsapi.Pod{
Labels: map[string]string{"bar": "foo"},
Annotations: map[string]string{"bar.io/foo": "foo"},
},
},
},
}
var userspaceProxySS, nonUserspaceProxySS appsv1.StatefulSet
if err := yaml.Unmarshal(userspaceProxyYaml, &userspaceProxySS); err != nil {
t.Fatalf("unmarshaling userspace proxy template: %v", err)
}
if err := yaml.Unmarshal(proxyYaml, &nonUserspaceProxySS); err != nil {
t.Fatalf("unmarshaling non-userspace proxy template: %v", err)
}
// Set a couple additional fields so we can test that we don't
// mistakenly override those.
labels := map[string]string{
LabelManaged: "true",
LabelParentName: "foo",
}
annots := map[string]string{
podAnnotationLastSetClusterIP: "1.2.3.4",
}
env := []corev1.EnvVar{{Name: "TS_HOSTNAME", Value: "nginx"}}
userspaceProxySS.Labels = labels
userspaceProxySS.Annotations = annots
userspaceProxySS.Spec.Template.Spec.Containers[0].Env = env
nonUserspaceProxySS.ObjectMeta.Labels = labels
nonUserspaceProxySS.ObjectMeta.Annotations = annots
nonUserspaceProxySS.Spec.Template.Spec.Containers[0].Env = env
// 1. Test that a ProxyClass with all fields set gets correctly applied
// to a Statefulset built from non-userspace proxy template.
wantSS := nonUserspaceProxySS.DeepCopy()
wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels)
wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels
wantSS.Spec.Template.Annotations = proxyClassAllOpts.Spec.StatefulSet.Pod.Annotations
wantSS.Spec.Template.Spec.SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.SecurityContext
wantSS.Spec.Template.Spec.ImagePullSecrets = proxyClassAllOpts.Spec.StatefulSet.Pod.ImagePullSecrets
wantSS.Spec.Template.Spec.NodeName = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeName
wantSS.Spec.Template.Spec.NodeSelector = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeSelector
wantSS.Spec.Template.Spec.Tolerations = proxyClassAllOpts.Spec.StatefulSet.Pod.Tolerations
wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext
wantSS.Spec.Template.Spec.InitContainers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleInitContainer.SecurityContext
wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources
wantSS.Spec.Template.Spec.InitContainers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleInitContainer.Resources
gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy())
if diff := cmp.Diff(gotSS, wantSS); diff != "" {
t.Fatalf("Unexpected result applying ProxyClass with all fields set to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
}
// 2. Test that a ProxyClass with custom labels and annotations for
// StatefulSet and Pod set gets correctly applied to a Statefulset built
// from non-userspace proxy template.
wantSS = nonUserspaceProxySS.DeepCopy()
wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels)
wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels
wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy())
if diff := cmp.Diff(gotSS, wantSS); diff != "" {
t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
}
// 3. Test that a ProxyClass with all fields set gets correctly applied
// to a Statefulset built from a userspace proxy template.
wantSS = userspaceProxySS.DeepCopy()
wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels)
wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels
wantSS.Spec.Template.Annotations = proxyClassAllOpts.Spec.StatefulSet.Pod.Annotations
wantSS.Spec.Template.Spec.SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.SecurityContext
wantSS.Spec.Template.Spec.ImagePullSecrets = proxyClassAllOpts.Spec.StatefulSet.Pod.ImagePullSecrets
wantSS.Spec.Template.Spec.NodeName = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeName
wantSS.Spec.Template.Spec.NodeSelector = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeSelector
wantSS.Spec.Template.Spec.Tolerations = proxyClassAllOpts.Spec.StatefulSet.Pod.Tolerations
wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext
wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources
gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy())
if diff := cmp.Diff(gotSS, wantSS); diff != "" {
t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
}
// 4. Test that a ProxyClass with custom labels and annotations gets correctly applied
// to a Statefulset built from a userspace proxy template.
wantSS = userspaceProxySS.DeepCopy()
wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels)
wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels
wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy())
if diff := cmp.Diff(gotSS, wantSS); diff != "" {
t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
}
}
func mergeMapKeys(a, b map[string]string) map[string]string {
for key, val := range b {
a[key] = val
}
return a
}
func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) {
tests := []struct {
name string
current map[string]string
custom map[string]string
managed []string
want map[string]string
}{
{
name: "no custom labels specified and none present in current labels, return current labels",
current: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
want: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
managed: tailscaleManagedLabels,
},
{
name: "no custom labels specified, but some present in current labels, return tailscale managed labels only from the current labels",
current: map[string]string{"foo": "bar", "something.io/foo": "bar", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
want: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
managed: tailscaleManagedLabels,
},
{
name: "custom labels specified, current labels only contain tailscale managed labels, return a union of both",
current: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
custom: map[string]string{"foo": "bar", "something.io/foo": "bar"},
want: map[string]string{"foo": "bar", "something.io/foo": "bar", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
managed: tailscaleManagedLabels,
},
{
name: "custom labels specified, current labels contain tailscale managed labels and custom labels, some of which re not present in the new custom labels, return a union of managed labels and the desired custom labels",
current: map[string]string{"foo": "bar", "bar": "baz", "app": "1234", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
custom: map[string]string{"foo": "bar", "something.io/foo": "bar"},
want: map[string]string{"foo": "bar", "something.io/foo": "bar", "app": "1234", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
managed: tailscaleManagedLabels,
},
{
name: "no current labels present, return custom labels only",
custom: map[string]string{"foo": "bar", "something.io/foo": "bar"},
want: map[string]string{"foo": "bar", "something.io/foo": "bar"},
managed: tailscaleManagedLabels,
},
{
name: "no current labels present, no custom labels specified, return empty map",
want: map[string]string{},
managed: tailscaleManagedLabels,
},
{
name: "no custom annots specified and none present in current annots, return current annots",
current: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"},
want: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"},
managed: tailscaleManagedAnnotations,
},
{
name: "no custom annots specified, but some present in current annots, return tailscale managed annots only from the current annots",
current: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"},
want: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"},
managed: tailscaleManagedAnnotations,
},
{
name: "custom annots specified, current annots only contain tailscale managed annots, return a union of both",
current: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"},
custom: map[string]string{"foo": "bar", "something.io/foo": "bar"},
want: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"},
managed: tailscaleManagedAnnotations,
},
{
name: "custom annots specified, current annots contain tailscale managed annots and custom annots, some of which are not present in the new custom annots, return a union of managed annots and the desired custom annots",
current: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"},
custom: map[string]string{"something.io/foo": "bar"},
want: map[string]string{"something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"},
managed: tailscaleManagedAnnotations,
},
{
name: "no current annots present, return custom annots only",
custom: map[string]string{"foo": "bar", "something.io/foo": "bar"},
want: map[string]string{"foo": "bar", "something.io/foo": "bar"},
managed: tailscaleManagedAnnotations,
},
{
name: "no current labels present, no custom labels specified, return empty map",
want: map[string]string{},
managed: tailscaleManagedAnnotations,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := mergeStatefulSetLabelsOrAnnots(tt.current, tt.custom, tt.managed); !reflect.DeepEqual(got, tt.want) {
t.Errorf("mergeStatefulSetLabels() = %v, want %v", got, tt.want)
}
})
}
}

26
cmd/k8s-operator/svc.go
View File

@ -20,6 +20,8 @@
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/util/clientmetric"
"tailscale.com/util/set"
)
@ -155,6 +157,17 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
a.logger.Error(msg)
return nil
}
proxyClass := proxyClassForObject(svc)
if proxyClass != "" {
if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
return fmt.Errorf("error verifying ProxyClass for Service: %w", err)
} else if !ready {
logger.Infof("ProxyClass %s specified for the Service, but is not (yet) Ready, waiting..", proxyClass)
return nil
}
}
hostname, err := nameForService(svc)
if err != nil {
return err
@ -183,6 +196,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
Hostname: hostname,
Tags: tags,
ChildResourceLabels: crl,
ProxyClass: proxyClass,
}
a.mu.Lock()
@ -318,3 +332,15 @@ func (a *ServiceReconciler) tailnetTargetAnnotation(svc *corev1.Service) string
}
return svc.Annotations[annotationTailnetTargetIPOld]
}
func proxyClassForObject(o client.Object) string {
return o.GetLabels()[LabelProxyClass]
}
func proxyClassIsReady(ctx context.Context, name string, cl client.Client) (bool, error) {
proxyClass := new(tsapi.ProxyClass)
if err := cl.Get(ctx, types.NamespacedName{Name: name}, proxyClass); err != nil {
return false, fmt.Errorf("error getting ProxyClass %s: %w", name, err)
}
return tsoperator.ProxyClassIsReady(proxyClass), nil
}

33
cmd/k8s-operator/testutils_test.go
View File

@ -24,6 +24,7 @@
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"tailscale.com/client/tailscale"
"tailscale.com/ipn"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/types/ptr"
"tailscale.com/util/mak"
)
@ -47,9 +48,11 @@ type configOpts struct {
confFileHash string
serveConfig *ipn.ServeConfig
shouldEnableForwardingClusterTrafficViaIngress bool
proxyClass string // configuration from the named ProxyClass should be applied to proxy resources
}
func expectedSTS(opts configOpts) *appsv1.StatefulSet {
func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
t.Helper()
tsContainer := corev1.Container{
Name: "tailscale",
Image: "tailscale/tailscale",
@ -141,7 +144,7 @@ func expectedSTS(opts configOpts) *appsv1.StatefulSet {
})
tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"})
}
return &appsv1.StatefulSet{
ss := &appsv1.StatefulSet{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
APIVersion: "apps/v1",
@ -194,9 +197,20 @@ func expectedSTS(opts configOpts) *appsv1.StatefulSet {
},
},
}
// If opts.proxyClass is set, retrieve the ProxyClass and apply
// configuration from that to the StatefulSet.
if opts.proxyClass != "" {
t.Logf("applying configuration from ProxyClass %s", opts.proxyClass)
proxyClass := new(tsapi.ProxyClass)
if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil {
t.Fatalf("error getting ProxyClass: %v", err)
}
return applyProxyClassToStatefulSet(proxyClass, ss)
}
return ss
}
func expectedSTSUserspace(opts configOpts) *appsv1.StatefulSet {
func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
tsContainer := corev1.Container{
Name: "tailscale",
Image: "tailscale/tailscale",
@ -213,7 +227,7 @@ func expectedSTSUserspace(opts configOpts) *appsv1.StatefulSet {
annots := make(map[string]string)
volumes := []corev1.Volume{{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}}}
annots["tailscale.com/operator-last-set-hostname"] = opts.hostname
return &appsv1.StatefulSet{
ss := &appsv1.StatefulSet{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
APIVersion: "apps/v1",
@ -255,6 +269,17 @@ func expectedSTSUserspace(opts configOpts) *appsv1.StatefulSet {
},
},
}
// If opts.proxyClass is set, retrieve the ProxyClass and apply
// configuration from that to the StatefulSet.
if opts.proxyClass != "" {
t.Logf("applying configuration from ProxyClass %s", opts.proxyClass)
proxyClass := new(tsapi.ProxyClass)
if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil {
t.Fatalf("error getting ProxyClass: %v", err)
}
return applyProxyClassToStatefulSet(proxyClass, ss)
}
return ss
}
func expectedHeadlessService(name string, parentType string) *corev1.Service {

2
k8s-operator/apis/v1alpha1/register.go
View File

@ -49,7 +49,7 @@ func init() {
// Adds the list of known types to api.Scheme.
func addKnownTypes(scheme *runtime.Scheme) error {
scheme.AddKnownTypes(SchemeGroupVersion, &Connector{}, &ConnectorList{})
scheme.AddKnownTypes(SchemeGroupVersion, &Connector{}, &ConnectorList{}, &ProxyClass{}, &ProxyClassList{})
metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
return nil

7
k8s-operator/apis/v1alpha1/types_connector.go
View File

@ -68,6 +68,12 @@ type ConnectorSpec struct {
// and 63 characters long.
// +optional
Hostname Hostname `json:"hostname,omitempty"`
// ProxyClass is the name of the ProxyClass custom resource that
// contains configuration options that should be applied to the
// resources created for this Connector. If unset, the operator will
// create resources with the default configuration.
// +optional
ProxyClass string `json:"proxyClass,omitempty"`
// SubnetRouter defines subnet routes that the Connector node should
// expose to tailnet. If unset, none are exposed.
// https://tailscale.com/kb/1019/subnets/
@ -181,4 +187,5 @@ type ConnectorCondition struct {
const (
ConnectorReady ConnectorConditionType = `ConnectorReady`
ProxyClassready ConnectorConditionType = `ProxyClassReady`
)

143
k8s-operator/apis/v1alpha1/types_proxyclass.go Normal file
View File

@ -0,0 +1,143 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package v1alpha1
import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
var ProxyClassKind = "ProxyClass"
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:resource:scope=Cluster
// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ProxyClassReady")].reason`,description="Status of the ProxyClass."
type ProxyClass struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Spec ProxyClassSpec `json:"spec"`
// +optional
Status ProxyClassStatus `json:"status"`
}
// +kubebuilder:object:root=true
type ProxyClassList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata"`
Items []ProxyClass `json:"items"`
}
type ProxyClassSpec struct {
// Proxy's StatefulSet spec.
StatefulSet *StatefulSet `json:"statefulSet"`
}
type StatefulSet struct {
// Labels that will be added to the StatefulSet created for the proxy.
// Any labels specified here will be merged with the default labels
// applied to the StatefulSet by the Tailscale Kubernetes operator as
// well as any other labels that might have been applied by other
// actors.
// Label keys and values must be valid Kubernetes label keys and values.
// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
// +optional
Labels map[string]string `json:"labels,omitempty"`
// Annotations that will be added to the StatefulSet created for the proxy.
// Any Annotations specified here will be merged with the default annotations
// applied to the StatefulSet by the Tailscale Kubernetes operator as
// well as any other annotations that might have been applied by other
// actors.
// Annotations must be valid Kubernetes annotations.
// https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set
// +optional
Annotations map[string]string `json:"annotations,omitempty"`
// Configuration for the proxy Pod.
// +optional
Pod *Pod `json:"pod,omitempty"`
}
type Pod struct {
// Labels that will be added to the proxy Pod.
// Any labels specified here will be merged with the default labels
// applied to the Pod by the Tailscale Kubernetes operator.
// Label keys and values must be valid Kubernetes label keys and values.
// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
// +optional
Labels map[string]string `json:"labels,omitempty"`
// Annotations that will be added to the proxy Pod.
// Any annotations specified here will be merged with the default
// annotations applied to the Pod by the Tailscale Kubernetes operator.
// Annotations must be valid Kubernetes annotations.
// https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set
// +optional
Annotations map[string]string `json:"annotations,omitempty"`
// Configuration for the proxy container running tailscale.
// +optional
TailscaleContainer *Container `json:"tailscaleContainer,omitempty"`
// Configuration for the proxy init container that enables forwarding.
// +optional
TailscaleInitContainer *Container `json:"tailscaleInitContainer,omitempty"`
// Proxy Pod's security context.
// By default Tailscale Kubernetes operator does not apply any Pod
// security context.
// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context-2
// +optional
SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"`
// Proxy Pod's image pull Secrets.
// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec
// +optional
ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
// Proxy Pod's node name.
// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling
// +optional
NodeName string `json:"nodeName,omitempty"`
// Proxy Pod's node selector.
// By default Tailscale Kubernetes operator does not apply any node
// selector.
// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling
// +optional
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
// Proxy Pod's tolerations.
// By default Tailscale Kubernetes operator does not apply any
// tolerations.
// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling
// +optional
Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
}
type Container struct {
// Container security context.
// Security context specified here will override the security context by the operator.
// By default the operator:
// - sets 'privileged: true' for the init container
// - set NET_ADMIN capability for tailscale container for proxies that
// are created for Services or Connector.
// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context
// +optional
SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"`
// Container resource requirements.
// By default Tailscale Kubernetes operator does not apply any resource
// requirements. The amount of resources required wil depend on the
// amount of resources the operator needs to parse, usage patterns and
// cluster size.
// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources
// +optional
Resources corev1.ResourceRequirements `json:"resources,omitempty"`
}
type ProxyClassStatus struct {
// List of status conditions to indicate the status of the ProxyClass.
// Known condition types are `ProxyClassReady`.
// +listType=map
// +listMapKey=type
// +optional
Conditions []ConnectorCondition `json:"conditions,omitempty"`
}

220
k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go
View File

@ -8,6 +8,7 @@
package v1alpha1
import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
)
@ -136,6 +137,191 @@ func (in *ConnectorStatus) DeepCopy() *ConnectorStatus {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Container) DeepCopyInto(out *Container) {
*out = *in
if in.SecurityContext != nil {
in, out := &in.SecurityContext, &out.SecurityContext
*out = new(v1.SecurityContext)
(*in).DeepCopyInto(*out)
}
in.Resources.DeepCopyInto(&out.Resources)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Container.
func (in *Container) DeepCopy() *Container {
if in == nil {
return nil
}
out := new(Container)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Pod) DeepCopyInto(out *Pod) {
*out = *in
if in.Labels != nil {
in, out := &in.Labels, &out.Labels
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.Annotations != nil {
in, out := &in.Annotations, &out.Annotations
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.TailscaleContainer != nil {
in, out := &in.TailscaleContainer, &out.TailscaleContainer
*out = new(Container)
(*in).DeepCopyInto(*out)
}
if in.TailscaleInitContainer != nil {
in, out := &in.TailscaleInitContainer, &out.TailscaleInitContainer
*out = new(Container)
(*in).DeepCopyInto(*out)
}
if in.SecurityContext != nil {
in, out := &in.SecurityContext, &out.SecurityContext
*out = new(v1.PodSecurityContext)
(*in).DeepCopyInto(*out)
}
if in.ImagePullSecrets != nil {
in, out := &in.ImagePullSecrets, &out.ImagePullSecrets
*out = make([]v1.LocalObjectReference, len(*in))
copy(*out, *in)
}
if in.NodeSelector != nil {
in, out := &in.NodeSelector, &out.NodeSelector
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.Tolerations != nil {
in, out := &in.Tolerations, &out.Tolerations
*out = make([]v1.Toleration, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Pod.
func (in *Pod) DeepCopy() *Pod {
if in == nil {
return nil
}
out := new(Pod)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ProxyClass) DeepCopyInto(out *ProxyClass) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
in.Spec.DeepCopyInto(&out.Spec)
in.Status.DeepCopyInto(&out.Status)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClass.
func (in *ProxyClass) DeepCopy() *ProxyClass {
if in == nil {
return nil
}
out := new(ProxyClass)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *ProxyClass) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ProxyClassList) DeepCopyInto(out *ProxyClassList) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
*out = make([]ProxyClass, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassList.
func (in *ProxyClassList) DeepCopy() *ProxyClassList {
if in == nil {
return nil
}
out := new(ProxyClassList)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *ProxyClassList) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) {
*out = *in
if in.StatefulSet != nil {
in, out := &in.StatefulSet, &out.StatefulSet
*out = new(StatefulSet)
(*in).DeepCopyInto(*out)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassSpec.
func (in *ProxyClassSpec) DeepCopy() *ProxyClassSpec {
if in == nil {
return nil
}
out := new(ProxyClassSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ProxyClassStatus) DeepCopyInto(out *ProxyClassStatus) {
*out = *in
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]ConnectorCondition, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassStatus.
func (in *ProxyClassStatus) DeepCopy() *ProxyClassStatus {
if in == nil {
return nil
}
out := new(ProxyClassStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in Routes) DeepCopyInto(out *Routes) {
{
@ -155,6 +341,40 @@ func (in Routes) DeepCopy() Routes {
return *out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *StatefulSet) DeepCopyInto(out *StatefulSet) {
*out = *in
if in.Labels != nil {
in, out := &in.Labels, &out.Labels
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.Annotations != nil {
in, out := &in.Annotations, &out.Annotations
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.Pod != nil {
in, out := &in.Pod, &out.Pod
*out = new(Pod)
(*in).DeepCopyInto(*out)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSet.
func (in *StatefulSet) DeepCopy() *StatefulSet {
if in == nil {
return nil
}
out := new(StatefulSet)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SubnetRouter) DeepCopyInto(out *SubnetRouter) {
*out = *in

54
k8s-operator/conditions.go
View File

@ -7,6 +7,7 @@
import (
"slices"
"time"
"go.uber.org/zap"
xslices "golang.org/x/exp/slices"
@ -17,8 +18,28 @@
// SetConnectorCondition ensures that Connector status has a condition with the
// given attributes. LastTransitionTime gets set every time condition's status
// changes
// changes.
func SetConnectorCondition(cn *tsapi.Connector, conditionType tsapi.ConnectorConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
conds := updateCondition(cn.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
cn.Status.Conditions = conds
}
// RemoveConnectorCondition will remove condition of the given type.
func RemoveConnectorCondition(conn *tsapi.Connector, conditionType tsapi.ConnectorConditionType) {
conn.Status.Conditions = slices.DeleteFunc(conn.Status.Conditions, func(cond tsapi.ConnectorCondition) bool {
return cond.Type == conditionType
})
}
// SetProxyClassCondition ensures that ProxyClass status has a condition with the
// given attributes. LastTransitionTime gets set every time condition's status
// changes.
func SetProxyClassCondition(pc *tsapi.ProxyClass, conditionType tsapi.ConnectorConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
conds := updateCondition(pc.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
pc.Status.Conditions = conds
}
func updateCondition(conds []tsapi.ConnectorCondition, conditionType tsapi.ConnectorConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) []tsapi.ConnectorCondition {
newCondition := tsapi.ConnectorCondition{
Type: conditionType,
Status: status,
@ -27,34 +48,37 @@ func SetConnectorCondition(cn *tsapi.Connector, conditionType tsapi.ConnectorCon
ObservedGeneration: gen,
}
nowTime := metav1.NewTime(clock.Now())
nowTime := metav1.NewTime(clock.Now().Truncate(time.Second))
newCondition.LastTransitionTime = &nowTime
idx := xslices.IndexFunc(cn.Status.Conditions, func(cond tsapi.ConnectorCondition) bool {
idx := xslices.IndexFunc(conds, func(cond tsapi.ConnectorCondition) bool {
return cond.Type == conditionType
})
if idx == -1 {
cn.Status.Conditions = append(cn.Status.Conditions, newCondition)
return
conds = append(conds, newCondition)
return conds
}
// Update the existing condition
cond := cn.Status.Conditions[idx]
cond := conds[idx] // update the existing condition
// If this update doesn't contain a state transition, we don't update
// the conditions LastTransitionTime to Now()
// the conditions LastTransitionTime to Now().
if cond.Status == status {
newCondition.LastTransitionTime = cond.LastTransitionTime
} else {
logger.Info("Status change for Connector condition %s from %s to %s", conditionType, cond.Status, status)
logger.Infof("Status change for condition %s from %s to %s", conditionType, cond.Status, status)
}
cn.Status.Conditions[idx] = newCondition
conds[idx] = newCondition
return conds
}
// RemoveConnectorCondition will remove condition of the given type
func RemoveConnectorCondition(conn *tsapi.Connector, conditionType tsapi.ConnectorConditionType) {
conn.Status.Conditions = slices.DeleteFunc(conn.Status.Conditions, func(cond tsapi.ConnectorCondition) bool {
return cond.Type == conditionType
func ProxyClassIsReady(pc *tsapi.ProxyClass) bool {
idx := xslices.IndexFunc(pc.Status.Conditions, func(cond tsapi.ConnectorCondition) bool {
return cond.Type == tsapi.ProxyClassready
})
if idx == -1 {
return false
}
cond := pc.Status.Conditions[idx]
return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pc.Generation
}

4
k8s-operator/conditions_test.go
View File

@ -19,8 +19,8 @@
func TestSetConnectorCondition(t *testing.T) {
cn := tsapi.Connector{}
clock := tstest.NewClock(tstest.ClockOpts{})
fakeNow := metav1.NewTime(clock.Now())
fakePast := metav1.NewTime(clock.Now().Add(-5 * time.Minute))
fakeNow := metav1.NewTime(clock.Now().Truncate(time.Second))
fakePast := metav1.NewTime(clock.Now().Truncate(time.Second).Add(-5 * time.Minute))
zl, err := zap.NewDevelopment()
assert.Nil(t, err)