cmd/k8s-operator,k8s-operator: add ProxyGroup CRD (#13591)

The ProxyGroup CRD specifies a set of N pods which will each be a
tailnet device, and will have M different ingress or egress services
mapped onto them. It is the mechanism for specifying how highly
available proxies need to be. This commit only adds the definition, no
controller loop, and so it is not currently functional.

This commit also splits out TailnetDevice and RecorderTailnetDevice
into separate structs because the URL field is specific to recorders,
but we want a more generic struct for use in the ProxyGroup status field.

Updates #13406

Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>

cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml

@@ -0,0 +1,188 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
+  name: proxygroups.tailscale.com
+spec:
+  group: tailscale.com
+  names:
+    kind: ProxyGroup
+    listKind: ProxyGroupList
+    plural: proxygroups
+    shortNames:
+      - pg
+    singular: proxygroup
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - description: Status of the deployed ProxyGroup resources.
+          jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
+          name: Status
+          type: string
+      name: v1alpha1
+      schema:
+        openAPIV3Schema:
+          type: object
+          required:
+            - spec
+          properties:
+            apiVersion:
+              description: |-
+                APIVersion defines the versioned schema of this representation of an object.
+                Servers should convert recognized schemas to the latest internal value, and
+                may reject unrecognized values.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+              type: string
+            kind:
+              description: |-
+                Kind is a string value representing the REST resource this object represents.
+                Servers may infer this from the endpoint the client submits requests to.
+                Cannot be updated.
+                In CamelCase.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: Spec describes the desired ProxyGroup instances.
+              type: object
+              required:
+                - type
+              properties:
+                hostnamePrefix:
+                  description: |-
+                    HostnamePrefix is the hostname prefix to use for tailnet devices created
+                    by the ProxyGroup. Each device will have the integer number from its
+                    StatefulSet pod appended to this prefix to form the full hostname.
+                    HostnamePrefix can contain lower case letters, numbers and dashes, it
+                    must not start with a dash and must be between 1 and 62 characters long.
+                  type: string
+                  pattern: ^[a-z0-9][a-z0-9-]{0,61}$
+                proxyClass:
+                  description: |-
+                    ProxyClass is the name of the ProxyClass custom resource that contains
+                    configuration options that should be applied to the resources created
+                    for this ProxyGroup. If unset, and no default ProxyClass is set, the
+                    operator will create resources with the default configuration.
+                  type: string
+                replicas:
+                  description: |-
+                    Replicas specifies how many replicas to create the StatefulSet with.
+                    Defaults to 2.
+                  type: integer
+                tags:
+                  description: |-
+                    Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
+                    If you specify custom tags here, make sure you also make the operator
+                    an owner of these tags.
+                    See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
+                    Tags cannot be changed once a ProxyGroup device has been created.
+                    Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
+                  type: array
+                  items:
+                    type: string
+                    pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
+                type:
+                  description: |-
+                    Type of the ProxyGroup, either ingress or egress. Each set of proxies
+                    managed by a single ProxyGroup definition operate as only ingress or
+                    only egress proxies.
+                  type: string
+                  enum:
+                    - egress
+            status:
+              description: |-
+                ProxyGroupStatus describes the status of the ProxyGroup resources. This is
+                set and managed by the Tailscale operator.
+              type: object
+              properties:
+                conditions:
+                  description: |-
+                    List of status conditions to indicate the status of the ProxyGroup
+                    resources. Known condition types are `ProxyGroupReady`.
+                  type: array
+                  items:
+                    description: Condition contains details for one aspect of the current state of this API Resource.
+                    type: object
+                    required:
+                      - lastTransitionTime
+                      - message
+                      - reason
+                      - status
+                      - type
+                    properties:
+                      lastTransitionTime:
+                        description: |-
+                          lastTransitionTime is the last time the condition transitioned from one status to another.
+                          This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                        type: string
+                        format: date-time
+                      message:
+                        description: |-
+                          message is a human readable message indicating details about the transition.
+                          This may be an empty string.
+                        type: string
+                        maxLength: 32768
+                      observedGeneration:
+                        description: |-
+                          observedGeneration represents the .metadata.generation that the condition was set based upon.
+                          For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                          with respect to the current state of the instance.
+                        type: integer
+                        format: int64
+                        minimum: 0
+                      reason:
+                        description: |-
+                          reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                          Producers of specific condition types may define expected values and meanings for this field,
+                          and whether the values are considered a guaranteed API.
+                          The value should be a CamelCase string.
+                          This field may not be empty.
+                        type: string
+                        maxLength: 1024
+                        minLength: 1
+                        pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      status:
+                        description: status of the condition, one of True, False, Unknown.
+                        type: string
+                        enum:
+                          - "True"
+                          - "False"
+                          - Unknown
+                      type:
+                        description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                        type: string
+                        maxLength: 316
+                        pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                  x-kubernetes-list-map-keys:
+                    - type
+                  x-kubernetes-list-type: map
+                devices:
+                  description: List of tailnet devices associated with the ProxyGroup StatefulSet.
+                  type: array
+                  items:
+                    type: object
+                    required:
+                      - hostname
+                    properties:
+                      hostname:
+                        description: |-
+                          Hostname is the fully qualified domain name of the device.
+                          If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
+                          node.
+                        type: string
+                      tailnetIPs:
+                        description: |-
+                          TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
+                          assigned to the device.
+                        type: array
+                        items:
+                          type: string
+                  x-kubernetes-list-map-keys:
+                    - hostname
+                  x-kubernetes-list-type: map
+      served: true
+      storage: true
+      subresources:
+        status: {}

cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml

@@ -1670,7 +1670,7 @@ spec:
                     - type
                   x-kubernetes-list-type: map
                 devices:
-                  description: List of tailnet devices associated with the Recorder statefulset.
+                  description: List of tailnet devices associated with the Recorder StatefulSet.
                   type: array
                   items:
                     type: object

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -2418,6 +2418,195 @@ spec:
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
+metadata:
+    annotations:
+        controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
+    name: proxygroups.tailscale.com
+spec:
+    group: tailscale.com
+    names:
+        kind: ProxyGroup
+        listKind: ProxyGroupList
+        plural: proxygroups
+        shortNames:
+            - pg
+        singular: proxygroup
+    scope: Cluster
+    versions:
+        - additionalPrinterColumns:
+            - description: Status of the deployed ProxyGroup resources.
+              jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
+              name: Status
+              type: string
+          name: v1alpha1
+          schema:
+            openAPIV3Schema:
+                properties:
+                    apiVersion:
+                        description: |-
+                            APIVersion defines the versioned schema of this representation of an object.
+                            Servers should convert recognized schemas to the latest internal value, and
+                            may reject unrecognized values.
+                            More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+                        type: string
+                    kind:
+                        description: |-
+                            Kind is a string value representing the REST resource this object represents.
+                            Servers may infer this from the endpoint the client submits requests to.
+                            Cannot be updated.
+                            In CamelCase.
+                            More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+                        type: string
+                    metadata:
+                        type: object
+                    spec:
+                        description: Spec describes the desired ProxyGroup instances.
+                        properties:
+                            hostnamePrefix:
+                                description: |-
+                                    HostnamePrefix is the hostname prefix to use for tailnet devices created
+                                    by the ProxyGroup. Each device will have the integer number from its
+                                    StatefulSet pod appended to this prefix to form the full hostname.
+                                    HostnamePrefix can contain lower case letters, numbers and dashes, it
+                                    must not start with a dash and must be between 1 and 62 characters long.
+                                pattern: ^[a-z0-9][a-z0-9-]{0,61}$
+                                type: string
+                            proxyClass:
+                                description: |-
+                                    ProxyClass is the name of the ProxyClass custom resource that contains
+                                    configuration options that should be applied to the resources created
+                                    for this ProxyGroup. If unset, and no default ProxyClass is set, the
+                                    operator will create resources with the default configuration.
+                                type: string
+                            replicas:
+                                description: |-
+                                    Replicas specifies how many replicas to create the StatefulSet with.
+                                    Defaults to 2.
+                                type: integer
+                            tags:
+                                description: |-
+                                    Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
+                                    If you specify custom tags here, make sure you also make the operator
+                                    an owner of these tags.
+                                    See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
+                                    Tags cannot be changed once a ProxyGroup device has been created.
+                                    Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
+                                items:
+                                    pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
+                                    type: string
+                                type: array
+                            type:
+                                description: |-
+                                    Type of the ProxyGroup, either ingress or egress. Each set of proxies
+                                    managed by a single ProxyGroup definition operate as only ingress or
+                                    only egress proxies.
+                                enum:
+                                    - egress
+                                type: string
+                        required:
+                            - type
+                        type: object
+                    status:
+                        description: |-
+                            ProxyGroupStatus describes the status of the ProxyGroup resources. This is
+                            set and managed by the Tailscale operator.
+                        properties:
+                            conditions:
+                                description: |-
+                                    List of status conditions to indicate the status of the ProxyGroup
+                                    resources. Known condition types are `ProxyGroupReady`.
+                                items:
+                                    description: Condition contains details for one aspect of the current state of this API Resource.
+                                    properties:
+                                        lastTransitionTime:
+                                            description: |-
+                                                lastTransitionTime is the last time the condition transitioned from one status to another.
+                                                This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                                            format: date-time
+                                            type: string
+                                        message:
+                                            description: |-
+                                                message is a human readable message indicating details about the transition.
+                                                This may be an empty string.
+                                            maxLength: 32768
+                                            type: string
+                                        observedGeneration:
+                                            description: |-
+                                                observedGeneration represents the .metadata.generation that the condition was set based upon.
+                                                For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                                                with respect to the current state of the instance.
+                                            format: int64
+                                            minimum: 0
+                                            type: integer
+                                        reason:
+                                            description: |-
+                                                reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                                                Producers of specific condition types may define expected values and meanings for this field,
+                                                and whether the values are considered a guaranteed API.
+                                                The value should be a CamelCase string.
+                                                This field may not be empty.
+                                            maxLength: 1024
+                                            minLength: 1
+                                            pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                                            type: string
+                                        status:
+                                            description: status of the condition, one of True, False, Unknown.
+                                            enum:
+                                                - "True"
+                                                - "False"
+                                                - Unknown
+                                            type: string
+                                        type:
+                                            description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                                            maxLength: 316
+                                            pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                            type: string
+                                    required:
+                                        - lastTransitionTime
+                                        - message
+                                        - reason
+                                        - status
+                                        - type
+                                    type: object
+                                type: array
+                                x-kubernetes-list-map-keys:
+                                    - type
+                                x-kubernetes-list-type: map
+                            devices:
+                                description: List of tailnet devices associated with the ProxyGroup StatefulSet.
+                                items:
+                                    properties:
+                                        hostname:
+                                            description: |-
+                                                Hostname is the fully qualified domain name of the device.
+                                                If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
+                                                node.
+                                            type: string
+                                        tailnetIPs:
+                                            description: |-
+                                                TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
+                                                assigned to the device.
+                                            items:
+                                                type: string
+                                            type: array
+                                    required:
+                                        - hostname
+                                    type: object
+                                type: array
+                                x-kubernetes-list-map-keys:
+                                    - hostname
+                                x-kubernetes-list-type: map
+                        type: object
+                required:
+                    - spec
+                type: object
+          served: true
+          storage: true
+          subresources:
+            status: {}
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
 metadata:
     annotations:
         controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
@@ -4084,7 +4273,7 @@ spec:
                                     - type
                                 x-kubernetes-list-type: map
                             devices:
-                                description: List of tailnet devices associated with the Recorder statefulset.
+                                description: List of tailnet devices associated with the Recorder StatefulSet.
                                 items:
                                     properties:
                                         hostname:

cmd/k8s-operator/generate/main.go

@@ -25,11 +25,13 @@
 	proxyClassCRDPath             = operatorDeploymentFilesPath + "/crds/tailscale.com_proxyclasses.yaml"
 	dnsConfigCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_dnsconfigs.yaml"
 	recorderCRDPath               = operatorDeploymentFilesPath + "/crds/tailscale.com_recorders.yaml"
+	proxyGroupCRDPath             = operatorDeploymentFilesPath + "/crds/tailscale.com_proxygroups.yaml"
 	helmTemplatesPath             = operatorDeploymentFilesPath + "/chart/templates"
 	connectorCRDHelmTemplatePath  = helmTemplatesPath + "/connector.yaml"
 	proxyClassCRDHelmTemplatePath = helmTemplatesPath + "/proxyclass.yaml"
 	dnsConfigCRDHelmTemplatePath  = helmTemplatesPath + "/dnsconfig.yaml"
 	recorderCRDHelmTemplatePath   = helmTemplatesPath + "/recorder.yaml"
+	proxyGroupCRDHelmTemplatePath = helmTemplatesPath + "/proxygroup.yaml"
 
 	helmConditionalStart = "{{ if .Values.installCRDs -}}\n"
 	helmConditionalEnd   = "{{- end -}}"
@@ -146,6 +148,7 @@ func generate(baseDir string) error {
 		{proxyClassCRDPath, proxyClassCRDHelmTemplatePath},
 		{dnsConfigCRDPath, dnsConfigCRDHelmTemplatePath},
 		{recorderCRDPath, recorderCRDHelmTemplatePath},
+		{proxyGroupCRDPath, proxyGroupCRDHelmTemplatePath},
 	} {
 		if err := addCRDToHelm(crd.crdPath, crd.templatePath); err != nil {
 			return fmt.Errorf("error adding %s CRD to Helm templates: %w", crd.crdPath, err)
@@ -161,6 +164,7 @@ func cleanup(baseDir string) error {
 		proxyClassCRDHelmTemplatePath,
 		dnsConfigCRDHelmTemplatePath,
 		recorderCRDHelmTemplatePath,
+		proxyGroupCRDHelmTemplatePath,
 	} {
 		if err := os.Remove(filepath.Join(baseDir, path)); err != nil && !os.IsNotExist(err) {
 			return fmt.Errorf("error cleaning up %s: %w", path, err)

cmd/k8s-operator/generate/main_test.go

@@ -62,6 +62,9 @@ func Test_generate(t *testing.T) {
 	if !strings.Contains(installContentsWithCRD.String(), "name: recorders.tailscale.com") {
 		t.Errorf("Recorder CRD not found in default chart install")
 	}
+	if !strings.Contains(installContentsWithCRD.String(), "name: proxygroups.tailscale.com") {
+		t.Errorf("ProxyGroup CRD not found in default chart install")
+	}
 
 	// Test that CRDs can be excluded from Helm chart install
 	installContentsWithoutCRD := bytes.NewBuffer([]byte{})
@@ -83,4 +86,7 @@ func Test_generate(t *testing.T) {
 	if strings.Contains(installContentsWithoutCRD.String(), "name: recorders.tailscale.com") {
 		t.Errorf("Recorder CRD found in chart install that should not contain a CRD")
 	}
+	if strings.Contains(installContentsWithoutCRD.String(), "name: proxygroups.tailscale.com") {
+		t.Errorf("ProxyGroup CRD found in chart install that should not contain a CRD")
+	}
 }

cmd/k8s-operator/tsrecorder.go

@@ -199,7 +199,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Reco
 		return fmt.Errorf("error creating StatefulSet: %w", err)
 	}
 
-	var devices []tsapi.TailnetDevice
+	var devices []tsapi.RecorderTailnetDevice
 
 	device, ok, err := r.getDeviceInfo(ctx, tsr.Name)
 	if err != nil {
@@ -337,20 +337,20 @@ func (r *RecorderReconciler) getNodeMetadata(ctx context.Context, tsrName string
 	return tailcfg.StableNodeID(profile.Config.NodeID), profile.Config.UserProfile.LoginName, ok, nil
 }
 
-func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string) (d tsapi.TailnetDevice, ok bool, err error) {
+func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
 	nodeID, dnsName, ok, err := r.getNodeMetadata(ctx, tsrName)
 	if !ok || err != nil {
-		return tsapi.TailnetDevice{}, false, err
+		return tsapi.RecorderTailnetDevice{}, false, err
 	}
 
 	// TODO(tomhjp): The profile info doesn't include addresses, which is why we
 	// need the API. Should we instead update the profile to include addresses?
 	device, err := r.tsClient.Device(ctx, string(nodeID), nil)
 	if err != nil {
-		return tsapi.TailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err)
+		return tsapi.RecorderTailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err)
 	}
 
-	d = tsapi.TailnetDevice{
+	d = tsapi.RecorderTailnetDevice{
 		Hostname:   device.Hostname,
 		TailnetIPs: device.Addresses,
 	}

cmd/k8s-operator/tsrecorder_test.go

@@ -105,7 +105,7 @@ func TestRecorder(t *testing.T) {
 		})
 
 		expectReconciled(t, reconciler, "", tsr.Name)
-		tsr.Status.Devices = []tsapi.TailnetDevice{
+		tsr.Status.Devices = []tsapi.RecorderTailnetDevice{
 			{
 				Hostname:   "test-device",
 				TailnetIPs: []string{"1.2.3.4", "::1"},

k8s-operator/api.md

@@ -14,6 +14,8 @@
 - [DNSConfigList](#dnsconfiglist)
 - [ProxyClass](#proxyclass)
 - [ProxyClassList](#proxyclasslist)
+- [ProxyGroup](#proxygroup)
+- [ProxyGroupList](#proxygrouplist)
 - [Recorder](#recorder)
 - [RecorderList](#recorderlist)
 
@@ -261,6 +263,21 @@ _Appears in:_
 
 
 
+#### HostnamePrefix
+
+_Underlying type:_ _string_
+
+
+
+_Validation:_
+- Pattern: `^[a-z0-9][a-z0-9-]{0,61}$`
+- Type: string
+
+_Appears in:_
+- [ProxyGroupSpec](#proxygroupspec)
+
+
+
 #### Metrics
 
 
@@ -450,6 +467,100 @@ _Appears in:_
 | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#condition-v1-meta) array_ | List of status conditions to indicate the status of the ProxyClass.<br />Known condition types are `ProxyClassReady`. |  |  |
 
 
+#### ProxyClassType
+
+_Underlying type:_ _string_
+
+
+
+_Validation:_
+- Enum: [egress]
+- Type: string
+
+_Appears in:_
+- [ProxyGroupSpec](#proxygroupspec)
+
+
+
+#### ProxyGroup
+
+
+
+
+
+
+
+_Appears in:_
+- [ProxyGroupList](#proxygrouplist)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `apiVersion` _string_ | `tailscale.com/v1alpha1` | | |
+| `kind` _string_ | `ProxyGroup` | | |
+| `kind` _string_ | Kind is a string value representing the REST resource this object represents.<br />Servers may infer this from the endpoint the client submits requests to.<br />Cannot be updated.<br />In CamelCase.<br />More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds |  |  |
+| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.<br />Servers should convert recognized schemas to the latest internal value, and<br />may reject unrecognized values.<br />More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources |  |  |
+| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. |  |  |
+| `spec` _[ProxyGroupSpec](#proxygroupspec)_ | Spec describes the desired ProxyGroup instances. |  |  |
+| `status` _[ProxyGroupStatus](#proxygroupstatus)_ | ProxyGroupStatus describes the status of the ProxyGroup resources. This is<br />set and managed by the Tailscale operator. |  |  |
+
+
+#### ProxyGroupList
+
+
+
+
+
+
+
+
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `apiVersion` _string_ | `tailscale.com/v1alpha1` | | |
+| `kind` _string_ | `ProxyGroupList` | | |
+| `kind` _string_ | Kind is a string value representing the REST resource this object represents.<br />Servers may infer this from the endpoint the client submits requests to.<br />Cannot be updated.<br />In CamelCase.<br />More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds |  |  |
+| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.<br />Servers should convert recognized schemas to the latest internal value, and<br />may reject unrecognized values.<br />More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources |  |  |
+| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. |  |  |
+| `items` _[ProxyGroup](#proxygroup) array_ |  |  |  |
+
+
+#### ProxyGroupSpec
+
+
+
+
+
+
+
+_Appears in:_
+- [ProxyGroup](#proxygroup)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `type` _[ProxyClassType](#proxyclasstype)_ | Type of the ProxyGroup, either ingress or egress. Each set of proxies<br />managed by a single ProxyGroup definition operate as only ingress or<br />only egress proxies. |  | Enum: [egress] <br />Type: string <br /> |
+| `tags` _[Tags](#tags)_ | Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].<br />If you specify custom tags here, make sure you also make the operator<br />an owner of these tags.<br />See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.<br />Tags cannot be changed once a ProxyGroup device has been created.<br />Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. |  | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$` <br />Type: string <br /> |
+| `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.<br />Defaults to 2. |  |  |
+| `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created<br />by the ProxyGroup. Each device will have the integer number from its<br />StatefulSet pod appended to this prefix to form the full hostname.<br />HostnamePrefix can contain lower case letters, numbers and dashes, it<br />must not start with a dash and must be between 1 and 62 characters long. |  | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$` <br />Type: string <br /> |
+| `proxyClass` _string_ | ProxyClass is the name of the ProxyClass custom resource that contains<br />configuration options that should be applied to the resources created<br />for this ProxyGroup. If unset, and no default ProxyClass is set, the<br />operator will create resources with the default configuration. |  |  |
+
+
+#### ProxyGroupStatus
+
+
+
+
+
+
+
+_Appears in:_
+- [ProxyGroup](#proxygroup)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#condition-v1-meta) array_ | List of status conditions to indicate the status of the ProxyGroup<br />resources. Known condition types are `ProxyGroupReady`. |  |  |
+| `devices` _[TailnetDevice](#tailnetdevice) array_ | List of tailnet devices associated with the ProxyGroup StatefulSet. |  |  |
+
+
 #### Recorder
 
 
@@ -586,7 +697,25 @@ _Appears in:_
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
 | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#condition-v1-meta) array_ | List of status conditions to indicate the status of the Recorder.<br />Known condition types are `RecorderReady`. |  |  |
-| `devices` _[TailnetDevice](#tailnetdevice) array_ | List of tailnet devices associated with the Recorder statefulset. |  |  |
+| `devices` _[RecorderTailnetDevice](#recordertailnetdevice) array_ | List of tailnet devices associated with the Recorder StatefulSet. |  |  |
+
+
+#### RecorderTailnetDevice
+
+
+
+
+
+
+
+_Appears in:_
+- [RecorderStatus](#recorderstatus)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `hostname` _string_ | Hostname is the fully qualified domain name of the device.<br />If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the<br />node. |  |  |
+| `tailnetIPs` _string array_ | TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)<br />assigned to the device. |  |  |
+| `url` _string_ | URL where the UI is available if enabled for replaying recordings. This<br />will be an HTTPS MagicDNS URL. You must be connected to the same tailnet<br />as the recorder to access it. |  |  |
 
 
 #### Route
@@ -748,6 +877,7 @@ _Validation:_
 
 _Appears in:_
 - [ConnectorSpec](#connectorspec)
+- [ProxyGroupSpec](#proxygroupspec)
 - [RecorderSpec](#recorderspec)
 
 
@@ -761,13 +891,12 @@ _Appears in:_
 
 
 _Appears in:_
-- [RecorderStatus](#recorderstatus)
+- [ProxyGroupStatus](#proxygroupstatus)
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
 | `hostname` _string_ | Hostname is the fully qualified domain name of the device.<br />If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the<br />node. |  |  |
 | `tailnetIPs` _string array_ | TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)<br />assigned to the device. |  |  |
-| `url` _string_ | URL where the UI is available if enabled for replaying recordings. This<br />will be an HTTPS MagicDNS URL. You must be connected to the same tailnet<br />as the recorder to access it. |  |  |
 
 
 #### TailscaleConfig

k8s-operator/apis/v1alpha1/types_proxygroup.go

@@ -0,0 +1,112 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:scope=Cluster,shortName=pg
+// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ProxyGroupReady")].reason`,description="Status of the deployed ProxyGroup resources."
+
+type ProxyGroup struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	// Spec describes the desired ProxyGroup instances.
+	Spec ProxyGroupSpec `json:"spec"`
+
+	// ProxyGroupStatus describes the status of the ProxyGroup resources. This is
+	// set and managed by the Tailscale operator.
+	// +optional
+	Status ProxyGroupStatus `json:"status"`
+}
+
+// +kubebuilder:object:root=true
+
+type ProxyGroupList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata"`
+
+	Items []ProxyGroup `json:"items"`
+}
+
+type ProxyGroupSpec struct {
+	// Type of the ProxyGroup, either ingress or egress. Each set of proxies
+	// managed by a single ProxyGroup definition operate as only ingress or
+	// only egress proxies.
+	Type ProxyClassType `json:"type"`
+
+	// Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
+	// If you specify custom tags here, make sure you also make the operator
+	// an owner of these tags.
+	// See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
+	// Tags cannot be changed once a ProxyGroup device has been created.
+	// Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
+	// +optional
+	Tags Tags `json:"tags,omitempty"`
+
+	// Replicas specifies how many replicas to create the StatefulSet with.
+	// Defaults to 2.
+	// +optional
+	Replicas *int `json:"replicas,omitempty"`
+
+	// HostnamePrefix is the hostname prefix to use for tailnet devices created
+	// by the ProxyGroup. Each device will have the integer number from its
+	// StatefulSet pod appended to this prefix to form the full hostname.
+	// HostnamePrefix can contain lower case letters, numbers and dashes, it
+	// must not start with a dash and must be between 1 and 62 characters long.
+	// +optional
+	HostnamePrefix HostnamePrefix `json:"hostnamePrefix,omitempty"`
+
+	// ProxyClass is the name of the ProxyClass custom resource that contains
+	// configuration options that should be applied to the resources created
+	// for this ProxyGroup. If unset, and no default ProxyClass is set, the
+	// operator will create resources with the default configuration.
+	// +optional
+	ProxyClass string `json:"proxyClass,omitempty"`
+}
+
+type ProxyGroupStatus struct {
+	// List of status conditions to indicate the status of the ProxyGroup
+	// resources. Known condition types are `ProxyGroupReady`.
+	// +listType=map
+	// +listMapKey=type
+	// +optional
+	Conditions []metav1.Condition `json:"conditions,omitempty"`
+
+	// List of tailnet devices associated with the ProxyGroup StatefulSet.
+	// +listType=map
+	// +listMapKey=hostname
+	// +optional
+	Devices []TailnetDevice `json:"devices,omitempty"`
+}
+
+type TailnetDevice struct {
+	// Hostname is the fully qualified domain name of the device.
+	// If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
+	// node.
+	Hostname string `json:"hostname"`
+
+	// TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
+	// assigned to the device.
+	// +optional
+	TailnetIPs []string `json:"tailnetIPs,omitempty"`
+}
+
+// +kubebuilder:validation:Type=string
+// +kubebuilder:validation:Enum=egress
+type ProxyClassType string
+
+const (
+	ProxyClassTypeEgress ProxyClassType = "egress"
+)
+
+// +kubebuilder:validation:Type=string
+// +kubebuilder:validation:Pattern=`^[a-z0-9][a-z0-9-]{0,61}$`
+type HostnamePrefix string

k8s-operator/apis/v1alpha1/types_recorder.go

@@ -223,14 +223,14 @@ type RecorderStatus struct {
 	// +optional
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
 
-	// List of tailnet devices associated with the Recorder statefulset.
+	// List of tailnet devices associated with the Recorder StatefulSet.
 	// +listType=map
 	// +listMapKey=hostname
 	// +optional
-	Devices []TailnetDevice `json:"devices,omitempty"`
+	Devices []RecorderTailnetDevice `json:"devices,omitempty"`
 }
 
-type TailnetDevice struct {
+type RecorderTailnetDevice struct {
 	// Hostname is the fully qualified domain name of the device.
 	// If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
 	// node.

k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go

@@ -515,6 +515,119 @@ func (in *ProxyClassStatus) DeepCopy() *ProxyClassStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProxyGroup) DeepCopyInto(out *ProxyGroup) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyGroup.
+func (in *ProxyGroup) DeepCopy() *ProxyGroup {
+	if in == nil {
+		return nil
+	}
+	out := new(ProxyGroup)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ProxyGroup) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProxyGroupList) DeepCopyInto(out *ProxyGroupList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]ProxyGroup, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyGroupList.
+func (in *ProxyGroupList) DeepCopy() *ProxyGroupList {
+	if in == nil {
+		return nil
+	}
+	out := new(ProxyGroupList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ProxyGroupList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProxyGroupSpec) DeepCopyInto(out *ProxyGroupSpec) {
+	*out = *in
+	if in.Tags != nil {
+		in, out := &in.Tags, &out.Tags
+		*out = make(Tags, len(*in))
+		copy(*out, *in)
+	}
+	if in.Replicas != nil {
+		in, out := &in.Replicas, &out.Replicas
+		*out = new(int)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyGroupSpec.
+func (in *ProxyGroupSpec) DeepCopy() *ProxyGroupSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(ProxyGroupSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProxyGroupStatus) DeepCopyInto(out *ProxyGroupStatus) {
+	*out = *in
+	if in.Conditions != nil {
+		in, out := &in.Conditions, &out.Conditions
+		*out = make([]v1.Condition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.Devices != nil {
+		in, out := &in.Devices, &out.Devices
+		*out = make([]TailnetDevice, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyGroupStatus.
+func (in *ProxyGroupStatus) DeepCopy() *ProxyGroupStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ProxyGroupStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Recorder) DeepCopyInto(out *Recorder) {
 	*out = *in
@@ -723,7 +836,7 @@ func (in *RecorderStatus) DeepCopyInto(out *RecorderStatus) {
 	}
 	if in.Devices != nil {
 		in, out := &in.Devices, &out.Devices
-		*out = make([]TailnetDevice, len(*in))
+		*out = make([]RecorderTailnetDevice, len(*in))
 		for i := range *in {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
@@ -740,6 +853,26 @@ func (in *RecorderStatus) DeepCopy() *RecorderStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RecorderTailnetDevice) DeepCopyInto(out *RecorderTailnetDevice) {
+	*out = *in
+	if in.TailnetIPs != nil {
+		in, out := &in.TailnetIPs, &out.TailnetIPs
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RecorderTailnetDevice.
+func (in *RecorderTailnetDevice) DeepCopy() *RecorderTailnetDevice {
+	if in == nil {
+		return nil
+	}
+	out := new(RecorderTailnetDevice)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in Routes) DeepCopyInto(out *Routes) {
 	{