all-kube: create Tailscale Service for HA kube-apiserver ProxyGroup (#16572)

Adds a new reconciler for ProxyGroups of type kube-apiserver that will
provision a Tailscale Service for each replica to advertise. Adds two
new condition types to the ProxyGroup, TailscaleServiceValid and
TailscaleServiceConfigured, to post updates on the state of that
reconciler in a way that's consistent with the service-pg reconciler.
The created Tailscale Service name is configurable via a new ProxyGroup
field spec.kubeAPISserver.ServiceName, which expects a string of the
form "svc:<dns-label>".

Lots of supporting changes were needed to implement this in a way that's
consistent with other operator workflows, including:

* Pulled containerboot's ensureServicesUnadvertised and certManager into
  kube/ libraries to be shared with k8s-proxy. Use those in k8s-proxy to
  aid Service cert sharing between replicas and graceful Service shutdown.
* For certManager, add an initial wait to the cert loop to wait until
  the domain appears in the devices's netmap to avoid a guaranteed error
  on the first issue attempt when it's quick to start.
* Made several methods in ingress-for-pg.go and svc-for-pg.go into
  functions to share with the new reconciler
* Added a Resource struct to the owner refs stored in Tailscale Service
  annotations to be able to distinguish between Ingress- and ProxyGroup-
  based Services that need cleaning up in the Tailscale API.
* Added a ListVIPServices method to the internal tailscale client to aid
  cleaning up orphaned Services
* Support for reading config from a kube Secret, and partial support for
  config reloading, to prevent us having to force Pod restarts when
  config changes.
* Fixed up the zap logger so it's possible to set debug log level.

Updates #13358

Change-Id: Ia9607441157dd91fb9b6ecbc318eecbef446e116
Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>

cmd/k8s-operator/proxygroup_specs.go

@@ -7,7 +7,6 @@ package main
 
 import (
 	"fmt"
-	"path/filepath"
 	"slices"
 	"strconv"
 	"strings"
@@ -16,6 +15,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/yaml"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
@@ -341,8 +341,11 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por
 										},
 									},
 									{
-										Name:  "TS_K8S_PROXY_CONFIG",
-										Value: filepath.Join("/etc/tsconfig/$(POD_NAME)/", kubeAPIServerConfigFile),
+										Name: "TS_K8S_PROXY_CONFIG",
+										Value: "kube:" + types.NamespacedName{
+											Namespace: namespace,
+											Name:      "$(POD_NAME)-config",
+										}.String(),
 									},
 								}
 
@@ -355,20 +358,6 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por
 
 								return envs
 							}(),
-							VolumeMounts: func() []corev1.VolumeMount {
-								var mounts []corev1.VolumeMount
-
-								// TODO(tomhjp): Read config directly from the Secret instead.
-								for i := range pgReplicas(pg) {
-									mounts = append(mounts, corev1.VolumeMount{
-										Name:      fmt.Sprintf("k8s-proxy-config-%d", i),
-										ReadOnly:  true,
-										MountPath: fmt.Sprintf("/etc/tsconfig/%s-%d", pg.Name, i),
-									})
-								}
-
-								return mounts
-							}(),
 							Ports: []corev1.ContainerPort{
 								{
 									Name:          "k8s-proxy",
@@ -378,21 +367,6 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por
 							},
 						},
 					},
-					Volumes: func() []corev1.Volume {
-						var volumes []corev1.Volume
-						for i := range pgReplicas(pg) {
-							volumes = append(volumes, corev1.Volume{
-								Name: fmt.Sprintf("k8s-proxy-config-%d", i),
-								VolumeSource: corev1.VolumeSource{
-									Secret: &corev1.SecretVolumeSource{
-										SecretName: pgConfigSecretName(pg.Name, i),
-									},
-								},
-							})
-						}
-
-						return volumes
-					}(),
 				},
 			},
 		},
@@ -426,6 +400,7 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
 				Resources: []string{"secrets"},
 				Verbs: []string{
 					"list",
+					"watch", // For k8s-proxy.
 				},
 			},
 			{
@@ -508,7 +483,7 @@ func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.S
 			ObjectMeta: metav1.ObjectMeta{
 				Name:            pgStateSecretName(pg.Name, i),
 				Namespace:       namespace,
-				Labels:          pgSecretLabels(pg.Name, "state"),
+				Labels:          pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeState),
 				OwnerReferences: pgOwnerReference(pg),
 			},
 		})