all-kube: create Tailscale Service for HA kube-apiserver ProxyGroup (#16572)

Adds a new reconciler for ProxyGroups of type kube-apiserver that will
provision a Tailscale Service for each replica to advertise. Adds two
new condition types to the ProxyGroup, TailscaleServiceValid and
TailscaleServiceConfigured, to post updates on the state of that
reconciler in a way that's consistent with the service-pg reconciler.
The created Tailscale Service name is configurable via a new ProxyGroup
field spec.kubeAPISserver.ServiceName, which expects a string of the
form "svc:<dns-label>".

Lots of supporting changes were needed to implement this in a way that's
consistent with other operator workflows, including:

* Pulled containerboot's ensureServicesUnadvertised and certManager into
  kube/ libraries to be shared with k8s-proxy. Use those in k8s-proxy to
  aid Service cert sharing between replicas and graceful Service shutdown.
* For certManager, add an initial wait to the cert loop to wait until
  the domain appears in the devices's netmap to avoid a guaranteed error
  on the first issue attempt when it's quick to start.
* Made several methods in ingress-for-pg.go and svc-for-pg.go into
  functions to share with the new reconciler
* Added a Resource struct to the owner refs stored in Tailscale Service
  annotations to be able to distinguish between Ingress- and ProxyGroup-
  based Services that need cleaning up in the Tailscale API.
* Added a ListVIPServices method to the internal tailscale client to aid
  cleaning up orphaned Services
* Support for reading config from a kube Secret, and partial support for
  config reloading, to prevent us having to force Pod restarts when
  config changes.
* Fixed up the zap logger so it's possible to set debug log level.

Updates #13358

Change-Id: Ia9607441157dd91fb9b6ecbc318eecbef446e116
Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>

kube/localclient/fake-client.go

@@ -0,0 +1,35 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package localclient
+
+import (
+	"context"
+	"fmt"
+
+	"tailscale.com/ipn"
+)
+
+type FakeLocalClient struct {
+	FakeIPNBusWatcher
+}
+
+func (f *FakeLocalClient) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (IPNBusWatcher, error) {
+	return &f.FakeIPNBusWatcher, nil
+}
+
+func (f *FakeLocalClient) CertPair(ctx context.Context, domain string) ([]byte, []byte, error) {
+	return nil, nil, fmt.Errorf("CertPair not implemented")
+}
+
+type FakeIPNBusWatcher struct {
+	NotifyChan chan ipn.Notify
+}
+
+func (f *FakeIPNBusWatcher) Close() error {
+	return nil
+}
+
+func (f *FakeIPNBusWatcher) Next() (ipn.Notify, error) {
+	return <-f.NotifyChan, nil
+}

kube/localclient/local-client.go

@@ -0,0 +1,49 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package localclient provides an interface for all the local.Client methods
+// kube needs to use, so that we can easily mock it in tests.
+package localclient
+
+import (
+	"context"
+	"io"
+
+	"tailscale.com/client/local"
+	"tailscale.com/ipn"
+)
+
+// LocalClient is roughly a subset of the local.Client struct's methods, used
+// for easier testing.
+type LocalClient interface {
+	WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (IPNBusWatcher, error)
+	CertIssuer
+}
+
+// IPNBusWatcher is local.IPNBusWatcher's methods restated in an interface to
+// allow for easier mocking in tests.
+type IPNBusWatcher interface {
+	io.Closer
+	Next() (ipn.Notify, error)
+}
+
+type CertIssuer interface {
+	CertPair(context.Context, string) ([]byte, []byte, error)
+}
+
+// New returns a LocalClient that wraps the provided local.Client.
+func New(lc *local.Client) LocalClient {
+	return &localClient{lc: lc}
+}
+
+type localClient struct {
+	lc *local.Client
+}
+
+func (l *localClient) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (IPNBusWatcher, error) {
+	return l.lc.WatchIPNBus(ctx, mask)
+}
+
+func (l *localClient) CertPair(ctx context.Context, domain string) ([]byte, []byte, error) {
+	return l.lc.CertPair(ctx, domain)
+}