ipn/{ipnlocal,localapi}: ensure watcher is installed before /watch-ipn-bus/ responds with 200

This change delays the first flush in the /watch-ipn-bus/ handler
until after the watcher has been successfully installed on the IPN
bus. It does this by adding a new onWatchAdded callback to
LocalBackend.WatchNotifications().

Without this, the endpoint returns a 200 almost immediatly, and
only then installs a watcher for IPN events.  This means there's a
small window where events could be missed by clients after calling
WatchIPNBus().

Fixes tailscale/corp#8594.

Signed-off-by: salman <salman@tailscale.com>

ipn/ipnlocal/local_test.go

@@ -5,6 +5,7 @@
 package ipnlocal
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/http"
@@ -820,3 +821,38 @@ type legacyBackend interface {
 // Verify that LocalBackend still implements the legacyBackend interface
 // for now, at least until the macOS and iOS clients move off of it.
 var _ legacyBackend = (*LocalBackend)(nil)
+
+func TestWatchNotificationsCallbacks(t *testing.T) {
+	b := new(LocalBackend)
+	n := new(ipn.Notify)
+	b.WatchNotifications(context.Background(), 0, func() {
+		b.mu.Lock()
+		defer b.mu.Unlock()
+
+		// Ensure a watcher has been installed.
+		if len(b.notifyWatchers) != 1 {
+			t.Fatalf("unexpected number of watchers in new LocalBackend, want: 1 got: %v", len(b.notifyWatchers))
+		}
+		// Send a notification. Range over notifyWatchers to get the channel
+		// because WatchNotifications doesn't expose the handle for it.
+		for _, c := range b.notifyWatchers {
+			select {
+			case c <- n:
+			default:
+				t.Fatalf("could not send notification")
+			}
+		}
+	}, func(roNotify *ipn.Notify) bool {
+		if roNotify != n {
+			t.Fatalf("unexpected notification received. want: %v got: %v", n, roNotify)
+		}
+		return false
+	})
+
+	// Ensure watchers have been cleaned up.
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if len(b.notifyWatchers) != 0 {
+		t.Fatalf("unexpected number of watchers in new LocalBackend, want: 0 got: %v", len(b.notifyWatchers))
+	}
+}