health: support delayed Warnable visibility (#12783)

Updates tailscale/tailscale#4136

To reduce the likelihood of presenting spurious warnings, add the ability to delay the visibility of certain Warnables, based on a TimeToVisible time.Duration field on each Warnable. The default is zero, meaning that a Warnable is immediately visible to the user when it enters an unhealthy state.

Signed-off-by: Andrea Gottardo <andrea@gottardo.me>

health/health_test.go

@@ -162,6 +162,53 @@ func TestWatcher(t *testing.T) {
 	}
 }
 
+// TestWatcherWithTimeToVisible tests that a registered watcher function gets called with the correct
+// Warnable and non-nil/nil UnhealthyState upon setting a Warnable to unhealthy/healthy, but the Warnable
+// has a TimeToVisible set, which means that a watcher should only be notified of an unhealthy state after
+// the TimeToVisible duration has passed.
+func TestSetUnhealthyWithTimeToVisible(t *testing.T) {
+	ht := Tracker{}
+	mw := Register(&Warnable{
+		Code:                "test-warnable-3-secs-to-visible",
+		Title:               "Test Warnable with 3 seconds to visible",
+		Text:                StaticMessage("Hello world"),
+		TimeToVisible:       2 * time.Second,
+		ImpactsConnectivity: true,
+	})
+	defer unregister(mw)
+
+	becameUnhealthy := make(chan struct{})
+	becameHealthy := make(chan struct{})
+
+	watchFunc := func(w *Warnable, us *UnhealthyState) {
+		if w != mw {
+			t.Fatalf("watcherFunc was called, but with an unexpected Warnable: %v, want: %v", w, w)
+		}
+
+		if us != nil {
+			t.Logf("watcherFunc was called with an UnhealthyState: %v", us)
+			becameUnhealthy <- struct{}{}
+		} else {
+			t.Logf("watcherFunc was called with an healthy state: %v", us)
+			becameHealthy <- struct{}{}
+		}
+	}
+
+	ht.RegisterWatcher(watchFunc)
+	ht.SetUnhealthy(mw, Args{ArgError: "Hello world"})
+
+	select {
+	case <-becameUnhealthy:
+		// Test failed because the watcher got notified of an unhealthy state
+		t.Fatalf("watcherFunc was called with an unhealthy state")
+	case <-becameHealthy:
+		// Test failed because the watcher got of a healthy state
+		t.Fatalf("watcherFunc was called with a healthy state")
+	case <-time.After(1 * time.Second):
+		// As expected, watcherFunc still had not been called after 1 second
+	}
+}
+
 func TestRegisterWarnablePanicsWithDuplicate(t *testing.T) {
 	w := &Warnable{
 		Code: "test-warnable-1",