util/syspolicy/source: add package for reading policy settings from external stores

We add package defining interfaces for policy stores, enabling creation of policy sources
and reading settings from them. It includes a Windows-specific PlatformPolicyStore for GP and MDM
policies stored in the Registry, and an in-memory TestStore for testing purposes.

We also include an internal package that tracks and reports policy usage metrics when a policy setting
is read from a store. Initially, it will be used only on Windows and Android, as macOS, iOS, and tvOS
report their own metrics. However, we plan to use it across all platforms eventually.

Updates #12687

Signed-off-by: Nick Khyl <nickk@tailscale.com>

util/syspolicy/internal/metrics/metrics.go

@@ -0,0 +1,320 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package metrics provides logging and reporting for policy settings and scopes.
+package metrics
+
+import (
+	"strings"
+	"sync"
+
+	xmaps "golang.org/x/exp/maps"
+
+	"tailscale.com/syncs"
+	"tailscale.com/types/lazy"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/mak"
+	"tailscale.com/util/slicesx"
+	"tailscale.com/util/syspolicy/internal"
+	"tailscale.com/util/syspolicy/internal/loggerx"
+	"tailscale.com/util/syspolicy/setting"
+	"tailscale.com/util/testenv"
+)
+
+var lazyReportMetrics lazy.SyncValue[bool] // used as a test hook
+
+// ShouldReport reports whether metrics should be reported on the current environment.
+func ShouldReport() bool {
+	return lazyReportMetrics.Get(func() bool {
+		// macOS, iOS and tvOS create their own metrics,
+		// and we don't have syspolicy on any other platforms.
+		return setting.PlatformList{"android", "windows"}.HasCurrent()
+	})
+}
+
+// Reset metrics for the specified policy origin.
+func Reset(origin *setting.Origin) {
+	scopeMetrics(origin).Reset()
+}
+
+// ReportConfigured updates metrics and logs that the specified setting is
+// configured with the given value in the origin.
+func ReportConfigured(origin *setting.Origin, setting *setting.Definition, value any) {
+	settingMetricsFor(setting).ReportValue(origin, value)
+}
+
+// ReportError updates metrics and logs that the specified setting has an error
+// in the origin.
+func ReportError(origin *setting.Origin, setting *setting.Definition, err error) {
+	settingMetricsFor(setting).ReportError(origin, err)
+}
+
+// ReportNotConfigured updates metrics and logs that the specified setting is
+// not configured in the origin.
+func ReportNotConfigured(origin *setting.Origin, setting *setting.Definition) {
+	settingMetricsFor(setting).Reset(origin)
+}
+
+// metric is an interface implemented by [clientmetric.Metric] and [funcMetric].
+type metric interface {
+	Add(v int64)
+	Set(v int64)
+}
+
+// policyScopeMetrics are metrics that apply to an entire policy scope rather
+// than a specific policy setting.
+type policyScopeMetrics struct {
+	hasAny     metric
+	numErrored metric
+}
+
+func newScopeMetrics(scope setting.Scope) *policyScopeMetrics {
+	prefix := metricScopeName(scope)
+	// {os}_syspolicy_{scope_unless_device}_any
+	// Example: windows_syspolicy_any or windows_syspolicy_user_any.
+	hasAny := newMetric([]string{prefix, "any"}, clientmetric.TypeGauge)
+	// {os}_syspolicy_{scope_unless_device}_errors
+	// Example: windows_syspolicy_errors or windows_syspolicy_user_errors.
+	//
+	// TODO(nickkhyl): maybe make the `{os}_syspolicy_errors` metric a gauge rather than a counter?
+	// It was a counter prior to https://github.com/tailscale/tailscale/issues/12687, so I kept it as such.
+	// But I think a gauge makes more sense: syspolicy errors indicate a mismatch between the expected
+	// policy value type or format and the actual value read from the underlying store (like the Windows Registry).
+	// We'll encounter the same error every time we re-read the policy setting from the backing store
+	// until the policy value is corrected by the user, or until we fix the bug in the code or ADMX.
+	// There's probably no reason to count and accumulate them over time.
+	//
+	// Brief discussion: https://github.com/tailscale/tailscale/pull/13113#discussion_r1723475136
+	numErrored := newMetric([]string{prefix, "errors"}, clientmetric.TypeCounter)
+	return &policyScopeMetrics{hasAny, numErrored}
+}
+
+// ReportHasSettings is called when there's any configured policy setting in the scope.
+func (m *policyScopeMetrics) ReportHasSettings() {
+	if m != nil {
+		m.hasAny.Set(1)
+	}
+}
+
+// ReportError is called when there's any errored policy setting in the scope.
+func (m *policyScopeMetrics) ReportError() {
+	if m != nil {
+		m.numErrored.Add(1)
+	}
+}
+
+// Reset is called to reset the policy scope metrics, such as when the policy scope
+// is about to be reloaded.
+func (m *policyScopeMetrics) Reset() {
+	if m != nil {
+		m.hasAny.Set(0)
+		// numErrored is a counter and cannot be (re-)set.
+	}
+}
+
+// settingMetrics are metrics for a single policy setting in one or more scopes.
+type settingMetrics struct {
+	definition *setting.Definition
+	isSet      []metric // by scope
+	hasErrors  []metric // by scope
+}
+
+// ReportValue is called when the policy setting is found to be configured in the specified source.
+func (m *settingMetrics) ReportValue(origin *setting.Origin, v any) {
+	if m == nil {
+		return
+	}
+	if scope := origin.Scope().Kind(); scope >= 0 && int(scope) < len(m.isSet) {
+		m.isSet[scope].Set(1)
+		m.hasErrors[scope].Set(0)
+	}
+	scopeMetrics(origin).ReportHasSettings()
+	loggerx.Verbosef("%v(%q) = %v", origin, m.definition.Key(), v)
+}
+
+// ReportError is called when there's an error with the policy setting in the specified source.
+func (m *settingMetrics) ReportError(origin *setting.Origin, err error) {
+	if m == nil {
+		return
+	}
+	if scope := origin.Scope().Kind(); int(scope) < len(m.hasErrors) {
+		m.isSet[scope].Set(0)
+		m.hasErrors[scope].Set(1)
+	}
+	scopeMetrics(origin).ReportError()
+	loggerx.Errorf("%v(%q): %v", origin, m.definition.Key(), err)
+}
+
+// Reset is called to reset the policy setting's metrics, such as when
+// the policy setting does not exist or the source containing the policy
+// is about to be reloaded.
+func (m *settingMetrics) Reset(origin *setting.Origin) {
+	if m == nil {
+		return
+	}
+	if scope := origin.Scope().Kind(); scope >= 0 && int(scope) < len(m.isSet) {
+		m.isSet[scope].Set(0)
+		m.hasErrors[scope].Set(0)
+	}
+}
+
+// metricFn is a function that adds or sets a metric value.
+type metricFn func(name string, typ clientmetric.Type, v int64)
+
+// funcMetric implements [metric] by calling the specified add and set functions.
+// Used for testing, and with nil functions on platforms that do not support
+// syspolicy, and on platforms that report policy metrics from the GUI.
+type funcMetric struct {
+	name     string
+	typ      clientmetric.Type
+	add, set metricFn
+}
+
+func (m funcMetric) Add(v int64) {
+	if m.add != nil {
+		m.add(m.name, m.typ, v)
+	}
+}
+
+func (m funcMetric) Set(v int64) {
+	if m.set != nil {
+		m.set(m.name, m.typ, v)
+	}
+}
+
+var (
+	lazyDeviceMetrics  lazy.SyncValue[*policyScopeMetrics]
+	lazyProfileMetrics lazy.SyncValue[*policyScopeMetrics]
+	lazyUserMetrics    lazy.SyncValue[*policyScopeMetrics]
+)
+
+func scopeMetrics(origin *setting.Origin) *policyScopeMetrics {
+	switch origin.Scope().Kind() {
+	case setting.DeviceSetting:
+		return lazyDeviceMetrics.Get(func() *policyScopeMetrics {
+			return newScopeMetrics(setting.DeviceSetting)
+		})
+	case setting.ProfileSetting:
+		return lazyProfileMetrics.Get(func() *policyScopeMetrics {
+			return newScopeMetrics(setting.ProfileSetting)
+		})
+	case setting.UserSetting:
+		return lazyUserMetrics.Get(func() *policyScopeMetrics {
+			return newScopeMetrics(setting.UserSetting)
+		})
+	default:
+		panic("unreachable")
+	}
+}
+
+var (
+	settingMetricsMu  sync.RWMutex
+	settingMetricsMap map[setting.Key]*settingMetrics
+)
+
+func settingMetricsFor(setting *setting.Definition) *settingMetrics {
+	settingMetricsMu.RLock()
+	metrics, ok := settingMetricsMap[setting.Key()]
+	settingMetricsMu.RUnlock()
+	if ok {
+		return metrics
+	}
+	return settingMetricsForSlow(setting)
+}
+
+func settingMetricsForSlow(d *setting.Definition) *settingMetrics {
+	settingMetricsMu.Lock()
+	defer settingMetricsMu.Unlock()
+	if metrics, ok := settingMetricsMap[d.Key()]; ok {
+		return metrics
+	}
+
+	// The loop below initializes metrics for each scope where a policy setting defined in 'd'
+	// can be configured. The [setting.Definition.Scope] returns the narrowest scope at which the policy
+	// setting may be configured, and more specific scopes always have higher numeric values.
+	// In other words, [setting.UserSetting] > [setting.ProfileScope] > [setting.DeviceScope].
+	// It's impossible for a policy setting to be configured in a scope with a higher numeric value than
+	// the [setting.Definition.Scope] returns. Therefore, a policy setting can be configured in at
+	// most d.Scope()+1 different scopes, and having d.Scope()+1 metrics for the corresponding scopes
+	// is always sufficient for [settingMetrics]; it won't access elements past the end of the slice
+	// or need to reallocate with a longer slice if one of those arrives.
+	isSet := make([]metric, d.Scope()+1)
+	hasErrors := make([]metric, d.Scope()+1)
+	for i := range isSet {
+		scope := setting.Scope(i)
+		// {os}_syspolicy_{key}_{scope_unless_device}
+		// Example: windows_syspolicy_AdminConsole or windows_syspolicy_AdminConsole_user.
+		isSet[i] = newSettingMetric(d.Key(), scope, "", clientmetric.TypeGauge)
+		// {os}_syspolicy_{key}_{scope_unless_device}_error
+		// Example: windows_syspolicy_AdminConsole_error or windows_syspolicy_TestSetting01_user_error.
+		hasErrors[i] = newSettingMetric(d.Key(), scope, "error", clientmetric.TypeGauge)
+	}
+	metrics := &settingMetrics{d, isSet, hasErrors}
+	mak.Set(&settingMetricsMap, d.Key(), metrics)
+	return metrics
+}
+
+// hooks for testing
+var addMetricTestHook, setMetricTestHook syncs.AtomicValue[metricFn]
+
+// SetHooksForTest sets the specified addMetric and setMetric functions
+// as the metric functions for the duration of tb and all its subtests.
+func SetHooksForTest(tb internal.TB, addMetric, setMetric metricFn) {
+	oldAddMetric := addMetricTestHook.Swap(addMetric)
+	oldSetMetric := setMetricTestHook.Swap(setMetric)
+	tb.Cleanup(func() {
+		addMetricTestHook.Store(oldAddMetric)
+		setMetricTestHook.Store(oldSetMetric)
+	})
+
+	settingMetricsMu.Lock()
+	oldSettingMetricsMap := xmaps.Clone(settingMetricsMap)
+	clear(settingMetricsMap)
+	settingMetricsMu.Unlock()
+	tb.Cleanup(func() {
+		settingMetricsMu.Lock()
+		settingMetricsMap = oldSettingMetricsMap
+		settingMetricsMu.Unlock()
+	})
+
+	// (re-)set the scope metrics to use the test hooks for the duration of tb.
+	lazyDeviceMetrics.SetForTest(tb, newScopeMetrics(setting.DeviceSetting), nil)
+	lazyProfileMetrics.SetForTest(tb, newScopeMetrics(setting.ProfileSetting), nil)
+	lazyUserMetrics.SetForTest(tb, newScopeMetrics(setting.UserSetting), nil)
+}
+
+func newSettingMetric(key setting.Key, scope setting.Scope, suffix string, typ clientmetric.Type) metric {
+	name := strings.ReplaceAll(string(key), setting.KeyPathSeparator, "_")
+	return newMetric([]string{name, metricScopeName(scope), suffix}, typ)
+}
+
+func newMetric(nameParts []string, typ clientmetric.Type) metric {
+	name := strings.Join(slicesx.Filter([]string{internal.OS(), "syspolicy"}, nameParts, isNonEmpty), "_")
+	switch {
+	case !ShouldReport():
+		return &funcMetric{name: name, typ: typ}
+	case testenv.InTest():
+		return &funcMetric{name, typ, addMetricTestHook.Load(), setMetricTestHook.Load()}
+	case typ == clientmetric.TypeCounter:
+		return clientmetric.NewCounter(name)
+	case typ == clientmetric.TypeGauge:
+		return clientmetric.NewGauge(name)
+	default:
+		panic("unreachable")
+	}
+}
+
+func isNonEmpty(s string) bool { return s != "" }
+
+func metricScopeName(scope setting.Scope) string {
+	switch scope {
+	case setting.DeviceSetting:
+		return ""
+	case setting.ProfileSetting:
+		return "profile"
+	case setting.UserSetting:
+		return "user"
+	default:
+		panic("unreachable")
+	}
+}

util/syspolicy/internal/metrics/metrics_test.go

@@ -0,0 +1,423 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package metrics
+
+import (
+	"errors"
+	"testing"
+
+	"tailscale.com/types/lazy"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/syspolicy/internal"
+	"tailscale.com/util/syspolicy/setting"
+)
+
+func TestSettingMetricNames(t *testing.T) {
+	tests := []struct {
+		name           string
+		key            setting.Key
+		scope          setting.Scope
+		suffix         string
+		typ            clientmetric.Type
+		osOverride     string
+		wantMetricName string
+	}{
+		{
+			name:           "windows-device-no-suffix",
+			key:            "AdminConsole",
+			scope:          setting.DeviceSetting,
+			suffix:         "",
+			typ:            clientmetric.TypeCounter,
+			osOverride:     "windows",
+			wantMetricName: "windows_syspolicy_AdminConsole",
+		},
+		{
+			name:           "windows-user-no-suffix",
+			key:            "AdminConsole",
+			scope:          setting.UserSetting,
+			suffix:         "",
+			typ:            clientmetric.TypeCounter,
+			osOverride:     "windows",
+			wantMetricName: "windows_syspolicy_AdminConsole_user",
+		},
+		{
+			name:           "windows-profile-no-suffix",
+			key:            "AdminConsole",
+			scope:          setting.ProfileSetting,
+			suffix:         "",
+			typ:            clientmetric.TypeCounter,
+			osOverride:     "windows",
+			wantMetricName: "windows_syspolicy_AdminConsole_profile",
+		},
+		{
+			name:           "windows-profile-err",
+			key:            "AdminConsole",
+			scope:          setting.ProfileSetting,
+			suffix:         "error",
+			typ:            clientmetric.TypeCounter,
+			osOverride:     "windows",
+			wantMetricName: "windows_syspolicy_AdminConsole_profile_error",
+		},
+		{
+			name:           "android-device-no-suffix",
+			key:            "AdminConsole",
+			scope:          setting.DeviceSetting,
+			suffix:         "",
+			typ:            clientmetric.TypeCounter,
+			osOverride:     "android",
+			wantMetricName: "android_syspolicy_AdminConsole",
+		},
+		{
+			name:           "key-path",
+			key:            "category/subcategory/setting",
+			scope:          setting.DeviceSetting,
+			suffix:         "",
+			typ:            clientmetric.TypeCounter,
+			osOverride:     "fakeos",
+			wantMetricName: "fakeos_syspolicy_category_subcategory_setting",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			internal.OSForTesting.SetForTest(t, tt.osOverride, nil)
+			metric, ok := newSettingMetric(tt.key, tt.scope, tt.suffix, tt.typ).(*funcMetric)
+			if !ok {
+				t.Fatal("metric is not a funcMetric")
+			}
+			if metric.name != tt.wantMetricName {
+				t.Errorf("got %q, want %q", metric.name, tt.wantMetricName)
+			}
+		})
+	}
+}
+
+func TestScopeMetrics(t *testing.T) {
+	tests := []struct {
+		name               string
+		scope              setting.Scope
+		osOverride         string
+		wantHasAnyName     string
+		wantNumErroredName string
+		wantHasAnyType     clientmetric.Type
+		wantNumErroredType clientmetric.Type
+	}{
+		{
+			name:               "windows-device",
+			scope:              setting.DeviceSetting,
+			osOverride:         "windows",
+			wantHasAnyName:     "windows_syspolicy_any",
+			wantHasAnyType:     clientmetric.TypeGauge,
+			wantNumErroredName: "windows_syspolicy_errors",
+			wantNumErroredType: clientmetric.TypeCounter,
+		},
+		{
+			name:               "windows-profile",
+			scope:              setting.ProfileSetting,
+			osOverride:         "windows",
+			wantHasAnyName:     "windows_syspolicy_profile_any",
+			wantHasAnyType:     clientmetric.TypeGauge,
+			wantNumErroredName: "windows_syspolicy_profile_errors",
+			wantNumErroredType: clientmetric.TypeCounter,
+		},
+		{
+			name:               "windows-user",
+			scope:              setting.UserSetting,
+			osOverride:         "windows",
+			wantHasAnyName:     "windows_syspolicy_user_any",
+			wantHasAnyType:     clientmetric.TypeGauge,
+			wantNumErroredName: "windows_syspolicy_user_errors",
+			wantNumErroredType: clientmetric.TypeCounter,
+		},
+		{
+			name:               "android-device",
+			scope:              setting.DeviceSetting,
+			osOverride:         "android",
+			wantHasAnyName:     "android_syspolicy_any",
+			wantHasAnyType:     clientmetric.TypeGauge,
+			wantNumErroredName: "android_syspolicy_errors",
+			wantNumErroredType: clientmetric.TypeCounter,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			internal.OSForTesting.SetForTest(t, tt.osOverride, nil)
+			metrics := newScopeMetrics(tt.scope)
+			hasAny, ok := metrics.hasAny.(*funcMetric)
+			if !ok {
+				t.Fatal("hasAny is not a funcMetric")
+			}
+			numErrored, ok := metrics.numErrored.(*funcMetric)
+			if !ok {
+				t.Fatal("numErrored is not a funcMetric")
+			}
+			if hasAny.name != tt.wantHasAnyName {
+				t.Errorf("hasAny.Name: got %q, want %q", hasAny.name, tt.wantHasAnyName)
+			}
+			if hasAny.typ != tt.wantHasAnyType {
+				t.Errorf("hasAny.Type: got %q, want %q", hasAny.typ, tt.wantHasAnyType)
+			}
+			if numErrored.name != tt.wantNumErroredName {
+				t.Errorf("numErrored.Name: got %q, want %q", numErrored.name, tt.wantNumErroredName)
+			}
+			if numErrored.typ != tt.wantNumErroredType {
+				t.Errorf("hasAny.Type: got %q, want %q", numErrored.typ, tt.wantNumErroredType)
+			}
+		})
+	}
+}
+
+type testSettingDetails struct {
+	definition *setting.Definition
+	origin     *setting.Origin
+	value      any
+	err        error
+}
+
+func TestReportMetrics(t *testing.T) {
+	tests := []struct {
+		name             string
+		osOverride       string
+		useMetrics       bool
+		settings         []testSettingDetails
+		wantMetrics      []TestState
+		wantResetMetrics []TestState
+	}{
+		{
+			name:        "none",
+			osOverride:  "windows",
+			settings:    []testSettingDetails{},
+			wantMetrics: []TestState{},
+		},
+		{
+			name:       "single-value",
+			osOverride: "windows",
+			settings: []testSettingDetails{
+				{
+					definition: setting.NewDefinition("TestSetting01", setting.DeviceSetting, setting.IntegerValue),
+					origin:     setting.NewOrigin(setting.DeviceScope),
+					value:      42,
+				},
+			},
+			wantMetrics: []TestState{
+				{"windows_syspolicy_any", 1},
+				{"windows_syspolicy_TestSetting01", 1},
+			},
+			wantResetMetrics: []TestState{
+				{"windows_syspolicy_any", 0},
+				{"windows_syspolicy_TestSetting01", 0},
+			},
+		},
+		{
+			name:       "single-error",
+			osOverride: "windows",
+			settings: []testSettingDetails{
+				{
+					definition: setting.NewDefinition("TestSetting02", setting.DeviceSetting, setting.IntegerValue),
+					origin:     setting.NewOrigin(setting.DeviceScope),
+					err:        errors.New("bang!"),
+				},
+			},
+			wantMetrics: []TestState{
+				{"windows_syspolicy_errors", 1},
+				{"windows_syspolicy_TestSetting02_error", 1},
+			},
+			wantResetMetrics: []TestState{
+				{"windows_syspolicy_errors", 1},
+				{"windows_syspolicy_TestSetting02_error", 0},
+			},
+		},
+		{
+			name:       "value-and-error",
+			osOverride: "windows",
+			settings: []testSettingDetails{
+				{
+					definition: setting.NewDefinition("TestSetting01", setting.DeviceSetting, setting.IntegerValue),
+					origin:     setting.NewOrigin(setting.DeviceScope),
+					value:      42,
+				},
+				{
+					definition: setting.NewDefinition("TestSetting02", setting.DeviceSetting, setting.IntegerValue),
+					origin:     setting.NewOrigin(setting.DeviceScope),
+					err:        errors.New("bang!"),
+				},
+			},
+
+			wantMetrics: []TestState{
+				{"windows_syspolicy_any", 1},
+				{"windows_syspolicy_errors", 1},
+				{"windows_syspolicy_TestSetting01", 1},
+				{"windows_syspolicy_TestSetting02_error", 1},
+			},
+			wantResetMetrics: []TestState{
+				{"windows_syspolicy_any", 0},
+				{"windows_syspolicy_errors", 1},
+				{"windows_syspolicy_TestSetting01", 0},
+				{"windows_syspolicy_TestSetting02_error", 0},
+			},
+		},
+		{
+			name:       "two-values",
+			osOverride: "windows",
+			settings: []testSettingDetails{
+				{
+					definition: setting.NewDefinition("TestSetting01", setting.DeviceSetting, setting.IntegerValue),
+					origin:     setting.NewOrigin(setting.DeviceScope),
+					value:      42,
+				},
+				{
+					definition: setting.NewDefinition("TestSetting02", setting.DeviceSetting, setting.IntegerValue),
+					origin:     setting.NewOrigin(setting.DeviceScope),
+					value:      17,
+				},
+			},
+			wantMetrics: []TestState{
+				{"windows_syspolicy_any", 1},
+				{"windows_syspolicy_TestSetting01", 1},
+				{"windows_syspolicy_TestSetting02", 1},
+			},
+			wantResetMetrics: []TestState{
+				{"windows_syspolicy_any", 0},
+				{"windows_syspolicy_TestSetting01", 0},
+				{"windows_syspolicy_TestSetting02", 0},
+			},
+		},
+		{
+			name:       "two-errors",
+			osOverride: "windows",
+			settings: []testSettingDetails{
+				{
+					definition: setting.NewDefinition("TestSetting01", setting.DeviceSetting, setting.IntegerValue),
+					origin:     setting.NewOrigin(setting.DeviceScope),
+					err:        errors.New("bang!"),
+				},
+				{
+					definition: setting.NewDefinition("TestSetting02", setting.DeviceSetting, setting.IntegerValue),
+					origin:     setting.NewOrigin(setting.DeviceScope),
+					err:        errors.New("bang!"),
+				},
+			},
+			wantMetrics: []TestState{
+				{"windows_syspolicy_errors", 2},
+				{"windows_syspolicy_TestSetting01_error", 1},
+				{"windows_syspolicy_TestSetting02_error", 1},
+			},
+			wantResetMetrics: []TestState{
+				{"windows_syspolicy_errors", 2},
+				{"windows_syspolicy_TestSetting01_error", 0},
+				{"windows_syspolicy_TestSetting02_error", 0},
+			},
+		},
+		{
+			name:       "multi-scope",
+			osOverride: "windows",
+			settings: []testSettingDetails{
+				{
+					definition: setting.NewDefinition("TestSetting01", setting.ProfileSetting, setting.IntegerValue),
+					origin:     setting.NewOrigin(setting.DeviceScope),
+					value:      42,
+				},
+				{
+					definition: setting.NewDefinition("TestSetting02", setting.ProfileSetting, setting.IntegerValue),
+					origin:     setting.NewOrigin(setting.CurrentProfileScope),
+					err:        errors.New("bang!"),
+				},
+				{
+					definition: setting.NewDefinition("TestSetting03", setting.UserSetting, setting.IntegerValue),
+					origin:     setting.NewOrigin(setting.CurrentUserScope),
+					value:      17,
+				},
+			},
+			wantMetrics: []TestState{
+				{"windows_syspolicy_any", 1},
+				{"windows_syspolicy_profile_errors", 1},
+				{"windows_syspolicy_user_any", 1},
+				{"windows_syspolicy_TestSetting01", 1},
+				{"windows_syspolicy_TestSetting02_profile_error", 1},
+				{"windows_syspolicy_TestSetting03_user", 1},
+			},
+			wantResetMetrics: []TestState{
+				{"windows_syspolicy_any", 0},
+				{"windows_syspolicy_profile_errors", 1},
+				{"windows_syspolicy_user_any", 0},
+				{"windows_syspolicy_TestSetting01", 0},
+				{"windows_syspolicy_TestSetting02_profile_error", 0},
+				{"windows_syspolicy_TestSetting03_user", 0},
+			},
+		},
+		{
+			name:       "report-metrics-on-android",
+			osOverride: "android",
+			settings: []testSettingDetails{
+				{
+					definition: setting.NewDefinition("TestSetting01", setting.DeviceSetting, setting.IntegerValue),
+					origin:     setting.NewOrigin(setting.DeviceScope),
+					value:      42,
+				},
+			},
+			wantMetrics: []TestState{
+				{"android_syspolicy_any", 1},
+				{"android_syspolicy_TestSetting01", 1},
+			},
+			wantResetMetrics: []TestState{
+				{"android_syspolicy_any", 0},
+				{"android_syspolicy_TestSetting01", 0},
+			},
+		},
+		{
+			name:       "do-not-report-metrics-on-macos",
+			osOverride: "macos",
+			settings: []testSettingDetails{
+				{
+					definition: setting.NewDefinition("TestSetting01", setting.DeviceSetting, setting.IntegerValue),
+					origin:     setting.NewOrigin(setting.DeviceScope),
+					value:      42,
+				},
+			},
+
+			wantMetrics: []TestState{}, // none reported
+		},
+		{
+			name:       "do-not-report-metrics-on-ios",
+			osOverride: "ios",
+			settings: []testSettingDetails{
+				{
+					definition: setting.NewDefinition("TestSetting01", setting.DeviceSetting, setting.IntegerValue),
+					origin:     setting.NewOrigin(setting.DeviceScope),
+					value:      42,
+				},
+			},
+
+			wantMetrics: []TestState{}, // none reported
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Reset the lazy value so it'll be re-evaluated with the osOverride.
+			lazyReportMetrics = lazy.SyncValue[bool]{}
+			t.Cleanup(func() {
+				// Also reset it during the cleanup.
+				lazyReportMetrics = lazy.SyncValue[bool]{}
+			})
+			internal.OSForTesting.SetForTest(t, tt.osOverride, nil)
+
+			h := NewTestHandler(t)
+			SetHooksForTest(t, h.AddMetric, h.SetMetric)
+
+			for _, s := range tt.settings {
+				if s.err != nil {
+					ReportError(s.origin, s.definition, s.err)
+				} else {
+					ReportConfigured(s.origin, s.definition, s.value)
+				}
+			}
+			h.MustEqual(tt.wantMetrics...)
+
+			for _, s := range tt.settings {
+				Reset(s.origin)
+				ReportNotConfigured(s.origin, s.definition)
+			}
+			h.MustEqual(tt.wantResetMetrics...)
+		})
+	}
+}

util/syspolicy/internal/metrics/test_handler.go

@@ -0,0 +1,88 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package metrics
+
+import (
+	"strings"
+
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/set"
+	"tailscale.com/util/syspolicy/internal"
+)
+
+// TestState represents a metric name and its expected value.
+type TestState struct {
+	Name  string // `$os` in the name will be replaced by the actual operating system name.
+	Value int64
+}
+
+// TestHandler facilitates testing of the code that uses metrics.
+type TestHandler struct {
+	t internal.TB
+
+	m map[string]int64
+}
+
+// NewTestHandler returns a new TestHandler.
+func NewTestHandler(t internal.TB) *TestHandler {
+	return &TestHandler{t, make(map[string]int64)}
+}
+
+// AddMetric increments the metric with the specified name and type by delta d.
+func (h *TestHandler) AddMetric(name string, typ clientmetric.Type, d int64) {
+	h.t.Helper()
+	if typ == clientmetric.TypeCounter && d < 0 {
+		h.t.Fatalf("an attempt was made to decrement a counter metric %q", name)
+	}
+	if v, ok := h.m[name]; ok || d != 0 {
+		h.m[name] = v + d
+	}
+}
+
+// SetMetric sets the metric with the specified name and type to the value v.
+func (h *TestHandler) SetMetric(name string, typ clientmetric.Type, v int64) {
+	h.t.Helper()
+	if typ == clientmetric.TypeCounter {
+		h.t.Fatalf("an attempt was made to set a counter metric %q", name)
+	}
+	if _, ok := h.m[name]; ok || v != 0 {
+		h.m[name] = v
+	}
+}
+
+// MustEqual fails the test if the actual metric state differs from the specified state.
+func (h *TestHandler) MustEqual(metrics ...TestState) {
+	h.t.Helper()
+	h.MustContain(metrics...)
+	h.mustNoExtra(metrics...)
+}
+
+// MustContain fails the test if the specified metrics are not set or have
+// different values than specified. It permits other metrics to be set in
+// addition to the ones being tested.
+func (h *TestHandler) MustContain(metrics ...TestState) {
+	h.t.Helper()
+	for _, m := range metrics {
+		name := strings.ReplaceAll(m.Name, "$os", internal.OS())
+		v, ok := h.m[name]
+		if !ok {
+			h.t.Errorf("%q: got (none), want %v", name, m.Value)
+		} else if v != m.Value {
+			h.t.Fatalf("%q: got %v, want %v", name, v, m.Value)
+		}
+	}
+}
+
+func (h *TestHandler) mustNoExtra(metrics ...TestState) {
+	h.t.Helper()
+	s := make(set.Set[string])
+	for i := range metrics {
+		s.Add(strings.ReplaceAll(metrics[i].Name, "$os", internal.OS()))
+	}
+	for n, v := range h.m {
+		if !s.Contains(n) {
+			h.t.Errorf("%q: got %v, want (none)", n, v)
+		}
+	}
+}