From 48ae5d58ac7507bb350e7e0d01dc6e6dec870995 Mon Sep 17 00:00:00 2001 From: Stefan Benz <46600784+stebenz@users.noreply.github.com> Date: Wed, 25 Oct 2023 14:09:15 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20add=20activity=20logs=20on=20user=20act?= =?UTF-8?q?ions=20with=20authentication,=20resource=E2=80=A6=20(#6748)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add activity logs on user actions with authentication, resourceAPI and sessionAPI * feat: add activity logs on user actions with authentication, resourceAPI and sessionAPI * feat: add activity logs on user actions with authentication, resourceAPI and sessionAPI * feat: add activity logs on user actions with authentication, resourceAPI and sessionAPI * feat: add activity logs on user actions with authentication, resourceAPI and sessionAPI * fix: add unit tests to info package for context changes * fix: add activity_interceptor.go suggestion Co-authored-by: Tim Möhlmann * fix: refactoring and fixes through PR review * fix: add auth service to lists of resourceAPIs --------- Co-authored-by: Tim Möhlmann Co-authored-by: Fabi --- cmd/start/start.go | 5 +- internal/activity/activity.go | 73 +++++++++ internal/api/grpc/admin/server.go | 4 + internal/api/grpc/management/server.go | 4 + .../server/middleware/activity_interceptor.go | 35 ++++ internal/api/grpc/server/server.go | 1 + internal/api/grpc/session/v2/session.go | 6 + .../http/middleware/activity_interceptor.go | 32 ++++ internal/api/info/info.go | 43 +++++ internal/api/info/info_test.go | 117 +++++++++++++ internal/api/oidc/auth_request.go | 18 ++ internal/api/saml/storage.go | 4 + .../handlers/quota_notifier_test.go | 155 ++++++++++++++++++ 13 files changed, 496 insertions(+), 1 deletion(-) create mode 100644 internal/activity/activity.go create mode 100644 internal/api/grpc/server/middleware/activity_interceptor.go create mode 100644 internal/api/http/middleware/activity_interceptor.go create mode 100644 internal/api/info/info.go create mode 100644 internal/api/info/info_test.go create mode 100644 internal/notification/handlers/quota_notifier_test.go diff --git a/cmd/start/start.go b/cmd/start/start.go index 078075a17b..9da5c114ff 100644 --- a/cmd/start/start.go +++ b/cmd/start/start.go @@ -316,8 +316,11 @@ func startAPIs( authZRepo, queries, } + oidcPrefixes := []string{"/.well-known/openid-configuration", "/oidc/v1", "/oauth/v2"} // always set the origin in the context if available in the http headers, no matter for what protocol router.Use(middleware.OriginHandler) + // adds used HTTPPathPattern and RequestMethod to context + router.Use(middleware.ActivityHandler(append(oidcPrefixes, saml.HandlerPrefix, admin.GatewayPathPrefix(), management.GatewayPathPrefix()))) verifier := internal_authz.Start(repo, http_util.BuildHTTP(config.ExternalDomain, config.ExternalPort, config.ExternalSecure), config.SystemAPIUsers) tlsConfig, err := config.TLS.Config() if err != nil { @@ -413,7 +416,7 @@ func startAPIs( if err != nil { return fmt.Errorf("unable to start oidc provider: %w", err) } - apis.RegisterHandlerPrefixes(oidcProvider.HttpHandler(), "/.well-known/openid-configuration", "/oidc/v1", "/oauth/v2") + apis.RegisterHandlerPrefixes(oidcProvider.HttpHandler(), oidcPrefixes...) samlProvider, err := saml.NewProvider(config.SAML, config.ExternalSecure, commands, queries, authRepo, keys.OIDC, keys.SAML, eventstore, dbClient, instanceInterceptor.Handler, userAgentInterceptor, limitingAccessInterceptor) if err != nil { diff --git a/internal/activity/activity.go b/internal/activity/activity.go new file mode 100644 index 0000000000..4ceb681cf7 --- /dev/null +++ b/internal/activity/activity.go @@ -0,0 +1,73 @@ +package activity + +import ( + "context" + + "github.com/zitadel/logging" + + "github.com/zitadel/zitadel/internal/api/authz" + http_utils "github.com/zitadel/zitadel/internal/api/http" + "github.com/zitadel/zitadel/internal/api/info" +) + +const ( + Activity = "activity" +) + +type TriggerMethod int + +const ( + Unspecified TriggerMethod = iota + ResourceAPI + OIDCAccessToken + OIDCRefreshToken + SessionAPI + SAMLResponse +) + +func (t TriggerMethod) String() string { + switch t { + case Unspecified: + return "unspecified" + case ResourceAPI: + return "resourceAPI" + case OIDCRefreshToken: + return "refreshToken" + case OIDCAccessToken: + return "accessToken" + case SessionAPI: + return "sessionAPI" + case SAMLResponse: + return "samlResponse" + default: + return "unknown" + } +} + +func Trigger(ctx context.Context, orgID, userID string, trigger TriggerMethod) { + triggerLog(authz.GetInstance(ctx).InstanceID(), orgID, userID, http_utils.ComposedOrigin(ctx), trigger, info.ActivityInfoFromContext(ctx)) +} + +func TriggerWithContext(ctx context.Context, trigger TriggerMethod) { + data := authz.GetCtxData(ctx) + ai := info.ActivityInfoFromContext(ctx) + // if GRPC call, path is prefilled with the grpc fullmethod and method is empty + if ai.Method == "" { + ai.Method = ai.Path + ai.Path = "" + } + triggerLog(authz.GetInstance(ctx).InstanceID(), data.OrgID, data.UserID, http_utils.ComposedOrigin(ctx), trigger, ai) +} + +func triggerLog(instanceID, orgID, userID, domain string, trigger TriggerMethod, ai *info.ActivityInfo) { + logging.WithFields( + "instance", instanceID, + "org", orgID, + "user", userID, + "domain", domain, + "trigger", trigger.String(), + "method", ai.Method, + "path", ai.Path, + "requestMethod", ai.RequestMethod, + ).Info(Activity) +} diff --git a/internal/api/grpc/admin/server.go b/internal/api/grpc/admin/server.go index 7fe8e32f34..e367765983 100644 --- a/internal/api/grpc/admin/server.go +++ b/internal/api/grpc/admin/server.go @@ -79,5 +79,9 @@ func (s *Server) RegisterGateway() server.RegisterGatewayFunc { } func (s *Server) GatewayPathPrefix() string { + return GatewayPathPrefix() +} + +func GatewayPathPrefix() string { return "/admin/v1" } diff --git a/internal/api/grpc/management/server.go b/internal/api/grpc/management/server.go index b0f9879278..638da236b7 100644 --- a/internal/api/grpc/management/server.go +++ b/internal/api/grpc/management/server.go @@ -71,5 +71,9 @@ func (s *Server) RegisterGateway() server.RegisterGatewayFunc { } func (s *Server) GatewayPathPrefix() string { + return GatewayPathPrefix() +} + +func GatewayPathPrefix() string { return "/management/v1" } diff --git a/internal/api/grpc/server/middleware/activity_interceptor.go b/internal/api/grpc/server/middleware/activity_interceptor.go new file mode 100644 index 0000000000..7b6f050265 --- /dev/null +++ b/internal/api/grpc/server/middleware/activity_interceptor.go @@ -0,0 +1,35 @@ +package middleware + +import ( + "context" + "slices" + "strings" + + "google.golang.org/grpc" + + "github.com/zitadel/zitadel/internal/activity" +) + +func ActivityInterceptor() grpc.UnaryServerInterceptor { + return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + resp, err := handler(ctx, req) + if isResourceAPI(info.FullMethod) { + activity.TriggerWithContext(ctx, activity.ResourceAPI) + } + return resp, err + } +} + +var resourcePrefixes = []string{ + "/zitadel.management.v1.ManagementService/", + "/zitadel.admin.v1.AdminService/", + "/zitadel.user.v2beta.UserService/", + "/zitadel.settings.v2beta.SettingsService/", + "/zitadel.auth.v1.AuthService/", +} + +func isResourceAPI(method string) bool { + return slices.ContainsFunc(resourcePrefixes, func(prefix string) bool { + return strings.HasPrefix(method, prefix) + }) +} diff --git a/internal/api/grpc/server/server.go b/internal/api/grpc/server/server.go index 9d7deb28ea..96c8066d1e 100644 --- a/internal/api/grpc/server/server.go +++ b/internal/api/grpc/server/server.go @@ -58,6 +58,7 @@ func CreateServer( middleware.TranslationHandler(), middleware.ValidationHandler(), middleware.ServiceHandler(), + middleware.ActivityInterceptor(), ), ), } diff --git a/internal/api/grpc/session/v2/session.go b/internal/api/grpc/session/v2/session.go index f98983936d..c766b3ef18 100644 --- a/internal/api/grpc/session/v2/session.go +++ b/internal/api/grpc/session/v2/session.go @@ -9,6 +9,8 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/muhlemmer/gu" + + "github.com/zitadel/zitadel/internal/activity" "github.com/zitadel/zitadel/internal/api/authz" "github.com/zitadel/zitadel/internal/api/grpc/object/v2" "github.com/zitadel/zitadel/internal/command" @@ -57,6 +59,7 @@ func (s *Server) CreateSession(ctx context.Context, req *session.CreateSessionRe if err != nil { return nil, err } + return &session.CreateSessionResponse{ Details: object.DomainToDetailsPb(set.ObjectDetails), SessionId: set.ID, @@ -310,6 +313,9 @@ func (s *Server) checksToCommand(ctx context.Context, checks *session.Checks) ([ if err != nil { return nil, err } + + // trigger activity log for session for user + activity.Trigger(ctx, user.ResourceOwner, user.ID, activity.SessionAPI) sessionChecks = append(sessionChecks, command.CheckUser(user.ID)) } if password := checks.GetPassword(); password != nil { diff --git a/internal/api/http/middleware/activity_interceptor.go b/internal/api/http/middleware/activity_interceptor.go new file mode 100644 index 0000000000..7cba3db421 --- /dev/null +++ b/internal/api/http/middleware/activity_interceptor.go @@ -0,0 +1,32 @@ +package middleware + +import ( + "net/http" + "strings" + + "github.com/zitadel/zitadel/internal/api/info" +) + +func ActivityHandler(handlerPrefixes []string) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + activityInfo := info.ActivityInfoFromContext(ctx) + hasPrefix := false + // only add path to context if handler is called + for _, prefix := range handlerPrefixes { + if strings.HasPrefix(r.URL.Path, prefix) { + activityInfo.SetPath(r.URL.Path) + hasPrefix = true + break + } + } + // last call is with grpc method as path + if !hasPrefix { + activityInfo.SetMethod(r.URL.Path) + } + ctx = activityInfo.SetRequestMethod(r.Method).IntoContext(ctx) + next.ServeHTTP(w, r.WithContext(ctx)) + }) + } +} diff --git a/internal/api/info/info.go b/internal/api/info/info.go new file mode 100644 index 0000000000..53d53518b1 --- /dev/null +++ b/internal/api/info/info.go @@ -0,0 +1,43 @@ +package info + +import ( + "context" +) + +type activityInfoKey struct{} + +type ActivityInfo struct { + Method string + Path string + RequestMethod string +} + +func (a *ActivityInfo) IntoContext(ctx context.Context) context.Context { + return context.WithValue(ctx, activityInfoKey{}, a) +} + +func ActivityInfoFromContext(ctx context.Context) *ActivityInfo { + m := ctx.Value(activityInfoKey{}) + if m == nil { + return &ActivityInfo{} + } + ai, ok := m.(*ActivityInfo) + if !ok { + return &ActivityInfo{} + } + return ai +} + +func (a *ActivityInfo) SetMethod(method string) *ActivityInfo { + a.Method = method + return a +} +func (a *ActivityInfo) SetPath(path string) *ActivityInfo { + a.Path = path + return a +} + +func (a *ActivityInfo) SetRequestMethod(method string) *ActivityInfo { + a.RequestMethod = method + return a +} diff --git a/internal/api/info/info_test.go b/internal/api/info/info_test.go new file mode 100644 index 0000000000..0b9ecc5fd6 --- /dev/null +++ b/internal/api/info/info_test.go @@ -0,0 +1,117 @@ +package info + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_ActivityInfo(t *testing.T) { + type args struct { + ctx context.Context + ok bool + path string + method string + requestMethod string + } + type want struct { + ok bool + path string + method string + requestMethod string + } + tests := []struct { + name string + args args + want want + }{ + { + "already set", + args{ + ctx: ctxWithActivityInfo(context.Background(), "set", "set", "set"), + ok: false, + }, + want{ + ok: true, + path: "set", + method: "set", + requestMethod: "set", + }, + }, + { + "not set, empty", + args{ + ctx: context.Background(), + ok: false, + }, + want{ + ok: true, + }, + }, + { + "set empty", + args{ + ctx: context.Background(), + ok: true, + }, + want{ + ok: true, + }, + }, + { + "set", + args{ + ctx: context.Background(), + ok: true, + path: "set", + method: "set", + requestMethod: "set", + }, + want{ + ok: true, + path: "set", + method: "set", + requestMethod: "set", + }, + }, + { + "reset", + args{ + ctx: ctxWithActivityInfo(context.Background(), "set", "set", "set"), + ok: true, + path: "set2", + method: "set2", + requestMethod: "set2", + }, + want{ + ok: true, + path: "set2", + method: "set2", + requestMethod: "set2", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ai := &ActivityInfo{} + ai.SetMethod(tt.args.method).SetPath(tt.args.path).SetRequestMethod(tt.args.requestMethod) + if tt.args.ok { + tt.args.ctx = ai.IntoContext(tt.args.ctx) + } + + res := ActivityInfoFromContext(tt.args.ctx) + if tt.want.ok { + assert.NotNil(t, res) + } + assert.Equal(t, tt.want.path, res.Path) + assert.Equal(t, tt.want.method, res.Method) + assert.Equal(t, tt.want.requestMethod, res.RequestMethod) + }) + } +} + +func ctxWithActivityInfo(ctx context.Context, method, path, requestMethod string) context.Context { + ai := &ActivityInfo{} + return ai.SetPath(path).SetRequestMethod(requestMethod).SetMethod(method).IntoContext(ctx) +} diff --git a/internal/api/oidc/auth_request.go b/internal/api/oidc/auth_request.go index 56c0902b11..267f674b2f 100644 --- a/internal/api/oidc/auth_request.go +++ b/internal/api/oidc/auth_request.go @@ -10,6 +10,7 @@ import ( "github.com/zitadel/oidc/v3/pkg/oidc" "github.com/zitadel/oidc/v3/pkg/op" + "github.com/zitadel/zitadel/internal/activity" "github.com/zitadel/zitadel/internal/api/authz" http_utils "github.com/zitadel/zitadel/internal/api/http" "github.com/zitadel/zitadel/internal/api/http/middleware" @@ -196,6 +197,8 @@ func (o *OPStorage) CreateAccessToken(ctx context.Context, req op.TokenRequest) applicationID = authReq.ApplicationID userOrgID = authReq.UserOrgID case *AuthRequestV2: + // trigger activity log for authentication for user + activity.Trigger(ctx, "", authReq.CurrentAuthRequest.UserID, activity.OIDCAccessToken) return o.command.AddOIDCSessionAccessToken(setContextUserSystem(ctx), authReq.GetID()) } @@ -208,6 +211,9 @@ func (o *OPStorage) CreateAccessToken(ctx context.Context, req op.TokenRequest) if err != nil { return "", time.Time{}, err } + + // trigger activity log for authentication for user + activity.Trigger(ctx, userOrgID, req.GetSubject(), activity.OIDCAccessToken) return resp.TokenID, resp.Expiration, nil } @@ -218,8 +224,12 @@ func (o *OPStorage) CreateAccessAndRefreshTokens(ctx context.Context, req op.Tok // handle V2 request directly switch tokenReq := req.(type) { case *AuthRequestV2: + // trigger activity log for authentication for user + activity.Trigger(ctx, "", tokenReq.GetSubject(), activity.OIDCRefreshToken) return o.command.AddOIDCSessionRefreshAndAccessToken(setContextUserSystem(ctx), tokenReq.GetID()) case *RefreshTokenRequestV2: + // trigger activity log for authentication for user + activity.Trigger(ctx, "", tokenReq.GetSubject(), activity.OIDCRefreshToken) return o.command.ExchangeOIDCSessionRefreshAndAccessToken(setContextUserSystem(ctx), tokenReq.OIDCSessionWriteModel.AggregateID, refreshToken, tokenReq.RequestedScopes) } @@ -246,6 +256,9 @@ func (o *OPStorage) CreateAccessAndRefreshTokens(ctx context.Context, req op.Tok } return "", "", time.Time{}, err } + + // trigger activity log for authentication for user + activity.Trigger(ctx, userOrgID, req.GetSubject(), activity.OIDCRefreshToken) return resp.TokenID, token, resp.Expiration, nil } @@ -274,6 +287,8 @@ func (o *OPStorage) TokenRequestByRefreshToken(ctx context.Context, refreshToken if err != nil { return nil, err } + // trigger activity log for authentication for user + activity.Trigger(ctx, "", oidcSession.UserID, activity.OIDCRefreshToken) return &RefreshTokenRequestV2{OIDCSessionWriteModel: oidcSession}, nil } @@ -281,6 +296,9 @@ func (o *OPStorage) TokenRequestByRefreshToken(ctx context.Context, refreshToken if err != nil { return nil, err } + + // trigger activity log for use of refresh token for user + activity.Trigger(ctx, tokenView.ResourceOwner, tokenView.UserID, activity.OIDCRefreshToken) return RefreshTokenRequestFromBusiness(tokenView), nil } diff --git a/internal/api/saml/storage.go b/internal/api/saml/storage.go index c5d348ccb7..87e6960d48 100644 --- a/internal/api/saml/storage.go +++ b/internal/api/saml/storage.go @@ -15,6 +15,7 @@ import ( "github.com/zitadel/zitadel/internal/actions" "github.com/zitadel/zitadel/internal/actions/object" + "github.com/zitadel/zitadel/internal/activity" "github.com/zitadel/zitadel/internal/api/http/middleware" "github.com/zitadel/zitadel/internal/auth/repository" "github.com/zitadel/zitadel/internal/command" @@ -148,6 +149,9 @@ func (p *Storage) SetUserinfoWithUserID(ctx context.Context, applicationID strin } setUserinfo(user, userinfo, attributes, customAttributes) + + // trigger activity log for authentication for user + activity.Trigger(ctx, user.ResourceOwner, user.ID, activity.SAMLResponse) return nil } diff --git a/internal/notification/handlers/quota_notifier_test.go b/internal/notification/handlers/quota_notifier_test.go new file mode 100644 index 0000000000..059d4cf041 --- /dev/null +++ b/internal/notification/handlers/quota_notifier_test.go @@ -0,0 +1,155 @@ +//go:build integration + +package handlers_test + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/zitadel/zitadel/internal/repository/quota" + "github.com/zitadel/zitadel/pkg/grpc/admin" + quota_pb "github.com/zitadel/zitadel/pkg/grpc/quota" + "github.com/zitadel/zitadel/pkg/grpc/system" +) + +func TestServer_QuotaNotification_Limit(t *testing.T) { + _, instanceID, iamOwnerCtx := Tester.UseIsolatedInstance(CTX, SystemCTX) + amount := 10 + percent := 50 + percentAmount := amount * percent / 100 + + _, err := Tester.Client.System.AddQuota(SystemCTX, &system.AddQuotaRequest{ + InstanceId: instanceID, + Unit: quota_pb.Unit_UNIT_REQUESTS_ALL_AUTHENTICATED, + From: timestamppb.Now(), + ResetInterval: durationpb.New(time.Minute * 5), + Amount: uint64(amount), + Limit: true, + Notifications: []*quota_pb.Notification{ + { + Percent: uint32(percent), + Repeat: true, + CallUrl: "http://localhost:8082", + }, + { + Percent: 100, + Repeat: true, + CallUrl: "http://localhost:8082", + }, + }, + }) + require.NoError(t, err) + + for i := 0; i < percentAmount; i++ { + _, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{}) + if err != nil { + require.NoError(t, fmt.Errorf("error in %d call of %d: %f", i, percentAmount, err)) + } + } + awaitNotification(t, time.Now(), Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, percent) + + for i := 0; i < (amount - percentAmount); i++ { + _, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{}) + require.NoError(t, err) + } + awaitNotification(t, time.Now(), Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, 100) + + _, limitErr := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{}) + require.Error(t, limitErr) +} + +func TestServer_QuotaNotification_NoLimit(t *testing.T) { + _, instanceID, iamOwnerCtx := Tester.UseIsolatedInstance(CTX, SystemCTX) + amount := 10 + percent := 50 + percentAmount := amount * percent / 100 + + _, err := Tester.Client.System.AddQuota(SystemCTX, &system.AddQuotaRequest{ + InstanceId: instanceID, + Unit: quota_pb.Unit_UNIT_REQUESTS_ALL_AUTHENTICATED, + From: timestamppb.Now(), + ResetInterval: durationpb.New(time.Minute * 5), + Amount: uint64(amount), + Limit: false, + Notifications: []*quota_pb.Notification{ + { + Percent: uint32(percent), + Repeat: false, + CallUrl: "http://localhost:8082", + }, + { + Percent: 100, + Repeat: true, + CallUrl: "http://localhost:8082", + }, + }, + }) + require.NoError(t, err) + + for i := 0; i < percentAmount; i++ { + _, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{}) + if err != nil { + require.NoError(t, fmt.Errorf("error in %d call of %d: %f", i, percentAmount, err)) + } + } + awaitNotification(t, time.Now(), Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, percent) + + for i := 0; i < (amount - percentAmount); i++ { + _, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{}) + if err != nil { + require.NoError(t, fmt.Errorf("error in %d call of %d: %f", percentAmount+i, amount, err)) + } + } + awaitNotification(t, time.Now(), Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, 100) + + for i := 0; i < amount; i++ { + _, err := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{}) + if err != nil { + require.NoError(t, fmt.Errorf("error in %d call of %d over limit: %f", i, amount, err)) + } + } + awaitNotification(t, time.Now(), Tester.QuotaNotificationChan, quota.RequestsAllAuthenticated, 200) + + _, limitErr := Tester.Client.Admin.GetDefaultOrg(iamOwnerCtx, &admin.GetDefaultOrgRequest{}) + require.NoError(t, limitErr) +} + +func awaitNotification(t *testing.T, start time.Time, bodies chan []byte, unit quota.Unit, percent int) { + for { + select { + case body := <-bodies: + plain := new(bytes.Buffer) + if err := json.Indent(plain, body, "", " "); err != nil { + t.Fatal(err) + } + t.Log("received notificationDueEvent", plain.String()) + event := struct { + Unit quota.Unit `json:"unit"` + ID string `json:"id"` + CallURL string `json:"callURL"` + PeriodStart time.Time `json:"periodStart"` + Threshold uint16 `json:"threshold"` + Usage uint64 `json:"usage"` + }{} + if err := json.Unmarshal(body, &event); err != nil { + t.Error(err) + } + if event.ID == "" { + continue + } + if event.Unit == unit && event.Threshold == uint16(percent) { + return + } + case <-time.After(20 * time.Second): + t.Fatalf("start %s stop %s timed out waiting for unit %s and percent %d", start.Format(time.RFC3339), time.Now().Format(time.RFC3339), strconv.Itoa(int(unit)), percent) + } + } +}