chore: improve integration tests (#8727)

Improve integration tests:
- spliting the tests in TokenExchange to isolated instances and in
parallel
- corrected some test structure so that the check for Details is no done
anymore if the test already failed
- replace required-calls with assert-calls to not stop the testing
- add gofakeit for application, project and usernames(emails)
- add eventually checks for testing in actions v2, so the request only
get called when the execution is defined
- check for length of results in list/search endpoints to avoid index
errors
This commit is contained in:
Stefan Benz
2024-10-17 23:20:57 +02:00
committed by GitHub
parent cc8d4fe17c
commit 8d97363642
47 changed files with 719 additions and 649 deletions

View File

@@ -65,6 +65,8 @@ func TestServer_ExecutionTarget(t *testing.T) {
targetRequest := instance.CreateTarget(ctx, t, "", urlRequest, domain.TargetTypeCall, false)
instance.SetExecution(ctx, t, conditionRequestFullMethod(fullMethod), executionTargetsSingleTarget(targetRequest.GetDetails().GetId()))
waitForExecutionOnCondition(ctx, t, instance, conditionRequestFullMethod(fullMethod))
// expected response from the GetTarget
expectedResponse := &action.GetTargetResponse{
Target: &action.GetTarget{
@@ -120,6 +122,7 @@ func TestServer_ExecutionTarget(t *testing.T) {
targetResponse := instance.CreateTarget(ctx, t, "", targetResponseURL, domain.TargetTypeCall, false)
instance.SetExecution(ctx, t, conditionResponseFullMethod(fullMethod), executionTargetsSingleTarget(targetResponse.GetDetails().GetId()))
waitForExecutionOnCondition(ctx, t, instance, conditionResponseFullMethod(fullMethod))
return func() {
closeRequest()
closeResponse()
@@ -163,6 +166,7 @@ func TestServer_ExecutionTarget(t *testing.T) {
// GetTarget with used target
request.Id = targetRequest.GetDetails().GetId()
waitForExecutionOnCondition(ctx, t, instance, conditionRequestFullMethod(fullMethod))
return func() {
closeRequest()
}, nil
@@ -232,6 +236,7 @@ func TestServer_ExecutionTarget(t *testing.T) {
targetResponse := instance.CreateTarget(ctx, t, "", targetResponseURL, domain.TargetTypeCall, true)
instance.SetExecution(ctx, t, conditionResponseFullMethod(fullMethod), executionTargetsSingleTarget(targetResponse.GetDetails().GetId()))
waitForExecutionOnCondition(ctx, t, instance, conditionResponseFullMethod(fullMethod))
return func() {
closeResponse()
}, nil
@@ -250,25 +255,20 @@ func TestServer_ExecutionTarget(t *testing.T) {
require.NoError(t, err)
defer close()
}
retryDuration := 5 * time.Second
if ctxDeadline, ok := isolatedIAMOwnerCTX.Deadline(); ok {
retryDuration = time.Until(ctxDeadline)
}
retryDuration, tick := integration.WaitForAndTickWithMaxDuration(isolatedIAMOwnerCTX, 5*time.Second)
require.EventuallyWithT(t, func(ttt *assert.CollectT) {
got, err := instance.Client.ActionV3Alpha.GetTarget(tt.ctx, tt.req)
if tt.wantErr {
assert.Error(ttt, err, "Error: "+err.Error())
} else {
assert.NoError(ttt, err)
}
if err != nil {
require.Error(ttt, err)
return
}
require.NoError(ttt, err)
integration.AssertResourceDetails(t, tt.want.GetTarget().GetDetails(), got.GetTarget().GetDetails())
assert.Equal(t, tt.want.GetTarget().GetConfig(), got.GetTarget().GetConfig())
}, retryDuration, time.Millisecond*100, "timeout waiting for expected execution result")
integration.AssertResourceDetails(ttt, tt.want.GetTarget().GetDetails(), got.GetTarget().GetDetails())
tt.want.Target.Details = got.GetTarget().GetDetails()
assert.EqualExportedValues(ttt, tt.want.GetTarget().GetConfig(), got.GetTarget().GetConfig())
}, retryDuration, tick, "timeout waiting for expected execution result")
if tt.clean != nil {
tt.clean(tt.ctx)
@@ -277,6 +277,26 @@ func TestServer_ExecutionTarget(t *testing.T) {
}
}
func waitForExecutionOnCondition(ctx context.Context, t *testing.T, instance *integration.Instance, condition *action.Condition) {
retryDuration, tick := integration.WaitForAndTickWithMaxDuration(ctx, 5*time.Second)
require.EventuallyWithT(t, func(ttt *assert.CollectT) {
got, err := instance.Client.ActionV3Alpha.SearchExecutions(ctx, &action.SearchExecutionsRequest{
Filters: []*action.ExecutionSearchFilter{
{Filter: &action.ExecutionSearchFilter_InConditionsFilter{
InConditionsFilter: &action.InConditionsFilter{Conditions: []*action.Condition{condition}},
}},
},
})
if !assert.NoError(ttt, err) {
return
}
if assert.Len(ttt, got.GetResult(), 1) {
return
}
}, retryDuration, tick, "timeout waiting for expected execution result")
return
}
func conditionRequestFullMethod(fullMethod string) *action.Condition {
return &action.Condition{
ConditionType: &action.Condition_Request{

View File

@@ -196,7 +196,6 @@ func TestServer_SetExecution_Request(t *testing.T) {
require.Error(t, err)
return
}
require.NoError(t, err)
integration.AssertResourceDetails(t, tt.want.Details, got.Details)

View File

@@ -216,16 +216,20 @@ func TestServer_GetTarget(t *testing.T) {
err := tt.args.dep(tt.args.ctx, tt.args.req, tt.want)
require.NoError(t, err)
}
got, getErr := instance.Client.ActionV3Alpha.GetTarget(tt.args.ctx, tt.args.req)
if tt.wantErr {
assert.Error(t, getErr, "Error: "+getErr.Error())
} else {
assert.NoError(t, getErr)
retryDuration, tick := integration.WaitForAndTickWithMaxDuration(isolatedIAMOwnerCTX, 5*time.Second)
require.EventuallyWithT(t, func(ttt *assert.CollectT) {
got, err := instance.Client.ActionV3Alpha.GetTarget(tt.args.ctx, tt.args.req)
if tt.wantErr {
require.Error(ttt, err, "Error: "+err.Error())
return
}
require.NoError(ttt, err)
wantTarget := tt.want.GetTarget()
gotTarget := got.GetTarget()
integration.AssertResourceDetails(t, wantTarget.GetDetails(), gotTarget.GetDetails())
assert.Equal(t, wantTarget.GetConfig(), gotTarget.GetConfig())
}
integration.AssertResourceDetails(ttt, wantTarget.GetDetails(), gotTarget.GetDetails())
assert.EqualExportedValues(ttt, wantTarget.GetConfig(), gotTarget.GetConfig())
}, retryDuration, tick, "timeout waiting for expected target result")
})
}
}
@@ -474,31 +478,24 @@ func TestServer_ListTargets(t *testing.T) {
require.NoError(t, err)
}
retryDuration := 5 * time.Second
if ctxDeadline, ok := isolatedIAMOwnerCTX.Deadline(); ok {
retryDuration = time.Until(ctxDeadline)
}
retryDuration, tick := integration.WaitForAndTickWithMaxDuration(isolatedIAMOwnerCTX, 5*time.Second)
require.EventuallyWithT(t, func(ttt *assert.CollectT) {
got, listErr := instance.Client.ActionV3Alpha.SearchTargets(tt.args.ctx, tt.args.req)
if tt.wantErr {
assert.Error(ttt, listErr, "Error: "+listErr.Error())
} else {
assert.NoError(ttt, listErr)
}
if listErr != nil {
require.Error(ttt, listErr, "Error: "+listErr.Error())
return
}
require.NoError(ttt, listErr)
// always first check length, otherwise its failed anyway
if !assert.Len(ttt, got.Result, len(tt.want.Result)) {
return
}
require.Len(ttt, got.Result, len(tt.want.Result))
for i := range tt.want.Result {
integration.AssertResourceDetails(ttt, tt.want.Result[i].GetDetails(), got.Result[i].GetDetails())
assert.Equal(ttt, tt.want.Result[i].GetConfig(), got.Result[i].GetConfig())
assert.EqualExportedValues(ttt, tt.want.Result[i].GetConfig(), got.Result[i].GetConfig())
}
integration.AssertResourceListDetails(ttt, tt.want, got)
}, retryDuration, time.Millisecond*100, "timeout waiting for expected execution result")
}, retryDuration, tick, "timeout waiting for expected execution result")
})
}
}
@@ -866,32 +863,27 @@ func TestServer_SearchExecutions(t *testing.T) {
require.NoError(t, err)
}
retryDuration := 5 * time.Second
if ctxDeadline, ok := isolatedIAMOwnerCTX.Deadline(); ok {
retryDuration = time.Until(ctxDeadline)
}
retryDuration, tick := integration.WaitForAndTickWithMaxDuration(isolatedIAMOwnerCTX, 5*time.Second)
require.EventuallyWithT(t, func(ttt *assert.CollectT) {
got, listErr := instance.Client.ActionV3Alpha.SearchExecutions(tt.args.ctx, tt.args.req)
if tt.wantErr {
assert.Error(ttt, listErr, "Error: "+listErr.Error())
} else {
assert.NoError(ttt, listErr)
}
if listErr != nil {
require.Error(ttt, listErr, "Error: "+listErr.Error())
return
}
require.NoError(ttt, listErr)
// always first check length, otherwise its failed anyway
assert.Len(ttt, got.Result, len(tt.want.Result))
require.Len(ttt, got.Result, len(tt.want.Result))
for i := range tt.want.Result {
// as not sorted, all elements have to be checked
// workaround as oneof elements can only be checked with assert.EqualExportedValues()
if j, found := containExecution(got.Result, tt.want.Result[i]); found {
assert.EqualExportedValues(t, tt.want.Result[i], got.Result[j])
integration.AssertResourceDetails(ttt, tt.want.Result[i].GetDetails(), got.Result[j].GetDetails())
got.Result[j].Details = tt.want.Result[i].GetDetails()
assert.EqualExportedValues(ttt, tt.want.Result[i], got.Result[j])
}
}
integration.AssertResourceListDetails(ttt, tt.want, got)
}, retryDuration, time.Millisecond*100, "timeout waiting for expected execution result")
}, retryDuration, tick, "timeout waiting for expected execution result")
})
}
}

View File

@@ -43,10 +43,8 @@ func ensureFeatureEnabled(t *testing.T, instance *integration.Instance) {
Actions: gu.Ptr(true),
})
require.NoError(t, err)
retryDuration := time.Minute
if ctxDeadline, ok := ctx.Deadline(); ok {
retryDuration = time.Until(ctxDeadline)
}
retryDuration, tick := integration.WaitForAndTickWithMaxDuration(ctx, time.Minute)
require.EventuallyWithT(t,
func(ttt *assert.CollectT) {
f, err := instance.Client.FeatureV2.GetInstanceFeatures(ctx, &feature.GetInstanceFeaturesRequest{
@@ -59,12 +57,13 @@ func ensureFeatureEnabled(t *testing.T, instance *integration.Instance) {
time.Second,
"timed out waiting for ensuring instance feature")
retryDuration, tick = integration.WaitForAndTickWithMaxDuration(ctx, time.Minute)
require.EventuallyWithT(t,
func(ttt *assert.CollectT) {
_, err := instance.Client.ActionV3Alpha.ListExecutionMethods(ctx, &action.ListExecutionMethodsRequest{})
assert.NoError(ttt, err)
},
retryDuration,
time.Second,
tick,
"timed out waiting for ensuring instance feature call")
}

View File

@@ -350,10 +350,10 @@ func TestServer_SetContactEmail(t *testing.T) {
}
got, err := instance.Client.UserV3Alpha.SetContactEmail(tt.ctx, tt.req)
if tt.wantErr {
assert.Error(t, err)
require.Error(t, err)
return
}
assert.NoError(t, err)
require.NoError(t, err)
integration.AssertResourceDetails(t, tt.res.want, got.Details)
if tt.res.returnCode {
assert.NotNil(t, got.VerificationCode)
@@ -545,10 +545,10 @@ func TestServer_VerifyContactEmail(t *testing.T) {
}
got, err := instance.Client.UserV3Alpha.VerifyContactEmail(tt.ctx, tt.req)
if tt.wantErr {
assert.Error(t, err)
require.Error(t, err)
return
}
assert.NoError(t, err)
require.NoError(t, err)
integration.AssertResourceDetails(t, tt.res.want, got.Details)
})
}
@@ -757,10 +757,10 @@ func TestServer_ResendContactEmailCode(t *testing.T) {
}
got, err := instance.Client.UserV3Alpha.ResendContactEmailCode(tt.ctx, tt.req)
if tt.wantErr {
assert.Error(t, err)
require.Error(t, err)
return
}
assert.NoError(t, err)
require.NoError(t, err)
integration.AssertResourceDetails(t, tt.res.want, got.Details)
if tt.res.returnCode {
assert.NotNil(t, got.VerificationCode)

View File

@@ -277,10 +277,10 @@ func TestServer_SetContactPhone(t *testing.T) {
}
got, err := instance.Client.UserV3Alpha.SetContactPhone(tt.ctx, tt.req)
if tt.wantErr {
assert.Error(t, err)
require.Error(t, err)
return
}
assert.NoError(t, err)
require.NoError(t, err)
integration.AssertResourceDetails(t, tt.res.want, got.Details)
if tt.res.returnCode {
assert.NotNil(t, got.VerificationCode)
@@ -474,10 +474,10 @@ func TestServer_VerifyContactPhone(t *testing.T) {
}
got, err := instance.Client.UserV3Alpha.VerifyContactPhone(tt.ctx, tt.req)
if tt.wantErr {
assert.Error(t, err)
require.Error(t, err)
return
}
assert.NoError(t, err)
require.NoError(t, err)
integration.AssertResourceDetails(t, tt.res.want, got.Details)
})
}
@@ -686,10 +686,10 @@ func TestServer_ResendContactPhoneCode(t *testing.T) {
}
got, err := instance.Client.UserV3Alpha.ResendContactPhoneCode(tt.ctx, tt.req)
if tt.wantErr {
assert.Error(t, err)
require.Error(t, err)
return
}
assert.NoError(t, err)
require.NoError(t, err)
integration.AssertResourceDetails(t, tt.res.want, got.Details)
if tt.res.returnCode {
assert.NotNil(t, got.VerificationCode)

View File

@@ -43,10 +43,7 @@ func ensureFeatureEnabled(t *testing.T, instance *integration.Instance) {
UserSchema: gu.Ptr(true),
})
require.NoError(t, err)
retryDuration := time.Minute
if ctxDeadline, ok := ctx.Deadline(); ok {
retryDuration = time.Until(ctxDeadline)
}
retryDuration, tick := integration.WaitForAndTickWithMaxDuration(ctx, 5*time.Minute)
require.EventuallyWithT(t,
func(ttt *assert.CollectT) {
f, err := instance.Client.FeatureV2.GetInstanceFeatures(ctx, &feature.GetInstanceFeaturesRequest{
@@ -58,15 +55,16 @@ func ensureFeatureEnabled(t *testing.T, instance *integration.Instance) {
}
},
retryDuration,
time.Second,
tick,
"timed out waiting for ensuring instance feature")
retryDuration, tick = integration.WaitForAndTickWithMaxDuration(ctx, 5*time.Minute)
require.EventuallyWithT(t,
func(ttt *assert.CollectT) {
_, err := instance.Client.UserV3Alpha.SearchUsers(ctx, &user.SearchUsersRequest{})
assert.NoError(ttt, err)
},
retryDuration,
time.Second,
tick,
"timed out waiting for ensuring instance feature call")
}

View File

@@ -224,6 +224,7 @@ func TestServer_CreateUser(t *testing.T) {
if tt.res.returnCodePhone {
require.NotNil(t, got.PhoneCode)
}
})
}
}
@@ -629,10 +630,10 @@ func TestServer_PatchUser(t *testing.T) {
}
got, err := instance.Client.UserV3Alpha.PatchUser(tt.ctx, tt.req)
if tt.wantErr {
assert.Error(t, err)
require.Error(t, err)
return
}
assert.NoError(t, err)
require.NoError(t, err)
integration.AssertResourceDetails(t, tt.res.want, got.Details)
if tt.res.returnCodeEmail {
assert.NotNil(t, got.EmailCode)
@@ -848,10 +849,10 @@ func TestServer_DeleteUser(t *testing.T) {
}
got, err := instance.Client.UserV3Alpha.DeleteUser(tt.ctx, tt.req)
if tt.wantErr {
assert.Error(t, err)
require.Error(t, err)
return
}
assert.NoError(t, err)
require.NoError(t, err)
integration.AssertResourceDetails(t, tt.want, got.Details)
})
}
@@ -1059,10 +1060,10 @@ func TestServer_LockUser(t *testing.T) {
}
got, err := instance.Client.UserV3Alpha.LockUser(tt.ctx, tt.req)
if tt.wantErr {
assert.Error(t, err)
require.Error(t, err)
return
}
assert.NoError(t, err)
require.NoError(t, err)
integration.AssertResourceDetails(t, tt.want, got.Details)
})
}
@@ -1242,10 +1243,10 @@ func TestServer_UnlockUser(t *testing.T) {
}
got, err := instance.Client.UserV3Alpha.UnlockUser(tt.ctx, tt.req)
if tt.wantErr {
assert.Error(t, err)
require.Error(t, err)
return
}
assert.NoError(t, err)
require.NoError(t, err)
integration.AssertResourceDetails(t, tt.want, got.Details)
})
}
@@ -1444,10 +1445,10 @@ func TestServer_DeactivateUser(t *testing.T) {
}
got, err := instance.Client.UserV3Alpha.DeactivateUser(tt.ctx, tt.req)
if tt.wantErr {
assert.Error(t, err)
require.Error(t, err)
return
}
assert.NoError(t, err)
require.NoError(t, err)
integration.AssertResourceDetails(t, tt.want, got.Details)
})
}
@@ -1627,10 +1628,10 @@ func TestServer_ActivateUser(t *testing.T) {
}
got, err := instance.Client.UserV3Alpha.ActivateUser(tt.ctx, tt.req)
if tt.wantErr {
assert.Error(t, err)
require.Error(t, err)
return
}
assert.NoError(t, err)
require.NoError(t, err)
integration.AssertResourceDetails(t, tt.want, got.Details)
})
}

View File

@@ -188,31 +188,26 @@ func TestServer_ListUserSchemas(t *testing.T) {
require.NoError(t, err)
}
retryDuration := 20 * time.Second
if ctxDeadline, ok := isolatedIAMOwnerCTX.Deadline(); ok {
retryDuration = time.Until(ctxDeadline)
}
retryDuration, tick := integration.WaitForAndTickWithMaxDuration(isolatedIAMOwnerCTX, 20*time.Second)
require.EventuallyWithT(t, func(ttt *assert.CollectT) {
got, err := instance.Client.UserSchemaV3.SearchUserSchemas(tt.args.ctx, tt.args.req)
if tt.wantErr {
require.Error(ttt, err)
return
}
assert.NoError(ttt, err)
require.NoError(ttt, err)
// always first check length, otherwise its failed anyway
assert.Len(ttt, got.Result, len(tt.want.Result))
require.Len(ttt, got.Result, len(tt.want.Result))
for i := range tt.want.Result {
want := tt.want.Result[i]
got := got.Result[i]
wantSchema := tt.want.Result[i]
gotSchema := got.Result[i]
integration.AssertResourceDetails(t, want.GetDetails(), got.GetDetails())
want.Details = got.Details
grpc.AllFieldsEqual(t, want.ProtoReflect(), got.ProtoReflect(), grpc.CustomMappers)
integration.AssertResourceDetails(ttt, wantSchema.GetDetails(), gotSchema.GetDetails())
wantSchema.Details = gotSchema.GetDetails()
grpc.AllFieldsEqual(ttt, wantSchema.ProtoReflect(), gotSchema.ProtoReflect(), grpc.CustomMappers)
}
integration.AssertListDetails(t, tt.want, got)
}, retryDuration, time.Millisecond*100, "timeout waiting for expected user schema result")
integration.AssertListDetails(ttt, tt.want, got)
}, retryDuration, tick, "timeout waiting for expected user schema result")
})
}
}
@@ -300,24 +295,21 @@ func TestServer_GetUserSchema(t *testing.T) {
require.NoError(t, err)
}
retryDuration := 5 * time.Second
if ctxDeadline, ok := isolatedIAMOwnerCTX.Deadline(); ok {
retryDuration = time.Until(ctxDeadline)
}
retryDuration, tick := integration.WaitForAndTickWithMaxDuration(isolatedIAMOwnerCTX, 5*time.Second)
require.EventuallyWithT(t, func(ttt *assert.CollectT) {
got, err := instance.Client.UserSchemaV3.GetUserSchema(tt.args.ctx, tt.args.req)
if tt.wantErr {
assert.Error(t, err, "Error: "+err.Error())
} else {
assert.NoError(t, err)
wantSchema := tt.want.GetUserSchema()
gotSchema := got.GetUserSchema()
integration.AssertResourceDetails(t, wantSchema.GetDetails(), gotSchema.GetDetails())
tt.want.UserSchema.Details = got.GetUserSchema().GetDetails()
grpc.AllFieldsEqual(t, tt.want.ProtoReflect(), got.ProtoReflect(), grpc.CustomMappers)
require.Error(ttt, err, "Error: "+err.Error())
return
}
}, retryDuration, time.Millisecond*100, "timeout waiting for expected user schema result")
require.NoError(ttt, err)
wantSchema := tt.want.GetUserSchema()
gotSchema := got.GetUserSchema()
integration.AssertResourceDetails(ttt, wantSchema.GetDetails(), gotSchema.GetDetails())
wantSchema.Details = got.GetUserSchema().GetDetails()
grpc.AllFieldsEqual(ttt, wantSchema.ProtoReflect(), gotSchema.ProtoReflect(), grpc.CustomMappers)
}, retryDuration, tick, "timeout waiting for expected user schema result")
})
}
}

View File

@@ -43,10 +43,8 @@ func ensureFeatureEnabled(t *testing.T, instance *integration.Instance) {
UserSchema: gu.Ptr(true),
})
require.NoError(t, err)
retryDuration := time.Minute
if ctxDeadline, ok := ctx.Deadline(); ok {
retryDuration = time.Until(ctxDeadline)
}
retryDuration, tick := integration.WaitForAndTickWithMaxDuration(ctx, 5*time.Minute)
require.EventuallyWithT(t,
func(ttt *assert.CollectT) {
f, err := instance.Client.FeatureV2.GetInstanceFeatures(ctx, &feature.GetInstanceFeaturesRequest{
@@ -58,15 +56,16 @@ func ensureFeatureEnabled(t *testing.T, instance *integration.Instance) {
}
},
retryDuration,
time.Second,
tick,
"timed out waiting for ensuring instance feature")
retryDuration, tick = integration.WaitForAndTickWithMaxDuration(ctx, 5*time.Minute)
require.EventuallyWithT(t,
func(ttt *assert.CollectT) {
_, err := instance.Client.UserSchemaV3.SearchUserSchemas(ctx, &schema.SearchUserSchemasRequest{})
assert.NoError(ttt, err)
},
retryDuration,
time.Second,
tick,
"timed out waiting for ensuring instance feature call")
}

View File

@@ -191,6 +191,8 @@ func createInstance(t *testing.T, enableFeature bool) (*integration.Instance, co
})
require.NoError(t, err)
}
retryDuration, tick := integration.WaitForAndTickWithMaxDuration(iamCTX, time.Minute)
assert.EventuallyWithT(t, func(collect *assert.CollectT) {
resp, err := instance.Client.WebKeyV3Alpha.ListWebKeys(iamCTX, &webkey.ListWebKeysRequest{})
if enableFeature {
@@ -199,7 +201,7 @@ func createInstance(t *testing.T, enableFeature bool) (*integration.Instance, co
} else {
assert.Error(collect, err)
}
}, time.Minute, time.Second)
}, retryDuration, tick)
return instance, iamCTX
}
@@ -213,6 +215,8 @@ func assertFeatureDisabledError(t *testing.T, err error) {
}
func checkWebKeyListState(ctx context.Context, t *testing.T, instance *integration.Instance, nKeys int, expectActiveKeyID string, config any) {
retryDuration, tick := integration.WaitForAndTickWithMaxDuration(ctx, time.Minute)
assert.EventuallyWithT(t, func(collect *assert.CollectT) {
resp, err := instance.Client.WebKeyV3Alpha.ListWebKeys(ctx, &webkey.ListWebKeysRequest{})
require.NoError(collect, err)
@@ -243,5 +247,5 @@ func checkWebKeyListState(ctx context.Context, t *testing.T, instance *integrati
if expectActiveKeyID != "" {
assert.Equal(collect, expectActiveKeyID, gotActiveKeyID)
}
}, time.Minute, time.Second)
}, retryDuration, tick)
}