fix: pubsub (#1122)

* start sub

* start implement subsciptions

* start subscription

* implementation for member done

* admin done

* fix: tests

* extend handlers

* prepary notification

* no errors in adminapi

* changed current sequence in all packages

* ignore mocks

* works

* subscriptions as singleton

* tests

* refactor: rename function scope var

* fix: process ALL previous sequences

* fix: spooler and pubsub

* handler check

* fix: process events until all done

* fix break on query err

* fix: handler

* fix: process sequence or return error

* check aggregate id

* fix: log only in error case

* fix tests

* fix: handlers

* fix: spooler

* fix: spooler

* fix: tests

* fix: continue

Co-authored-by: Livio Amstutz <livio.a@gmail.com>

internal/management/repository/eventsourcing/view/application.go

@@ -1,6 +1,7 @@
 package view
 
 import (
+	"github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/eventstore/models"
 	proj_model "github.com/caos/zitadel/internal/project/model"
 	"github.com/caos/zitadel/internal/project/repository/view"
@@ -26,7 +27,7 @@ func (v *View) SearchApplications(request *proj_model.ApplicationSearchRequest)
 
 func (v *View) PutApplication(app *model.ApplicationView, event *models.Event) error {
 	err := view.PutApplication(v.Db, applicationTable, app)
-	if err != nil {
+	if err != nil && !errors.IsNotFound(err) {
 		return err
 	}
 	return v.ProcessedApplicationSequence(event)
@@ -43,7 +44,7 @@ func (v *View) PutApplications(apps []*model.ApplicationView, event *models.Even
 func (v *View) DeleteApplication(appID string, event *models.Event) error {
 	err := view.DeleteApplication(v.Db, applicationTable, appID)
 	if err != nil {
-		return nil
+		return err
 	}
 	return v.ProcessedApplicationSequence(event)
 }

internal/management/repository/eventsourcing/view/machine_keys.go

@@ -1,6 +1,7 @@
 package view
 
 import (
+	"github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/eventstore/models"
 	usr_model "github.com/caos/zitadel/internal/user/model"
 	"github.com/caos/zitadel/internal/user/repository/view"
@@ -29,24 +30,21 @@ func (v *View) PutMachineKey(org *model.MachineKeyView, event *models.Event) err
 	if err != nil {
 		return err
 	}
-	if event.Sequence != 0 {
-		return v.ProcessedMachineKeySequence(event)
-	}
-	return nil
+	return v.ProcessedMachineKeySequence(event)
 }
 
 func (v *View) DeleteMachineKey(keyID string, event *models.Event) error {
 	err := view.DeleteMachineKey(v.Db, machineKeyTable, keyID)
-	if err != nil {
-		return nil
+	if err != nil && !errors.IsNotFound(err) {
+		return err
 	}
 	return v.ProcessedMachineKeySequence(event)
 }
 
 func (v *View) DeleteMachineKeysByUserID(userID string, event *models.Event) error {
 	err := view.DeleteMachineKey(v.Db, machineKeyTable, userID)
-	if err != nil {
-		return nil
+	if err != nil && !errors.IsNotFound(err) {
+		return err
 	}
 	return v.ProcessedMachineKeySequence(event)
 }

internal/management/repository/eventsourcing/view/org_domain.go

@@ -1,6 +1,7 @@
 package view
 
 import (
+	"github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/eventstore/models"
 	org_model "github.com/caos/zitadel/internal/org/model"
 	"github.com/caos/zitadel/internal/org/repository/view"
@@ -33,10 +34,7 @@ func (v *View) PutOrgDomain(org *model.OrgDomainView, event *models.Event) error
 	if err != nil {
 		return err
 	}
-	if event.Sequence != 0 {
-		return v.ProcessedOrgDomainSequence(event)
-	}
-	return nil
+	return v.ProcessedOrgDomainSequence(event)
 }
 
 func (v *View) PutOrgDomains(domains []*model.OrgDomainView, event *models.Event) error {
@@ -49,8 +47,8 @@ func (v *View) PutOrgDomains(domains []*model.OrgDomainView, event *models.Event
 
 func (v *View) DeleteOrgDomain(orgID, domain string, event *models.Event) error {
 	err := view.DeleteOrgDomain(v.Db, orgDomainTable, orgID, domain)
-	if err != nil {
-		return nil
+	if err != nil && !errors.IsNotFound(err) {
+		return err
 	}
 	return v.ProcessedOrgDomainSequence(event)
 }

internal/management/repository/eventsourcing/view/org_member.go

@@ -1,6 +1,7 @@
 package view
 
 import (
+	"github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/eventstore/models"
 	org_model "github.com/caos/zitadel/internal/org/model"
 	"github.com/caos/zitadel/internal/org/repository/view"
@@ -42,16 +43,16 @@ func (v *View) PutOrgMembers(members []*model.OrgMemberView, event *models.Event
 
 func (v *View) DeleteOrgMember(orgID, userID string, event *models.Event) error {
 	err := view.DeleteOrgMember(v.Db, orgMemberTable, orgID, userID)
-	if err != nil {
-		return nil
+	if err != nil && !errors.IsNotFound(err) {
+		return err
 	}
 	return v.ProcessedOrgMemberSequence(event)
 }
 
 func (v *View) DeleteOrgMembersByUserID(userID string, event *models.Event) error {
 	err := view.DeleteOrgMembersByUserID(v.Db, orgMemberTable, userID)
-	if err != nil {
-		return nil
+	if err != nil && !errors.IsNotFound(err) {
+		return err
 	}
 	return v.ProcessedOrgMemberSequence(event)
 }

internal/management/repository/eventsourcing/view/project.go

@@ -1,6 +1,7 @@
 package view
 
 import (
+	"github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/eventstore/models"
 	proj_model "github.com/caos/zitadel/internal/project/model"
 	"github.com/caos/zitadel/internal/project/repository/view"
@@ -30,8 +31,8 @@ func (v *View) PutProject(project *model.ProjectView, event *models.Event) error
 
 func (v *View) DeleteProject(projectID string, event *models.Event) error {
 	err := view.DeleteProject(v.Db, projectTable, projectID)
-	if err != nil {
-		return nil
+	if err != nil && !errors.IsNotFound(err) {
+		return err
 	}
 	return v.ProcessedProjectSequence(event)
 }

internal/management/repository/eventsourcing/view/project_grant_member.go

@@ -1,6 +1,7 @@
 package view
 
 import (
+	"github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/eventstore/models"
 	proj_model "github.com/caos/zitadel/internal/project/model"
 	"github.com/caos/zitadel/internal/project/repository/view"
@@ -46,8 +47,8 @@ func (v *View) PutProjectGrantMembers(members []*model.ProjectGrantMemberView, e
 
 func (v *View) DeleteProjectGrantMember(grantID, userID string, event *models.Event) error {
 	err := view.DeleteProjectGrantMember(v.Db, projectGrantMemberTable, grantID, userID)
-	if err != nil {
-		return nil
+	if err != nil && !errors.IsNotFound(err) {
+		return err
 	}
 	return v.ProcessedProjectGrantMemberSequence(event)
 }

internal/management/repository/eventsourcing/view/project_member.go

@@ -1,6 +1,7 @@
 package view
 
 import (
+	"github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/eventstore/models"
 	proj_model "github.com/caos/zitadel/internal/project/model"
 	"github.com/caos/zitadel/internal/project/repository/view"
@@ -46,8 +47,8 @@ func (v *View) PutProjectMembers(project []*model.ProjectMemberView, event *mode
 
 func (v *View) DeleteProjectMember(projectID, userID string, event *models.Event) error {
 	err := view.DeleteProjectMember(v.Db, projectMemberTable, projectID, userID)
-	if err != nil {
-		return nil
+	if err != nil && !errors.IsNotFound(err) {
+		return err
 	}
 	return v.ProcessedProjectMemberSequence(event)
 }

internal/management/repository/eventsourcing/view/project_role.go

@@ -1,6 +1,7 @@
 package view
 
 import (
+	"github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/eventstore/models"
 	proj_model "github.com/caos/zitadel/internal/project/model"
 	"github.com/caos/zitadel/internal/project/repository/view"
@@ -42,8 +43,8 @@ func (v *View) PutProjectRole(project *model.ProjectRoleView, event *models.Even
 
 func (v *View) DeleteProjectRole(projectID, orgID, key string, event *models.Event) error {
 	err := view.DeleteProjectRole(v.Db, projectRoleTable, projectID, orgID, key)
-	if err != nil {
-		return nil
+	if err != nil && !errors.IsNotFound(err) {
+		return err
 	}
 	return v.ProcessedProjectRoleSequence(event)
 }

internal/management/repository/eventsourcing/view/user.go

@@ -1,6 +1,7 @@
 package view
 
 import (
+	"github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/eventstore/models"
 	usr_model "github.com/caos/zitadel/internal/user/model"
 	"github.com/caos/zitadel/internal/user/repository/view"
@@ -53,16 +54,13 @@ func (v *View) PutUser(user *model.UserView, event *models.Event) error {
 	if err != nil {
 		return err
 	}
-	if event.Sequence != 0 {
-		return v.ProcessedUserSequence(event)
-	}
-	return nil
+	return v.ProcessedUserSequence(event)
 }
 
 func (v *View) DeleteUser(userID string, event *models.Event) error {
 	err := view.DeleteUser(v.Db, userTable, userID)
-	if err != nil {
-		return nil
+	if err != nil && !errors.IsNotFound(err) {
+		return err
 	}
 	return v.ProcessedUserSequence(event)
 }

internal/management/repository/eventsourcing/view/user_grant.go

@@ -1,6 +1,7 @@
 package view
 
 import (
+	"github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/eventstore/models"
 	grant_model "github.com/caos/zitadel/internal/usergrant/model"
 	"github.com/caos/zitadel/internal/usergrant/repository/view"
@@ -58,8 +59,8 @@ func (v *View) PutUserGrants(grants []*model.UserGrantView, event *models.Event)
 
 func (v *View) DeleteUserGrant(grantID string, event *models.Event) error {
 	err := view.DeleteUserGrant(v.Db, userGrantTable, grantID)
-	if err != nil {
-		return nil
+	if err != nil && !errors.IsNotFound(err) {
+		return err
 	}
 	return v.ProcessedUserGrantSequence(event)
 }

internal/management/repository/eventsourcing/view/user_membership.go

@@ -1,6 +1,7 @@
 package view
 
 import (
+	"github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/eventstore/models"
 	usr_model "github.com/caos/zitadel/internal/user/model"
 	"github.com/caos/zitadel/internal/user/repository/view"
@@ -42,32 +43,32 @@ func (v *View) BulkPutUserMemberships(memberships []*model.UserMembershipView, e
 
 func (v *View) DeleteUserMembership(userID, aggregateID, objectID string, memberType usr_model.MemberType, event *models.Event) error {
 	err := view.DeleteUserMembership(v.Db, userMembershipTable, userID, aggregateID, objectID, memberType)
-	if err != nil {
-		return nil
+	if err != nil && !errors.IsNotFound(err) {
+		return err
 	}
 	return v.ProcessedUserMembershipSequence(event)
 }
 
 func (v *View) DeleteUserMembershipsByUserID(userID string, event *models.Event) error {
 	err := view.DeleteUserMembershipsByUserID(v.Db, userMembershipTable, userID)
-	if err != nil {
-		return nil
+	if err != nil && !errors.IsNotFound(err) {
+		return err
 	}
 	return v.ProcessedUserMembershipSequence(event)
 }
 
 func (v *View) DeleteUserMembershipsByAggregateID(aggregateID string, event *models.Event) error {
 	err := view.DeleteUserMembershipsByAggregateID(v.Db, userMembershipTable, aggregateID)
-	if err != nil {
-		return nil
+	if err != nil && !errors.IsNotFound(err) {
+		return err
 	}
 	return v.ProcessedUserMembershipSequence(event)
 }
 
 func (v *View) DeleteUserMembershipsByAggregateIDAndObjectID(aggregateID, objectID string, event *models.Event) error {
 	err := view.DeleteUserMembershipsByAggregateIDAndObjectID(v.Db, userMembershipTable, aggregateID, objectID)
-	if err != nil {
-		return nil
+	if err != nil && !errors.IsNotFound(err) {
+		return err
 	}
 	return v.ProcessedUserMembershipSequence(event)
 }