feat(eventstore): increase parallel write capabilities (#5940)

This implementation increases parallel write capabilities of the eventstore.
Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and  [06](https://zitadel.com/docs/support/advisory/a10006).
The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`.
If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
This commit is contained in:
Silvan
2023-10-19 12:19:10 +02:00
committed by GitHub
parent 259faba3f0
commit b5564572bc
791 changed files with 30326 additions and 43202 deletions

View File

@@ -2,15 +2,12 @@ package user
import (
"context"
"encoding/json"
"time"
"github.com/zitadel/zitadel/internal/api/http"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/crypto"
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/eventstore/repository"
"github.com/zitadel/zitadel/internal/eventstore"
)
const (
@@ -42,11 +39,11 @@ type HumanOTPAddedEvent struct {
Secret *crypto.CryptoValue `json:"otpSecret,omitempty"`
}
func (e *HumanOTPAddedEvent) Data() interface{} {
func (e *HumanOTPAddedEvent) Payload() interface{} {
return e
}
func (e *HumanOTPAddedEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (e *HumanOTPAddedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
@@ -65,11 +62,11 @@ func NewHumanOTPAddedEvent(
}
}
func HumanOTPAddedEventMapper(event *repository.Event) (eventstore.Event, error) {
func HumanOTPAddedEventMapper(event eventstore.Event) (eventstore.Event, error) {
otpAdded := &HumanOTPAddedEvent{
BaseEvent: *eventstore.BaseEventFromRepo(event),
}
err := json.Unmarshal(event.Data, otpAdded)
err := event.Unmarshal(otpAdded)
if err != nil {
return nil, errors.ThrowInternal(err, "USER-Ns9df", "unable to unmarshal human otp added")
}
@@ -81,11 +78,11 @@ type HumanOTPVerifiedEvent struct {
UserAgentID string `json:"userAgentID,omitempty"`
}
func (e *HumanOTPVerifiedEvent) Data() interface{} {
func (e *HumanOTPVerifiedEvent) Payload() interface{} {
return e
}
func (e *HumanOTPVerifiedEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (e *HumanOTPVerifiedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
@@ -104,7 +101,7 @@ func NewHumanOTPVerifiedEvent(
}
}
func HumanOTPVerifiedEventMapper(event *repository.Event) (eventstore.Event, error) {
func HumanOTPVerifiedEventMapper(event eventstore.Event) (eventstore.Event, error) {
return &HumanOTPVerifiedEvent{
BaseEvent: *eventstore.BaseEventFromRepo(event),
}, nil
@@ -114,11 +111,11 @@ type HumanOTPRemovedEvent struct {
eventstore.BaseEvent `json:"-"`
}
func (e *HumanOTPRemovedEvent) Data() interface{} {
func (e *HumanOTPRemovedEvent) Payload() interface{} {
return nil
}
func (e *HumanOTPRemovedEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (e *HumanOTPRemovedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
@@ -135,7 +132,7 @@ func NewHumanOTPRemovedEvent(
}
}
func HumanOTPRemovedEventMapper(event *repository.Event) (eventstore.Event, error) {
func HumanOTPRemovedEventMapper(event eventstore.Event) (eventstore.Event, error) {
return &HumanOTPRemovedEvent{
BaseEvent: *eventstore.BaseEventFromRepo(event),
}, nil
@@ -146,11 +143,11 @@ type HumanOTPCheckSucceededEvent struct {
*AuthRequestInfo
}
func (e *HumanOTPCheckSucceededEvent) Data() interface{} {
func (e *HumanOTPCheckSucceededEvent) Payload() interface{} {
return e
}
func (e *HumanOTPCheckSucceededEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (e *HumanOTPCheckSucceededEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
@@ -169,11 +166,11 @@ func NewHumanOTPCheckSucceededEvent(
}
}
func HumanOTPCheckSucceededEventMapper(event *repository.Event) (eventstore.Event, error) {
func HumanOTPCheckSucceededEventMapper(event eventstore.Event) (eventstore.Event, error) {
otpAdded := &HumanOTPCheckSucceededEvent{
BaseEvent: *eventstore.BaseEventFromRepo(event),
}
err := json.Unmarshal(event.Data, otpAdded)
err := event.Unmarshal(otpAdded)
if err != nil {
return nil, errors.ThrowInternal(err, "USER-Ns9df", "unable to unmarshal human otp check succeeded")
}
@@ -185,11 +182,11 @@ type HumanOTPCheckFailedEvent struct {
*AuthRequestInfo
}
func (e *HumanOTPCheckFailedEvent) Data() interface{} {
func (e *HumanOTPCheckFailedEvent) Payload() interface{} {
return e
}
func (e *HumanOTPCheckFailedEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (e *HumanOTPCheckFailedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
@@ -208,11 +205,11 @@ func NewHumanOTPCheckFailedEvent(
}
}
func HumanOTPCheckFailedEventMapper(event *repository.Event) (eventstore.Event, error) {
func HumanOTPCheckFailedEventMapper(event eventstore.Event) (eventstore.Event, error) {
otpAdded := &HumanOTPCheckFailedEvent{
BaseEvent: *eventstore.BaseEventFromRepo(event),
}
err := json.Unmarshal(event.Data, otpAdded)
err := event.Unmarshal(otpAdded)
if err != nil {
return nil, errors.ThrowInternal(err, "USER-Ns9df", "unable to unmarshal human otp check failed")
}
@@ -223,11 +220,11 @@ type HumanOTPSMSAddedEvent struct {
eventstore.BaseEvent `json:"-"`
}
func (e *HumanOTPSMSAddedEvent) Data() interface{} {
func (e *HumanOTPSMSAddedEvent) Payload() interface{} {
return nil
}
func (e *HumanOTPSMSAddedEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (e *HumanOTPSMSAddedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
@@ -252,11 +249,11 @@ type HumanOTPSMSRemovedEvent struct {
eventstore.BaseEvent `json:"-"`
}
func (e *HumanOTPSMSRemovedEvent) Data() interface{} {
func (e *HumanOTPSMSRemovedEvent) Payload() interface{} {
return nil
}
func (e *HumanOTPSMSRemovedEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (e *HumanOTPSMSRemovedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
@@ -286,11 +283,11 @@ type HumanOTPSMSCodeAddedEvent struct {
*AuthRequestInfo
}
func (e *HumanOTPSMSCodeAddedEvent) Data() interface{} {
func (e *HumanOTPSMSCodeAddedEvent) Payload() interface{} {
return e
}
func (e *HumanOTPSMSCodeAddedEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (e *HumanOTPSMSCodeAddedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
@@ -330,11 +327,11 @@ type HumanOTPSMSCodeSentEvent struct {
*AuthRequestInfo
}
func (e *HumanOTPSMSCodeSentEvent) Data() interface{} {
func (e *HumanOTPSMSCodeSentEvent) Payload() interface{} {
return e
}
func (e *HumanOTPSMSCodeSentEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (e *HumanOTPSMSCodeSentEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
@@ -360,11 +357,11 @@ type HumanOTPSMSCheckSucceededEvent struct {
*AuthRequestInfo
}
func (e *HumanOTPSMSCheckSucceededEvent) Data() interface{} {
func (e *HumanOTPSMSCheckSucceededEvent) Payload() interface{} {
return e
}
func (e *HumanOTPSMSCheckSucceededEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (e *HumanOTPSMSCheckSucceededEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
@@ -392,11 +389,11 @@ type HumanOTPSMSCheckFailedEvent struct {
*AuthRequestInfo
}
func (e *HumanOTPSMSCheckFailedEvent) Data() interface{} {
func (e *HumanOTPSMSCheckFailedEvent) Payload() interface{} {
return e
}
func (e *HumanOTPSMSCheckFailedEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (e *HumanOTPSMSCheckFailedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
@@ -423,11 +420,11 @@ type HumanOTPEmailAddedEvent struct {
eventstore.BaseEvent `json:"-"`
}
func (e *HumanOTPEmailAddedEvent) Data() interface{} {
func (e *HumanOTPEmailAddedEvent) Payload() interface{} {
return nil
}
func (e *HumanOTPEmailAddedEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (e *HumanOTPEmailAddedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
@@ -452,11 +449,11 @@ type HumanOTPEmailRemovedEvent struct {
eventstore.BaseEvent `json:"-"`
}
func (e *HumanOTPEmailRemovedEvent) Data() interface{} {
func (e *HumanOTPEmailRemovedEvent) Payload() interface{} {
return nil
}
func (e *HumanOTPEmailRemovedEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (e *HumanOTPEmailRemovedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
@@ -486,11 +483,11 @@ type HumanOTPEmailCodeAddedEvent struct {
*AuthRequestInfo
}
func (e *HumanOTPEmailCodeAddedEvent) Data() interface{} {
func (e *HumanOTPEmailCodeAddedEvent) Payload() interface{} {
return e
}
func (e *HumanOTPEmailCodeAddedEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (e *HumanOTPEmailCodeAddedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
@@ -530,11 +527,11 @@ type HumanOTPEmailCodeSentEvent struct {
*AuthRequestInfo
}
func (e *HumanOTPEmailCodeSentEvent) Data() interface{} {
func (e *HumanOTPEmailCodeSentEvent) Payload() interface{} {
return e
}
func (e *HumanOTPEmailCodeSentEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (e *HumanOTPEmailCodeSentEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
@@ -560,11 +557,11 @@ type HumanOTPEmailCheckSucceededEvent struct {
*AuthRequestInfo
}
func (e *HumanOTPEmailCheckSucceededEvent) Data() interface{} {
func (e *HumanOTPEmailCheckSucceededEvent) Payload() interface{} {
return e
}
func (e *HumanOTPEmailCheckSucceededEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (e *HumanOTPEmailCheckSucceededEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}
@@ -592,11 +589,11 @@ type HumanOTPEmailCheckFailedEvent struct {
*AuthRequestInfo
}
func (e *HumanOTPEmailCheckFailedEvent) Data() interface{} {
func (e *HumanOTPEmailCheckFailedEvent) Payload() interface{} {
return e
}
func (e *HumanOTPEmailCheckFailedEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (e *HumanOTPEmailCheckFailedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
return nil
}