TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-12-03 05:02:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: query data AS OF SYSTEM TIME (#5231)

Browse Source 
Queries the data in the storage layser at the timestamp when the call hit the API layer
This commit is contained in:
Silvan
2023-02-27 22:36:43 +01:00
committed by GitHub
parent 80003939ad
commit e38abdcdf3
170 changed files with 3101 additions and 3169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
internal/api/assets/asset.go
View File

@@ -82,7 +82,7 @@ func DefaultErrorHandler(w http.ResponseWriter, r *http.Request, err error, code
http.Error(w, err.Error(), code)
}
func NewHandler(commands *command.Commands, verifier *authz.TokenVerifier, authConfig authz.Config, idGenerator id.Generator, storage static.Storage, queries *query.Queries, instanceInterceptor, assetCacheInterceptor, accessInterceptor func(handler http.Handler) http.Handler) http.Handler {
func NewHandler(commands *command.Commands, verifier *authz.TokenVerifier, authConfig authz.Config, idGenerator id.Generator, storage static.Storage, queries *query.Queries, callDurationInterceptor, instanceInterceptor, assetCacheInterceptor, accessInterceptor func(handler http.Handler) http.Handler) http.Handler {
h := &Handler{
commands: commands,
errorHandler: DefaultErrorHandler,
@@ -94,7 +94,7 @@ func NewHandler(commands *command.Commands, verifier *authz.TokenVerifier, authC
verifier.RegisterServer("Assets-API", "assets", AssetsService_AuthMethods)
router := mux.NewRouter()
router.Use(instanceInterceptor, assetCacheInterceptor, accessInterceptor)
router.Use(callDurationInterceptor, instanceInterceptor, assetCacheInterceptor, accessInterceptor)
RegisterRoutes(router, h)
router.PathPrefix("/{owner}").Methods("GET").HandlerFunc(DownloadHandleFunc(h, h.GetFile()))
return http_util.CopyHeadersToContext(http_mw.CORSInterceptor(router))

17
internal/api/authz/detach.go Normal file
View File

@@ -0,0 +1,17 @@
package authz
import (
"context"
"time"
)
func Detach(ctx context.Context) context.Context { return detachedContext{ctx} }
type detachedContext struct {
parent context.Context
}
func (v detachedContext) Deadline() (time.Time, bool) { return time.Time{}, false }
func (v detachedContext) Done() <-chan struct{} { return nil }
func (v detachedContext) Err() error { return nil }
func (v detachedContext) Value(key interface{}) interface{} { return v.parent.Value(key) }

38
internal/api/call/duration.go Normal file
View File

@@ -0,0 +1,38 @@
package call
import (
"context"
"time"
)
type durationKey struct{}
var key *durationKey = (*durationKey)(nil)
// WithTimestamp sets [time.Now()] adds the call field to the context
// if it's not already set
func WithTimestamp(parent context.Context) context.Context {
if parent.Value(key) != nil {
return parent
}
return context.WithValue(parent, key, time.Now())
}
// FromContext returns the [time.Time] the call hit the api
func FromContext(ctx context.Context) (t time.Time) {
value := ctx.Value(key)
if t, ok := value.(time.Time); ok {
return t
}
return t
}
// Took returns the time the call took so far
func Took(ctx context.Context) time.Duration {
start := FromContext(ctx)
if start.IsZero() {
return 0
}
return time.Since(start)
}

119
internal/api/call/duration_test.go Normal file
View File

@@ -0,0 +1,119 @@
package call
import (
"context"
"testing"
"time"
)
func TestTook(t *testing.T) {
type args struct {
ctx context.Context
}
tests := []struct {
name string
args args
startIsZero bool
}{
{
name: "no start",
args: args{
ctx: context.Background(),
},
startIsZero: true,
},
{
name: "with start",
args: args{
ctx: WithTimestamp(context.Background()),
},
startIsZero: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := Took(tt.args.ctx)
if tt.startIsZero && got != 0 {
t.Errorf("Duration should be 0 but was %v", got)
}
if !tt.startIsZero && got <= 0 {
t.Errorf("Duration should be greater 0 but was %d", got)
}
})
}
}
func TestFromContext(t *testing.T) {
type args struct {
ctx context.Context
}
tests := []struct {
name string
args args
isZero bool
}{
{
name: "no start",
args: args{
ctx: context.Background(),
},
isZero: true,
},
{
name: "with start",
args: args{
ctx: WithTimestamp(context.Background()),
},
isZero: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := FromContext(tt.args.ctx)
if tt.isZero != got.IsZero() {
t.Errorf("Time is zero should be %v but was %v", tt.isZero, got.IsZero())
}
})
}
}
func TestWithTimestamp(t *testing.T) {
start := time.Date(2019, 4, 29, 0, 0, 0, 0, time.UTC)
type args struct {
ctx context.Context
}
tests := []struct {
name string
args args
noPrevious bool
}{
{
name: "fresh context",
args: args{
ctx: context.WithValue(context.Background(), key, start),
},
noPrevious: true,
},
{
name: "with start",
args: args{
ctx: WithTimestamp(context.Background()),
},
noPrevious: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := WithTimestamp(tt.args.ctx)
val := got.Value(key).(time.Time)
if !tt.noPrevious && val.Before(start) {
t.Errorf("time should be now not %v", val)
}
if tt.noPrevious && val.After(start) {
t.Errorf("time should be start not %v", val)
}
})
}
}

13
internal/api/grpc/admin/import.go
View File

@@ -88,17 +88,6 @@ func (c *count) getProgress() string {
"project_grant_members " + strconv.Itoa(c.projectGrantMemberCount) + "/" + strconv.Itoa(c.projectGrantMemberLen)
}
func Detach(ctx context.Context) context.Context { return detachedContext{ctx} }
type detachedContext struct {
parent context.Context
}
func (v detachedContext) Deadline() (time.Time, bool) { return time.Time{}, false }
func (v detachedContext) Done() <-chan struct{} { return nil }
func (v detachedContext) Err() error { return nil }
func (v detachedContext) Value(key interface{}) interface{} { return v.parent.Value(key) }
func (s *Server) ImportData(ctx context.Context, req *admin_pb.ImportDataRequest) (_ *admin_pb.ImportDataResponse, err error) {
ctx, span := tracing.NewSpan(ctx)
defer func() { span.EndWithError(err) }()
@@ -169,7 +158,7 @@ func (s *Server) ImportData(ctx context.Context, req *admin_pb.ImportDataRequest
if err != nil {
return nil, err
}
dctx := Detach(ctx)
dctx := authz.Detach(ctx)
go func() {
ch := make(chan importResponse, 1)
ctxTimeout, cancel := context.WithTimeout(dctx, timeoutDuration)

1
internal/api/grpc/server/gateway.go
View File

@@ -71,6 +71,7 @@ func CreateGateway(ctx context.Context, g Gateway, port uint16, http1HostName st
}
func addInterceptors(handler http.Handler, http1HostName string) http.Handler {
handler = http_mw.CallDurationHandler(handler)
handler = http1Host(handler, http1HostName)
handler = http_mw.CORSInterceptor(handler)
handler = http_mw.DefaultTelemetryHandler(handler)

16
internal/api/grpc/server/middleware/call_interceptor.go Normal file
View File

@@ -0,0 +1,16 @@
package middleware
import (
"context"
"google.golang.org/grpc"
"github.com/zitadel/zitadel/internal/api/call"
)
func CallDurationHandler() grpc.UnaryServerInterceptor {
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
ctx = call.WithTimestamp(ctx)
return handler(ctx, req)
}
}

1
internal/api/grpc/server/server.go
View File

@@ -35,6 +35,7 @@ func CreateServer(
serverOptions := []grpc.ServerOption{
grpc.UnaryInterceptor(
grpc_middleware.ChainUnaryServer(
middleware.CallDurationHandler(),
middleware.DefaultTracingServer(),
middleware.MetricsHandler(metricTypes, grpc_api.Probes...),
middleware.NoCacheInterceptor(),

13
internal/api/http/middleware/call_interceptor.go Normal file
View File

@@ -0,0 +1,13 @@
package middleware
import (
"net/http"
"github.com/zitadel/zitadel/internal/api/call"
)
func CallDurationHandler(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
next.ServeHTTP(w, r.WithContext(call.WithTimestamp(r.Context())))
})
}

1
internal/api/oidc/key.go
View File

@@ -187,6 +187,7 @@ func (o *OPStorage) getMaxKeySequence(ctx context.Context) (uint64, error) {
return o.eventstore.LatestSequence(ctx,
eventstore.NewSearchQueryBuilder(eventstore.ColumnsMaxSequence).
ResourceOwner(authz.GetInstance(ctx).InstanceID()).
AllowTimeTravel().
AddQuery().
AggregateTypes(keypair.AggregateType, instance.AggregateType).
Builder(),

8
internal/api/oidc/op.go
View File

@@ -2,7 +2,6 @@ package oidc
import (
"context"
"database/sql"
"fmt"
"net/http"
"time"
@@ -18,6 +17,7 @@ import (
"github.com/zitadel/zitadel/internal/auth/repository"
"github.com/zitadel/zitadel/internal/command"
"github.com/zitadel/zitadel/internal/crypto"
"github.com/zitadel/zitadel/internal/database"
caos_errs "github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/eventstore/handler/crdb"
@@ -73,7 +73,7 @@ type OPStorage struct {
assetAPIPrefix func(ctx context.Context) string
}
func NewProvider(ctx context.Context, config Config, defaultLogoutRedirectURI string, externalSecure bool, command *command.Commands, query *query.Queries, repo repository.Repository, encryptionAlg crypto.EncryptionAlgorithm, cryptoKey []byte, es *eventstore.Eventstore, projections *sql.DB, userAgentCookie, instanceHandler, accessHandler func(http.Handler) http.Handler) (op.OpenIDProvider, error) {
func NewProvider(ctx context.Context, config Config, defaultLogoutRedirectURI string, externalSecure bool, command *command.Commands, query *query.Queries, repo repository.Repository, encryptionAlg crypto.EncryptionAlgorithm, cryptoKey []byte, es *eventstore.Eventstore, projections *database.DB, userAgentCookie, instanceHandler, accessHandler func(http.Handler) http.Handler) (op.OpenIDProvider, error) {
opConfig, err := createOPConfig(config, defaultLogoutRedirectURI, cryptoKey)
if err != nil {
return nil, caos_errs.ThrowInternal(err, "OIDC-EGrqd", "cannot create op config: %w")
@@ -169,7 +169,7 @@ func customEndpoints(endpointConfig *EndpointConfig) []op.Option {
return options
}
func newStorage(config Config, command *command.Commands, query *query.Queries, repo repository.Repository, encAlg crypto.EncryptionAlgorithm, es *eventstore.Eventstore, projections *sql.DB, externalSecure bool) *OPStorage {
func newStorage(config Config, command *command.Commands, query *query.Queries, repo repository.Repository, encAlg crypto.EncryptionAlgorithm, es *eventstore.Eventstore, db *database.DB, externalSecure bool) *OPStorage {
return &OPStorage{
repo: repo,
command: command,
@@ -182,7 +182,7 @@ func newStorage(config Config, command *command.Commands, query *query.Queries,
defaultRefreshTokenIdleExpiration: config.DefaultRefreshTokenIdleExpiration,
defaultRefreshTokenExpiration: config.DefaultRefreshTokenExpiration,
encAlg: encAlg,
locker: crdb.NewLocker(projections, locksTable, signingKey),
locker: crdb.NewLocker(db.DB, locksTable, signingKey),
assetAPIPrefix: assets.AssetAPI(externalSecure),
}
}

8
internal/api/saml/provider.go
View File

@@ -2,7 +2,6 @@ package saml
import (
"context"
"database/sql"
"fmt"
"net/http"
@@ -14,6 +13,7 @@ import (
"github.com/zitadel/zitadel/internal/auth/repository"
"github.com/zitadel/zitadel/internal/command"
"github.com/zitadel/zitadel/internal/crypto"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/eventstore/handler/crdb"
"github.com/zitadel/zitadel/internal/query"
@@ -38,7 +38,7 @@ func NewProvider(
encAlg crypto.EncryptionAlgorithm,
certEncAlg crypto.EncryptionAlgorithm,
es *eventstore.Eventstore,
projections *sql.DB,
projections *database.DB,
instanceHandler,
userAgentCookie,
accessHandler func(http.Handler) http.Handler,
@@ -89,12 +89,12 @@ func newStorage(
encAlg crypto.EncryptionAlgorithm,
certEncAlg crypto.EncryptionAlgorithm,
es *eventstore.Eventstore,
projections *sql.DB,
db *database.DB,
) (*Storage, error) {
return &Storage{
encAlg: encAlg,
certEncAlg: certEncAlg,
locker: crdb.NewLocker(projections, locksTable, signingKey),
locker: crdb.NewLocker(db.DB, locksTable, signingKey),
eventstore: es,
repo: repo,
command: command,

4
internal/api/ui/console/console.go
View File

@@ -88,7 +88,7 @@ func (f *file) Stat() (_ fs.FileInfo, err error) {
return f, nil
}
func Start(config Config, externalSecure bool, issuer op.IssuerFromRequest, instanceHandler, accessInterceptor func(http.Handler) http.Handler, customerPortal string) (http.Handler, error) {
func Start(config Config, externalSecure bool, issuer op.IssuerFromRequest, callDurationInterceptor, instanceHandler, accessInterceptor func(http.Handler) http.Handler, customerPortal string) (http.Handler, error) {
fSys, err := fs.Sub(static, "static")
if err != nil {
return nil, err
@@ -103,7 +103,7 @@ func Start(config Config, externalSecure bool, issuer op.IssuerFromRequest, inst
handler := mux.NewRouter()
handler.Use(instanceHandler, security, accessInterceptor)
handler.Use(callDurationInterceptor, instanceHandler, security, accessInterceptor)
handler.Handle(envRequestPath, middleware.TelemetryHandler()(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
url := http_util.BuildOrigin(r.Host, externalSecure)
environmentJSON, err := createEnvironmentJSON(url, issuer(r), authz.GetInstance(r.Context()).ConsoleClientID(), customerPortal)

4
internal/api/ui/login/login.go
View File

@@ -67,8 +67,8 @@ func CreateLogin(config Config,
userAgentCookie,
issuerInterceptor,
oidcInstanceHandler,
samlInstanceHandler mux.MiddlewareFunc,
assetCache mux.MiddlewareFunc,
samlInstanceHandler,
assetCache,
accessHandler mux.MiddlewareFunc,
userCodeAlg crypto.EncryptionAlgorithm,
idpConfigAlg crypto.EncryptionAlgorithm,