feat: store assets in database (#3290)

* feat: use database as asset storage * being only uploading assets if allowed * tests * fixes * cleanup after merge * renaming * various fixes * fix: change to repository event types and removed unused code * feat: set default features * error handling * error handling and naming * fix tests * fix tests * fix merge * rename
2025-08-11 21:37:32 +00:00 · 2022-04-06 08:13:40 +02:00
parent b949b8fc65
commit 4a0d61d75a
36 changed files with 2016 additions and 967 deletions
--- a/cmd/admin/setup/02.go
+++ b/cmd/admin/setup/02.go
@@ -2,21 +2,36 @@ package setup

 import (
 	"context"
-
-	command "github.com/caos/zitadel/internal/command/v2"
+	"database/sql"
+	_ "embed"
 )

-type DefaultInstance struct {
-	cmd           *command.Command
-	InstanceSetup command.InstanceSetup
+const (
+	createAssets = `
+CREATE TABLE system.assets (
+    instance_id TEXT,
+    asset_type TEXT,
+    resource_owner TEXT,
+    name TEXT,
+    content_type TEXT,
+    hash TEXT AS (md5(data)) STORED,
+    data BYTES,
+    updated_at TIMESTAMPTZ,
+
+    PRIMARY KEY (instance_id, resource_owner, name)
+);
+`
+)
+
+type AssetTable struct {
+	dbClient *sql.DB
 }

-func (mig *DefaultInstance) Execute(ctx context.Context) error {
-	_, err := mig.cmd.SetUpInstance(ctx, &mig.InstanceSetup)
-
+func (mig *AssetTable) Execute(ctx context.Context) error {
+	_, err := mig.dbClient.ExecContext(ctx, createAssets)
 	return err
 }

-func (mig *DefaultInstance) String() string {
-	return "02_default_instance"
+func (mig *AssetTable) String() string {
+	return "02_assets"
 }
--- a/cmd/admin/setup/03.go
+++ b/cmd/admin/setup/03.go
@@ -0,0 +1,22 @@
+package setup
+
+import (
+	"context"
+
+	"github.com/caos/zitadel/internal/command/v2"
+)
+
+type DefaultInstance struct {
+	cmd           *command.Command
+	InstanceSetup command.InstanceSetup
+}
+
+func (mig *DefaultInstance) Execute(ctx context.Context) error {
+	_, err := mig.cmd.SetUpInstance(ctx, &mig.InstanceSetup)
+
+	return err
+}
+
+func (mig *DefaultInstance) String() string {
+	return "03_default_instance"
+}
--- a/cmd/admin/setup/config.go
+++ b/cmd/admin/setup/config.go
@@ -35,8 +35,9 @@ func MustNewConfig(v *viper.Viper) *Config {
 }

 type Steps struct {
-	S1ProjectionTable *ProjectionTable
-	S2DefaultInstance *DefaultInstance
+	s1ProjectionTable *ProjectionTable
+	s2AssetsTable     *AssetTable
+	S3DefaultInstance *DefaultInstance
 }

 func MustNewSteps(v *viper.Viper) *Steps {
--- a/cmd/admin/setup/setup.go
+++ b/cmd/admin/setup/setup.go
@@ -9,7 +9,7 @@ import (
 	"github.com/spf13/viper"

 	http_util "github.com/caos/zitadel/internal/api/http"
-	command "github.com/caos/zitadel/internal/command/v2"
+	"github.com/caos/zitadel/internal/command/v2"
 	"github.com/caos/zitadel/internal/database"
 	"github.com/caos/zitadel/internal/eventstore"
 	"github.com/caos/zitadel/internal/migration"
@@ -46,12 +46,14 @@ func Setup(config *Config, steps *Steps) {

 	cmd := command.New(eventstoreClient, "localhost", config.SystemDefaults)

-	steps.S2DefaultInstance.cmd = cmd
-	steps.S1ProjectionTable = &ProjectionTable{dbClient: dbClient}
-	steps.S2DefaultInstance.InstanceSetup.Zitadel.IsDevMode = !config.ExternalSecure
-	steps.S2DefaultInstance.InstanceSetup.Zitadel.BaseURL = http_util.BuildHTTP(config.ExternalDomain, config.ExternalPort, config.ExternalSecure)
+	steps.s1ProjectionTable = &ProjectionTable{dbClient: dbClient}
+	steps.s2AssetsTable = &AssetTable{dbClient: dbClient}
+	steps.S3DefaultInstance.cmd = cmd
+	steps.S3DefaultInstance.InstanceSetup.Zitadel.IsDevMode = !config.ExternalSecure
+	steps.S3DefaultInstance.InstanceSetup.Zitadel.BaseURL = http_util.BuildHTTP(config.ExternalDomain, config.ExternalPort, config.ExternalSecure)

 	ctx := context.Background()
-	migration.Migrate(ctx, eventstoreClient, steps.S1ProjectionTable)
-	migration.Migrate(ctx, eventstoreClient, steps.S2DefaultInstance)
+	migration.Migrate(ctx, eventstoreClient, steps.s1ProjectionTable)
+	migration.Migrate(ctx, eventstoreClient, steps.s2AssetsTable)
+	migration.Migrate(ctx, eventstoreClient, steps.S3DefaultInstance)
 }
--- a/cmd/admin/setup/steps.yaml
+++ b/cmd/admin/setup/steps.yaml
@@ -1,4 +1,4 @@
-S2DefaultInstance:
+S3DefaultInstance:
  InstanceSetup:
    Org:
      Name: ZITADEL
@@ -13,6 +13,29 @@ S2DefaultInstance:
        Gender:
        Phone:
        Password: Password1!
+    Features:
+      TierName: Default Tier
+      TierDescription: ""
+      State: 1 #active
+      StateDescription: ""
+      Retention: 8760h #1year
+      LoginPolicyFactors: true
+      LoginPolicyIDP: true
+      LoginPolicyPasswordless: true
+      LoginPolicyRegistration: true
+      LoginPolicyUsernameLogin: true
+      LoginPolicyPasswordReset: true
+      PasswordComplexityPolicy: true
+      LabelPolicyPrivateLabel: true
+      LabelPolicyWatermark: true
+      CustomDomain: true
+      PrivacyPolicy: true
+      MetadataUser: true
+      CustomTextMessage: true
+      CustomTextLogin: true
+      LockoutPolicy: true
+      ActionsAllowed: 2 #ActionsAllowedUnlimited
+      MaxActions: #not necessary because of ActionsAllowedUnlimited
    PasswordComplexityPolicy: 
      MinLength: 8
      HasLowercase: true 
--- a/cmd/admin/start/start.go
+++ b/cmd/admin/start/start.go
@@ -94,12 +94,6 @@ func startZitadel(config *Config, masterKey string) error {
 		return err
 	}

-	var storage static.Storage
-	//TODO: enable when storage is implemented again
-	//if *assetsEnabled {
-	//storage, err = config.AssetStorage.Config.NewStorage()
-	//logging.Log("MAIN-Bfhe2").OnError(err).Fatal("Unable to start asset storage")
-	//}
 	eventstoreClient, err := eventstore.Start(dbClient)
 	if err != nil {
 		return fmt.Errorf("cannot start eventstore for queries: %w", err)
@@ -114,6 +108,11 @@ func startZitadel(config *Config, masterKey string) error {
 	if err != nil {
 		return fmt.Errorf("error starting authz repo: %w", err)
 	}
+
+	storage, err := config.AssetStorage.NewStorage(dbClient)
+	if err != nil {
+		return fmt.Errorf("cannot start asset storage client: %w", err)
+	}
 	webAuthNConfig := webauthn.Config{
 		ID:          config.ExternalDomain,
 		Origin:      http_util.BuildHTTP(config.ExternalDomain, config.ExternalPort, config.ExternalSecure),
@@ -163,13 +162,13 @@ func startAPIs(ctx context.Context, router *mux.Router, commands *command.Comman
 		return err
 	}

-	apis.RegisterHandler(assets.HandlerPrefix, assets.NewHandler(commands, verifier, config.InternalAuthZ, id.SonyFlakeGenerator, store, queries))
+	instanceInterceptor := middleware.InstanceInterceptor(queries, config.HTTP1HostHeader)
+	apis.RegisterHandler(assets.HandlerPrefix, assets.NewHandler(commands, verifier, config.InternalAuthZ, id.SonyFlakeGenerator, store, queries, instanceInterceptor.Handler))

 	userAgentInterceptor, err := middleware.NewUserAgentHandler(config.UserAgentCookie, keys.UserAgentCookieKey, config.ExternalDomain, id.SonyFlakeGenerator, config.ExternalSecure)
 	if err != nil {
 		return err
 	}
-	instanceInterceptor := middleware.InstanceInterceptor(queries, config.HTTP1HostHeader)

 	issuer := oidc.Issuer(config.ExternalDomain, config.ExternalPort, config.ExternalSecure)
 	oidcProvider, err := oidc.NewProvider(ctx, config.OIDC, issuer, login.DefaultLoggedOutPath, commands, queries, authRepo, config.SystemDefaults.KeyConfig, keys.OIDC, keys.OIDCKey, eventstore, dbClient, keyChan, userAgentInterceptor, instanceInterceptor.Handler)