feat: asset storage (#1696)

* feat: remove assets * feat: minio implementation * fix: remove assets from tests * feat: minio implementation * feat: Env vars * fix: sprintf * fix: sprintf * Update internal/eventstore/repository/repository.go Co-authored-by: Livio Amstutz <livio.a@gmail.com> * fix: error handling Co-authored-by: Livio Amstutz <livio.a@gmail.com>
2025-08-11 21:37:32 +00:00 · 2021-05-03 10:15:50 +02:00
parent a5c6bf5498
commit 667cc30291
110 changed files with 306 additions and 938 deletions
--- a/internal/static/i18n/de.yaml
+++ b/internal/static/i18n/de.yaml
@@ -4,6 +4,21 @@ Errors:
  OriginNotAllowed: Dieser "Origin" ist nicht freigeschaltet
  IDMissing: ID fehlt
  ResourceOwnerMissing: Organisation fehlt
+  Assets:
+    Store:
+      NotInitialized: Assets Speicher konnte nicht initialisiert werden
+    Bucket:
+      Internal: Interner Fehler beim erstellen eines Buckets
+      AlreadyExists: Bucket existiert bereits
+      CreateFailed: Bucket konnte nicht erstellt werden
+      ListFailed: Buckets konnten nicht gelesen werden
+      RemoveFailed: Bucket konnte nicht gelöscht werden
+    Object:
+      PutFailed: Objekt konnte nicht erstellt werden
+      GetFaieled: Objekt konnte nicht gelesen werden
+      PresignedTokenFailed: Signiertes Token konnte nicht erstellt werden
+      ListFailed: Objektliste konnte nicht gelesen werden
+      RemoveFailed: Objekt konnte nicht gelöscht werden
  Limit:
    ExceedsDefault: Limit überschreitet default Limit
  User:
--- a/internal/static/i18n/en.yaml
+++ b/internal/static/i18n/en.yaml
@@ -4,6 +4,21 @@ Errors:
  OriginNotAllowed: This "Origin" is not allowed
  IDMissing: ID missing
  ResourceOwnerMissing: Resource Owner Organisation missing
+  Assets:
+    Store:
+      NotInitialized: Assets storage not initialized
+    Bucket:
+      Internal: Internal error on create bucket
+      AlreadyExists: Bucket already exists
+      CreateFailed: Bucket not created
+      ListFailed: Buckets could not be read
+      RemoveFailed: Bucket not deleted
+    Object:
+      PutFailed: Objekt not created
+      GetFaieled: Objekt could not be read
+      PresignedTokenFailed: Signed token could not be created
+      ListFailed: Objectlist could not be read
+      RemoveFailed: Object could not be removed
  Limit:
    ExceedsDefault: Limit exceeds default limit
  User:
--- a/internal/static/s3/config.go
+++ b/internal/static/s3/config.go
@@ -0,0 +1,14 @@
+package s3
+
+type AssetStorage struct {
+	Type   string
+	Config S3Config
+}
+
+type S3Config struct {
+	Endpoint        string
+	AccessKeyID     string
+	SecretAccessKey string
+	SSL             bool
+	Location        string
+}
--- a/internal/static/s3/minio.go
+++ b/internal/static/s3/minio.go
@@ -0,0 +1,155 @@
+package s3
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/url"
+	"time"
+
+	"github.com/caos/logging"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+
+	"github.com/caos/zitadel/internal/domain"
+	caos_errs "github.com/caos/zitadel/internal/errors"
+)
+
+type Minio struct {
+	Client   *minio.Client
+	Location string
+}
+
+func NewMinio(config S3Config) (*Minio, error) {
+	minioClient, err := minio.New(config.Endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""),
+		Secure: config.SSL,
+		Region: config.Location,
+	})
+	if err != nil {
+		return nil, caos_errs.ThrowInternal(err, "MINIO-4m90d", "Errors.Assets.Store.NotInitialized")
+	}
+	return &Minio{
+		Client:   minioClient,
+		Location: config.Location,
+	}, nil
+}
+
+func (m *Minio) CreateBucket(ctx context.Context, name, location string) error {
+	if location == "" {
+		location = m.Location
+	}
+	exists, err := m.Client.BucketExists(ctx, name)
+	if err != nil {
+		return caos_errs.ThrowInternal(err, "MINIO-4m90d", "Errors.Assets.Bucket.Internal")
+	}
+	if exists {
+		return caos_errs.ThrowAlreadyExists(nil, "MINIO-9n3MK", "Errors.Assets.Bucket.AlreadyExists")
+	}
+	err = m.Client.MakeBucket(ctx, name, minio.MakeBucketOptions{Region: location})
+	if err != nil {
+		return caos_errs.ThrowInternal(err, "MINIO-4m90d", "Errors.Assets.Bucket.CreateFailed")
+	}
+	return nil
+}
+
+func (m *Minio) ListBuckets(ctx context.Context) ([]*domain.BucketInfo, error) {
+	infos, err := m.Client.ListBuckets(ctx)
+	if err != nil {
+		return nil, caos_errs.ThrowInternal(err, "MINIO-390OP", "Errors.Assets.Bucket.ListFailed")
+	}
+	buckets := make([]*domain.BucketInfo, len(infos))
+	for i, info := range infos {
+		buckets[i] = &domain.BucketInfo{
+			Name:         info.Name,
+			CreationDate: info.CreationDate,
+		}
+	}
+	return buckets, nil
+}
+
+func (m *Minio) RemoveBucket(ctx context.Context, name string) error {
+	err := m.Client.RemoveBucket(ctx, name)
+	if err != nil {
+		return caos_errs.ThrowInternal(err, "MINIO-338Hs", "Errors.Assets.Bucket.RemoveFailed")
+	}
+	return nil
+}
+
+func (m *Minio) PutObject(ctx context.Context, bucketName, objectName, contentType string, object io.Reader, objectSize int64) (*domain.AssetInfo, error) {
+	info, err := m.Client.PutObject(ctx, bucketName, objectName, object, objectSize, minio.PutObjectOptions{ContentType: contentType})
+	if err != nil {
+		return nil, caos_errs.ThrowInternal(err, "MINIO-590sw", "Errors.Assets.Object.PutFailed")
+	}
+	return &domain.AssetInfo{
+		Bucket:       info.Bucket,
+		Key:          info.Key,
+		ETag:         info.ETag,
+		Size:         info.Size,
+		LastModified: info.LastModified,
+		Location:     info.Location,
+		VersionID:    info.VersionID,
+	}, nil
+}
+
+func (m *Minio) GetObjectInfo(ctx context.Context, bucketName, objectName string) (*domain.AssetInfo, error) {
+	object, err := m.Client.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{})
+	if err != nil {
+		return nil, caos_errs.ThrowInternal(err, "MINIO-1vySX", "Errors.Assets.Object.GetFailed")
+	}
+	info, err := object.Stat()
+	if err != nil {
+		return nil, caos_errs.ThrowInternal(err, "MINIO-F96xF", "Errors.Assets.Object.GetFailed")
+	}
+	return m.objectToAssetInfo(bucketName, info), nil
+}
+
+func (m *Minio) GetObjectPresignedURL(ctx context.Context, bucketName, objectName string, expiration time.Duration) (*url.URL, error) {
+	reqParams := make(url.Values)
+	reqParams.Set("response-content-disposition", fmt.Sprintf("attachment; filename=\"%s\"", objectName))
+	presignedURL, err := m.Client.PresignedGetObject(ctx, bucketName, objectName, expiration, reqParams)
+	if err != nil {
+		return nil, caos_errs.ThrowInternal(err, "MINIO-19Mp0", "Errors.Assets.Object.PresignedTokenFailed")
+	}
+	return presignedURL, nil
+}
+
+func (m *Minio) ListObjectInfos(ctx context.Context, bucketName, prefix string, recursive bool) ([]*domain.AssetInfo, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	objectCh := m.Client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{
+		Prefix:    prefix,
+		Recursive: recursive,
+	})
+	assetInfos := make([]*domain.AssetInfo, 0)
+	for object := range objectCh {
+		if object.Err != nil {
+			logging.LogWithFields("MINIO-wC8sd", "bucket-name", bucketName, "prefix", prefix).WithError(object.Err).Debug("unable to get object")
+			return nil, caos_errs.ThrowInternal(object.Err, "MINIO-1m09S", "Errors.Assets.Object.ListFailed")
+		}
+		assetInfos = append(assetInfos, m.objectToAssetInfo(bucketName, object))
+	}
+	return assetInfos, nil
+}
+
+func (m *Minio) RemoveObject(ctx context.Context, bucketName, objectName string) error {
+	err := m.Client.RemoveObject(ctx, bucketName, objectName, minio.RemoveObjectOptions{})
+	if err != nil {
+		return caos_errs.ThrowInternal(err, "MINIO-x85RT", "Errors.Assets.Object.RemoveFailed")
+	}
+	return nil
+}
+
+func (m *Minio) objectToAssetInfo(bucketName string, object minio.ObjectInfo) *domain.AssetInfo {
+	return &domain.AssetInfo{
+		Bucket:          bucketName,
+		Key:             object.Key,
+		ETag:            object.ETag,
+		Size:            object.Size,
+		LastModified:    object.LastModified,
+		VersionID:       object.VersionID,
+		Expiration:      object.Expiration,
+		AutheticatedURL: m.Client.EndpointURL().String() + "/" + object.Key,
+	}
+}
--- a/internal/static/s3/storage_client.go
+++ b/internal/static/s3/storage_client.go
@@ -0,0 +1,21 @@
+package s3
+
+import (
+	"context"
+	"io"
+	"net/url"
+	"time"
+
+	"github.com/caos/zitadel/internal/domain"
+)
+
+type Client interface {
+	CreateBucket(ctx context.Context, name, location string) error
+	RemoveBucket(ctx context.Context, name string) error
+	ListBuckets(ctx context.Context) ([]*domain.BucketInfo, error)
+	PutObject(ctx context.Context, bucketName, objectName, contentType string, object io.Reader, objectSize int64) (*domain.AssetInfo, error)
+	GetObjectInfo(ctx context.Context, bucketName, objectName string) (*domain.AssetInfo, error)
+	ListObjectInfos(ctx context.Context, bucketName, prefix string) ([]*domain.AssetInfo, error)
+	GetObjectPresignedURL(ctx context.Context, bucketName, objectName string, expiration time.Duration) (*url.URL, error)
+	RemoveObject(ctx context.Context, bucketName, objectName string) error
+}