feat: store assets in database (#3290)

* feat: use database as asset storage

* being only uploading assets if allowed

* tests

* fixes

* cleanup after merge

* renaming

* various fixes

* fix: change to repository event types and removed unused code

* feat: set default features

* error handling

* error handling and naming

* fix tests

* fix tests

* fix merge

* rename

internal/api/assets/asset.go

@@ -1,24 +1,21 @@
 package assets
 
 import (
-	"bytes"
 	"context"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"net/http"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/caos/logging"
 	sentryhttp "github.com/getsentry/sentry-go/http"
 	"github.com/gorilla/mux"
-	"github.com/superseriousbusiness/exifremove/pkg/exifremove"
 
 	"github.com/caos/zitadel/internal/api/authz"
+	http_util "github.com/caos/zitadel/internal/api/http"
 	http_mw "github.com/caos/zitadel/internal/api/http/middleware"
 	"github.com/caos/zitadel/internal/command"
-	"github.com/caos/zitadel/internal/domain"
 	"github.com/caos/zitadel/internal/id"
 	"github.com/caos/zitadel/internal/query"
 	"github.com/caos/zitadel/internal/static"
@@ -54,26 +51,27 @@ func (h *Handler) Storage() static.Storage {
 }
 
 type Uploader interface {
-	Callback(ctx context.Context, info *domain.AssetInfo, orgID string, commands *command.Commands) error
+	UploadAsset(ctx context.Context, info string, asset *command.AssetUpload, commands *command.Commands) error
 	ObjectName(data authz.CtxData) (string, error)
-	BucketName(data authz.CtxData) string
+	ResourceOwner(instance authz.Instance, data authz.CtxData) string
 	ContentTypeAllowed(contentType string) bool
 	MaxFileSize() int64
+	ObjectType() static.ObjectType
 }
 
 type Downloader interface {
 	ObjectName(ctx context.Context, path string) (string, error)
-	BucketName(ctx context.Context, id string) string
+	ResourceOwner(ctx context.Context, ownerPath string) string
 }
 
 type ErrorHandler func(http.ResponseWriter, *http.Request, error, int)
 
 func DefaultErrorHandler(w http.ResponseWriter, r *http.Request, err error, code int) {
-	logging.Log("ASSET-g5ef1").WithError(err).WithField("uri", r.RequestURI).Error("error occurred on asset api")
+	logging.WithFields("uri", r.RequestURI).WithError(err).Warn("error occurred on asset api")
 	http.Error(w, err.Error(), code)
 }
 
-func NewHandler(commands *command.Commands, verifier *authz.TokenVerifier, authConfig authz.Config, idGenerator id.Generator, storage static.Storage, queries *query.Queries) http.Handler {
+func NewHandler(commands *command.Commands, verifier *authz.TokenVerifier, authConfig authz.Config, idGenerator id.Generator, storage static.Storage, queries *query.Queries, instanceInterceptor func(handler http.Handler) http.Handler) http.Handler {
 	h := &Handler{
 		commands:        commands,
 		errorHandler:    DefaultErrorHandler,
@@ -85,9 +83,9 @@ func NewHandler(commands *command.Commands, verifier *authz.TokenVerifier, authC
 
 	verifier.RegisterServer("Management-API", "assets", AssetsService_AuthMethods) //TODO: separate api?
 	router := mux.NewRouter()
-	router.Use(sentryhttp.New(sentryhttp.Options{}).Handle)
+	router.Use(sentryhttp.New(sentryhttp.Options{}).Handle, instanceInterceptor)
 	RegisterRoutes(router, h)
-	router.PathPrefix("/{id}").Methods("GET").HandlerFunc(DownloadHandleFunc(h, h.GetFile()))
+	router.PathPrefix("/{owner}").Methods("GET").HandlerFunc(DownloadHandleFunc(h, h.GetFile()))
 	return router
 }
 
@@ -101,8 +99,8 @@ func (l *publicFileDownloader) ObjectName(_ context.Context, path string) (strin
 	return path, nil
 }
 
-func (l *publicFileDownloader) BucketName(_ context.Context, id string) string {
-	return id
+func (l *publicFileDownloader) ResourceOwner(_ context.Context, ownerPath string) string {
+	return ownerPath
 }
 
 const maxMemory = 2 << 20
@@ -120,7 +118,7 @@ func UploadHandleFunc(s AssetsService, uploader Uploader) func(http.ResponseWrit
 		}
 		defer func() {
 			err = file.Close()
-			logging.Log("UPLOAD-GDg34").OnError(err).Warn("could not close file")
+			logging.OnError(err).Warn("could not close file")
 		}()
 		contentType := handler.Header.Get("content-type")
 		size := handler.Size
@@ -133,24 +131,21 @@ func UploadHandleFunc(s AssetsService, uploader Uploader) func(http.ResponseWrit
 			return
 		}
 
-		bucketName := uploader.BucketName(ctxData)
+		resourceOwner := uploader.ResourceOwner(authz.GetInstance(ctx), ctxData)
 		objectName, err := uploader.ObjectName(ctxData)
 		if err != nil {
 			s.ErrorHandler()(w, r, fmt.Errorf("upload failed: %v", err), http.StatusInternalServerError)
 			return
 		}
-		cleanedFile, cleanedSize, err := removeExif(file, size, contentType)
-		if err != nil {
-			s.ErrorHandler()(w, r, fmt.Errorf("remove exif error: %v", err), http.StatusInternalServerError)
-			return
+		uploadInfo := &command.AssetUpload{
+			ResourceOwner: resourceOwner,
+			ObjectName:    objectName,
+			ContentType:   contentType,
+			ObjectType:    uploader.ObjectType(),
+			File:          file,
+			Size:          size,
 		}
-
-		info, err := s.Commands().UploadAsset(ctx, bucketName, objectName, contentType, cleanedFile, cleanedSize)
-		if err != nil {
-			s.ErrorHandler()(w, r, fmt.Errorf("upload failed: %v", err), http.StatusInternalServerError)
-			return
-		}
-		err = uploader.Callback(ctx, info, ctxData.OrgID, s.Commands())
+		err = uploader.UploadAsset(ctx, ctxData.OrgID, uploadInfo, s.Commands())
 		if err != nil {
 			s.ErrorHandler()(w, r, fmt.Errorf("upload failed: %v", err), http.StatusInternalServerError)
 			return
@@ -164,11 +159,11 @@ func DownloadHandleFunc(s AssetsService, downloader Downloader) func(http.Respon
 			return
 		}
 		ctx := r.Context()
-		id := mux.Vars(r)["id"]
-		bucketName := downloader.BucketName(ctx, id)
+		ownerPath := mux.Vars(r)["owner"]
+		resourceOwner := downloader.ResourceOwner(ctx, ownerPath)
 		path := ""
-		if id != "" {
-			path = strings.Split(r.RequestURI, id+"/")[1]
+		if ownerPath != "" {
+			path = strings.Split(r.RequestURI, ownerPath+"/")[1]
 		}
 		objectName, err := downloader.ObjectName(ctx, path)
 		if err != nil {
@@ -176,49 +171,33 @@ func DownloadHandleFunc(s AssetsService, downloader Downloader) func(http.Respon
 			return
 		}
 		if objectName == "" {
-			s.ErrorHandler()(w, r, fmt.Errorf("file not found: %v", objectName), http.StatusNotFound)
+			s.ErrorHandler()(w, r, fmt.Errorf("file not found: %v", path), http.StatusNotFound)
 			return
 		}
-		reader, getInfo, err := s.Storage().GetObject(ctx, bucketName, objectName)
-		if err != nil {
-			s.ErrorHandler()(w, r, fmt.Errorf("download failed: %v", err), http.StatusInternalServerError)
-			return
+		if err = GetAsset(w, r, resourceOwner, objectName, s.Storage()); err != nil {
+			s.ErrorHandler()(w, r, err, http.StatusInternalServerError)
 		}
-		data, err := ioutil.ReadAll(reader)
-		if err != nil {
-			s.ErrorHandler()(w, r, fmt.Errorf("download failed: %v", err), http.StatusInternalServerError)
-			return
-		}
-		info, err := getInfo()
-		if err != nil {
-			s.ErrorHandler()(w, r, fmt.Errorf("download failed: %v", err), http.StatusInternalServerError)
-			return
-		}
-		w.Header().Set("content-length", strconv.FormatInt(info.Size, 10))
-		w.Header().Set("content-type", info.ContentType)
-		w.Header().Set("ETag", info.ETag)
-		w.Write(data)
 	}
 }
 
-func removeExif(file io.Reader, size int64, contentType string) (io.Reader, int64, error) {
-	if !isAllowedContentType(contentType) {
-		return file, size, nil
-	}
-	buf := new(bytes.Buffer)
-	_, err := buf.ReadFrom(file)
+func GetAsset(w http.ResponseWriter, r *http.Request, resourceOwner, objectName string, storage static.Storage) error {
+	data, getInfo, err := storage.GetObject(r.Context(), authz.GetInstance(r.Context()).InstanceID(), resourceOwner, objectName)
 	if err != nil {
-		return file, 0, err
+		return fmt.Errorf("download failed: %v", err)
 	}
-	data, err := exifremove.Remove(buf.Bytes())
+	info, err := getInfo()
 	if err != nil {
-		return nil, 0, err
+		return fmt.Errorf("download failed: %v", err)
 	}
-	return bytes.NewReader(data), int64(len(data)), nil
-}
-
-func isAllowedContentType(contentType string) bool {
-	return strings.HasSuffix(contentType, "png") ||
-		strings.HasSuffix(contentType, "jpg") ||
-		strings.HasSuffix(contentType, "jpeg")
+	if info.Hash == r.Header.Get(http_util.IfNoneMatch) {
+		w.WriteHeader(304)
+		return nil
+	}
+	w.Header().Set(http_util.ContentLength, strconv.FormatInt(info.Size, 10))
+	w.Header().Set(http_util.ContentType, info.ContentType)
+	w.Header().Set(http_util.LastModified, info.LastModified.Format(time.RFC1123))
+	w.Header().Set(http_util.Etag, info.Hash)
+	_, err = w.Write(data)
+	logging.New().OnError(err).Error("error writing response for asset")
+	return nil
 }