TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-12 08:57:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: store assets in database (#3290)

Browse Source 
* feat: use database as asset storage

* being only uploading assets if allowed

* tests

* fixes

* cleanup after merge

* renaming

* various fixes

* fix: change to repository event types and removed unused code

* feat: set default features

* error handling

* error handling and naming

* fix tests

* fix tests

* fix merge

* rename
This commit is contained in:
Livio Amstutz
2022-04-06 08:13:40 +02:00
committed by GitHub
parent b949b8fc65
commit 4a0d61d75a
36 changed files with 2016 additions and 967 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
internal/api/assets/asset.go
View File

@@ -1,24 +1,21 @@
package assets
import (
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"net/http"
"strconv"
"strings"
"time"
"github.com/caos/logging"
sentryhttp "github.com/getsentry/sentry-go/http"
"github.com/gorilla/mux"
"github.com/superseriousbusiness/exifremove/pkg/exifremove"
"github.com/caos/zitadel/internal/api/authz"
http_util "github.com/caos/zitadel/internal/api/http"
http_mw "github.com/caos/zitadel/internal/api/http/middleware"
"github.com/caos/zitadel/internal/command"
"github.com/caos/zitadel/internal/domain"
"github.com/caos/zitadel/internal/id"
"github.com/caos/zitadel/internal/query"
"github.com/caos/zitadel/internal/static"
@@ -54,26 +51,27 @@ func (h *Handler) Storage() static.Storage {
}
type Uploader interface {
Callback(ctx context.Context, info *domain.AssetInfo, orgID string, commands *command.Commands) error
UploadAsset(ctx context.Context, info string, asset *command.AssetUpload, commands *command.Commands) error
ObjectName(data authz.CtxData) (string, error)
BucketName(data authz.CtxData) string
ResourceOwner(instance authz.Instance, data authz.CtxData) string
ContentTypeAllowed(contentType string) bool
MaxFileSize() int64
ObjectType() static.ObjectType
}
type Downloader interface {
ObjectName(ctx context.Context, path string) (string, error)
BucketName(ctx context.Context, id string) string
ResourceOwner(ctx context.Context, ownerPath string) string
}
type ErrorHandler func(http.ResponseWriter, *http.Request, error, int)
func DefaultErrorHandler(w http.ResponseWriter, r *http.Request, err error, code int) {
logging.Log("ASSET-g5ef1").WithError(err).WithField("uri", r.RequestURI).Error("error occurred on asset api")
logging.WithFields("uri", r.RequestURI).WithError(err).Warn("error occurred on asset api")
http.Error(w, err.Error(), code)
}
func NewHandler(commands *command.Commands, verifier *authz.TokenVerifier, authConfig authz.Config, idGenerator id.Generator, storage static.Storage, queries *query.Queries) http.Handler {
func NewHandler(commands *command.Commands, verifier *authz.TokenVerifier, authConfig authz.Config, idGenerator id.Generator, storage static.Storage, queries *query.Queries, instanceInterceptor func(handler http.Handler) http.Handler) http.Handler {
h := &Handler{
commands: commands,
errorHandler: DefaultErrorHandler,
@@ -85,9 +83,9 @@ func NewHandler(commands *command.Commands, verifier *authz.TokenVerifier, authC
verifier.RegisterServer("Management-API", "assets", AssetsService_AuthMethods) //TODO: separate api?
router := mux.NewRouter()
router.Use(sentryhttp.New(sentryhttp.Options{}).Handle)
router.Use(sentryhttp.New(sentryhttp.Options{}).Handle, instanceInterceptor)
RegisterRoutes(router, h)
router.PathPrefix("/{id}").Methods("GET").HandlerFunc(DownloadHandleFunc(h, h.GetFile()))
router.PathPrefix("/{owner}").Methods("GET").HandlerFunc(DownloadHandleFunc(h, h.GetFile()))
return router
}
@@ -101,8 +99,8 @@ func (l *publicFileDownloader) ObjectName(_ context.Context, path string) (strin
return path, nil
}
func (l *publicFileDownloader) BucketName(_ context.Context, id string) string {
return id
func (l *publicFileDownloader) ResourceOwner(_ context.Context, ownerPath string) string {
return ownerPath
}
const maxMemory = 2 << 20
@@ -120,7 +118,7 @@ func UploadHandleFunc(s AssetsService, uploader Uploader) func(http.ResponseWrit
}
defer func() {
err = file.Close()
logging.Log("UPLOAD-GDg34").OnError(err).Warn("could not close file")
logging.OnError(err).Warn("could not close file")
}()
contentType := handler.Header.Get("content-type")
size := handler.Size
@@ -133,24 +131,21 @@ func UploadHandleFunc(s AssetsService, uploader Uploader) func(http.ResponseWrit
return
}
bucketName := uploader.BucketName(ctxData)
resourceOwner := uploader.ResourceOwner(authz.GetInstance(ctx), ctxData)
objectName, err := uploader.ObjectName(ctxData)
if err != nil {
s.ErrorHandler()(w, r, fmt.Errorf("upload failed: %v", err), http.StatusInternalServerError)
return
}
cleanedFile, cleanedSize, err := removeExif(file, size, contentType)
if err != nil {
s.ErrorHandler()(w, r, fmt.Errorf("remove exif error: %v", err), http.StatusInternalServerError)
return
uploadInfo := &command.AssetUpload{
ResourceOwner: resourceOwner,
ObjectName: objectName,
ContentType: contentType,
ObjectType: uploader.ObjectType(),
File: file,
Size: size,
}
info, err := s.Commands().UploadAsset(ctx, bucketName, objectName, contentType, cleanedFile, cleanedSize)
if err != nil {
s.ErrorHandler()(w, r, fmt.Errorf("upload failed: %v", err), http.StatusInternalServerError)
return
}
err = uploader.Callback(ctx, info, ctxData.OrgID, s.Commands())
err = uploader.UploadAsset(ctx, ctxData.OrgID, uploadInfo, s.Commands())
if err != nil {
s.ErrorHandler()(w, r, fmt.Errorf("upload failed: %v", err), http.StatusInternalServerError)
return
@@ -164,11 +159,11 @@ func DownloadHandleFunc(s AssetsService, downloader Downloader) func(http.Respon
return
}
ctx := r.Context()
id := mux.Vars(r)["id"]
bucketName := downloader.BucketName(ctx, id)
ownerPath := mux.Vars(r)["owner"]
resourceOwner := downloader.ResourceOwner(ctx, ownerPath)
path := ""
if id != "" {
path = strings.Split(r.RequestURI, id+"/")[1]
if ownerPath != "" {
path = strings.Split(r.RequestURI, ownerPath+"/")[1]
}
objectName, err := downloader.ObjectName(ctx, path)
if err != nil {
@@ -176,49 +171,33 @@ func DownloadHandleFunc(s AssetsService, downloader Downloader) func(http.Respon
return
}
if objectName == "" {
s.ErrorHandler()(w, r, fmt.Errorf("file not found: %v", objectName), http.StatusNotFound)
s.ErrorHandler()(w, r, fmt.Errorf("file not found: %v", path), http.StatusNotFound)
return
}
reader, getInfo, err := s.Storage().GetObject(ctx, bucketName, objectName)
if err != nil {
s.ErrorHandler()(w, r, fmt.Errorf("download failed: %v", err), http.StatusInternalServerError)
return
if err = GetAsset(w, r, resourceOwner, objectName, s.Storage()); err != nil {
s.ErrorHandler()(w, r, err, http.StatusInternalServerError)
}
data, err := ioutil.ReadAll(reader)
if err != nil {
s.ErrorHandler()(w, r, fmt.Errorf("download failed: %v", err), http.StatusInternalServerError)
return
}
info, err := getInfo()
if err != nil {
s.ErrorHandler()(w, r, fmt.Errorf("download failed: %v", err), http.StatusInternalServerError)
return
}
w.Header().Set("content-length", strconv.FormatInt(info.Size, 10))
w.Header().Set("content-type", info.ContentType)
w.Header().Set("ETag", info.ETag)
w.Write(data)
}
}
func removeExif(file io.Reader, size int64, contentType string) (io.Reader, int64, error) {
if !isAllowedContentType(contentType) {
return file, size, nil
}
buf := new(bytes.Buffer)
_, err := buf.ReadFrom(file)
func GetAsset(w http.ResponseWriter, r *http.Request, resourceOwner, objectName string, storage static.Storage) error {
data, getInfo, err := storage.GetObject(r.Context(), authz.GetInstance(r.Context()).InstanceID(), resourceOwner, objectName)
if err != nil {
return file, 0, err
return fmt.Errorf("download failed: %v", err)
}
data, err := exifremove.Remove(buf.Bytes())
info, err := getInfo()
if err != nil {
return nil, 0, err
return fmt.Errorf("download failed: %v", err)
}
return bytes.NewReader(data), int64(len(data)), nil
}
func isAllowedContentType(contentType string) bool {
return strings.HasSuffix(contentType, "png") ||
strings.HasSuffix(contentType, "jpg") ||
strings.HasSuffix(contentType, "jpeg")
if info.Hash == r.Header.Get(http_util.IfNoneMatch) {
w.WriteHeader(304)
return nil
}
w.Header().Set(http_util.ContentLength, strconv.FormatInt(info.Size, 10))
w.Header().Set(http_util.ContentType, info.ContentType)
w.Header().Set(http_util.LastModified, info.LastModified.Format(time.RFC1123))
w.Header().Set(http_util.Etag, info.Hash)
_, err = w.Write(data)
logging.New().OnError(err).Error("error writing response for asset")
return nil
}

4
internal/api/assets/generator/asset.yaml
View File

@@ -1,6 +1,6 @@
Services:
IAM:
Prefix: "/iam"
Prefix: "/instance"
Methods:
DefaultLabelPolicyLogo:
Path: "/policy/label/logo"
@@ -116,4 +116,4 @@ Services:
- Name: Get
Comment:
Type: download
Permission: authenticated
Permission: authenticated

69
internal/api/assets/login_policy.go
View File

@@ -9,6 +9,7 @@ import (
"github.com/caos/zitadel/internal/domain"
"github.com/caos/zitadel/internal/id"
"github.com/caos/zitadel/internal/query"
"github.com/caos/zitadel/internal/static"
)
func (h *Handler) UploadDefaultLabelPolicyLogo() Uploader {
@@ -44,6 +45,10 @@ func (l *labelPolicyLogoUploader) ContentTypeAllowed(contentType string) bool {
return false
}
func (l *labelPolicyLogoUploader) ObjectType() static.ObjectType {
return static.ObjectTypeStyling
}
func (l *labelPolicyLogoUploader) MaxFileSize() int64 {
return l.maxSize
}
@@ -60,27 +65,27 @@ func (l *labelPolicyLogoUploader) ObjectName(_ authz.CtxData) (string, error) {
return prefix + "-" + suffixID, nil
}
func (l *labelPolicyLogoUploader) BucketName(ctxData authz.CtxData) string {
func (l *labelPolicyLogoUploader) ResourceOwner(instance authz.Instance, ctxData authz.CtxData) string {
if l.defaultPolicy {
return domain.IAMID
return instance.InstanceID()
}
return ctxData.OrgID
}
func (l *labelPolicyLogoUploader) Callback(ctx context.Context, info *domain.AssetInfo, orgID string, commands *command.Commands) error {
func (l *labelPolicyLogoUploader) UploadAsset(ctx context.Context, orgID string, upload *command.AssetUpload, commands *command.Commands) error {
if l.defaultPolicy {
if l.darkMode {
_, err := commands.AddLogoDarkDefaultLabelPolicy(ctx, info.Key)
_, err := commands.AddLogoDarkDefaultLabelPolicy(ctx, upload)
return err
}
_, err := commands.AddLogoDefaultLabelPolicy(ctx, info.Key)
_, err := commands.AddLogoDefaultLabelPolicy(ctx, upload)
return err
}
if l.darkMode {
_, err := commands.AddLogoDarkLabelPolicy(ctx, orgID, info.Key)
_, err := commands.AddLogoDarkLabelPolicy(ctx, orgID, upload)
return err
}
_, err := commands.AddLogoLabelPolicy(ctx, orgID, info.Key)
_, err := commands.AddLogoLabelPolicy(ctx, orgID, upload)
return err
}
@@ -134,8 +139,8 @@ func (l *labelPolicyLogoDownloader) ObjectName(ctx context.Context, path string)
return policy.Light.LogoURL, nil
}
func (l *labelPolicyLogoDownloader) BucketName(ctx context.Context, id string) string {
return getLabelPolicyBucketName(ctx, l.defaultPolicy, l.preview, l.query)
func (l *labelPolicyLogoDownloader) ResourceOwner(ctx context.Context, _ string) string {
return getLabelPolicyResourceOwner(ctx, l.defaultPolicy, l.preview, l.query)
}
func (h *Handler) UploadDefaultLabelPolicyIcon() Uploader {
@@ -171,6 +176,10 @@ func (l *labelPolicyIconUploader) ContentTypeAllowed(contentType string) bool {
return false
}
func (l *labelPolicyIconUploader) ObjectType() static.ObjectType {
return static.ObjectTypeStyling
}
func (l *labelPolicyIconUploader) MaxFileSize() int64 {
return l.maxSize
}
@@ -187,28 +196,28 @@ func (l *labelPolicyIconUploader) ObjectName(_ authz.CtxData) (string, error) {
return prefix + "-" + suffixID, nil
}
func (l *labelPolicyIconUploader) BucketName(ctxData authz.CtxData) string {
func (l *labelPolicyIconUploader) ResourceOwner(instance authz.Instance, ctxData authz.CtxData) string {
if l.defaultPolicy {
return domain.IAMID
return instance.InstanceID()
}
return ctxData.OrgID
}
func (l *labelPolicyIconUploader) Callback(ctx context.Context, info *domain.AssetInfo, orgID string, commands *command.Commands) error {
func (l *labelPolicyIconUploader) UploadAsset(ctx context.Context, orgID string, upload *command.AssetUpload, commands *command.Commands) error {
if l.defaultPolicy {
if l.darkMode {
_, err := commands.AddIconDarkDefaultLabelPolicy(ctx, info.Key)
_, err := commands.AddIconDarkDefaultLabelPolicy(ctx, upload)
return err
}
_, err := commands.AddIconDefaultLabelPolicy(ctx, info.Key)
_, err := commands.AddIconDefaultLabelPolicy(ctx, upload)
return err
}
if l.darkMode {
_, err := commands.AddIconDarkLabelPolicy(ctx, orgID, info.Key)
_, err := commands.AddIconDarkLabelPolicy(ctx, orgID, upload)
return err
}
_, err := commands.AddIconLabelPolicy(ctx, orgID, info.Key)
_, err := commands.AddIconLabelPolicy(ctx, orgID, upload)
return err
}
@@ -262,8 +271,8 @@ func (l *labelPolicyIconDownloader) ObjectName(ctx context.Context, path string)
return policy.Light.IconURL, nil
}
func (l *labelPolicyIconDownloader) BucketName(ctx context.Context, id string) string {
return getLabelPolicyBucketName(ctx, l.defaultPolicy, l.preview, l.query)
func (l *labelPolicyIconDownloader) ResourceOwner(ctx context.Context, _ string) string {
return getLabelPolicyResourceOwner(ctx, l.defaultPolicy, l.preview, l.query)
}
func (h *Handler) UploadDefaultLabelPolicyFont() Uploader {
@@ -290,6 +299,10 @@ func (l *labelPolicyFontUploader) ContentTypeAllowed(contentType string) bool {
return false
}
func (l *labelPolicyFontUploader) ObjectType() static.ObjectType {
return static.ObjectTypeStyling
}
func (l *labelPolicyFontUploader) MaxFileSize() int64 {
return l.maxSize
}
@@ -303,19 +316,19 @@ func (l *labelPolicyFontUploader) ObjectName(_ authz.CtxData) (string, error) {
return prefix + "-" + suffixID, nil
}
func (l *labelPolicyFontUploader) BucketName(ctxData authz.CtxData) string {
func (l *labelPolicyFontUploader) ResourceOwner(instance authz.Instance, ctxData authz.CtxData) string {
if l.defaultPolicy {
return domain.IAMID
return instance.InstanceID()
}
return ctxData.OrgID
}
func (l *labelPolicyFontUploader) Callback(ctx context.Context, info *domain.AssetInfo, orgID string, commands *command.Commands) error {
func (l *labelPolicyFontUploader) UploadAsset(ctx context.Context, orgID string, upload *command.AssetUpload, commands *command.Commands) error {
if l.defaultPolicy {
_, err := commands.AddFontDefaultLabelPolicy(ctx, info.Key)
_, err := commands.AddFontDefaultLabelPolicy(ctx, upload)
return err
}
_, err := commands.AddFontLabelPolicy(ctx, orgID, info.Key)
_, err := commands.AddFontLabelPolicy(ctx, orgID, upload)
return err
}
@@ -349,8 +362,8 @@ func (l *labelPolicyFontDownloader) ObjectName(ctx context.Context, path string)
return policy.FontURL, nil
}
func (l *labelPolicyFontDownloader) BucketName(ctx context.Context, id string) string {
return getLabelPolicyBucketName(ctx, l.defaultPolicy, l.preview, l.query)
func (l *labelPolicyFontDownloader) ResourceOwner(ctx context.Context, _ string) string {
return getLabelPolicyResourceOwner(ctx, l.defaultPolicy, l.preview, l.query)
}
func getLabelPolicy(ctx context.Context, defaultPolicy, preview bool, queries *query.Queries) (*query.LabelPolicy, error) {
@@ -366,16 +379,16 @@ func getLabelPolicy(ctx context.Context, defaultPolicy, preview bool, queries *q
return queries.ActiveLabelPolicyByOrg(ctx, authz.GetCtxData(ctx).OrgID)
}
func getLabelPolicyBucketName(ctx context.Context, defaultPolicy, preview bool, queries *query.Queries) string {
func getLabelPolicyResourceOwner(ctx context.Context, defaultPolicy, preview bool, queries *query.Queries) string {
if defaultPolicy {
return domain.IAMID
return authz.GetInstance(ctx).InstanceID()
}
policy, err := getLabelPolicy(ctx, defaultPolicy, preview, queries)
if err != nil {
return ""
}
if policy.IsDefault {
return domain.IAMID
return authz.GetInstance(ctx).InstanceID()
}
return authz.GetCtxData(ctx).OrgID
}

17
internal/api/assets/user_avatar.go
View File

@@ -7,6 +7,7 @@ import (
"github.com/caos/zitadel/internal/api/authz"
"github.com/caos/zitadel/internal/command"
"github.com/caos/zitadel/internal/domain"
"github.com/caos/zitadel/internal/static"
)
func (h *Handler) UploadMyUserAvatar() Uploader {
@@ -27,6 +28,10 @@ func (l *myHumanAvatarUploader) ContentTypeAllowed(contentType string) bool {
return false
}
func (l *myHumanAvatarUploader) ObjectType() static.ObjectType {
return static.ObjectTypeUserAvatar
}
func (l *myHumanAvatarUploader) MaxFileSize() int64 {
return l.maxSize
}
@@ -35,12 +40,12 @@ func (l *myHumanAvatarUploader) ObjectName(ctxData authz.CtxData) (string, error
return domain.GetHumanAvatarAssetPath(ctxData.UserID), nil
}
func (l *myHumanAvatarUploader) BucketName(ctxData authz.CtxData) string {
return ctxData.OrgID
func (l *myHumanAvatarUploader) ResourceOwner(_ authz.Instance, ctxData authz.CtxData) string {
return ctxData.ResourceOwner
}
func (l *myHumanAvatarUploader) Callback(ctx context.Context, info *domain.AssetInfo, orgID string, commands *command.Commands) error {
_, err := commands.AddHumanAvatar(ctx, orgID, authz.GetCtxData(ctx).UserID, info.Key)
func (l *myHumanAvatarUploader) UploadAsset(ctx context.Context, orgID string, upload *command.AssetUpload, commands *command.Commands) error {
_, err := commands.AddHumanAvatar(ctx, orgID, authz.GetCtxData(ctx).UserID, upload)
return err
}
@@ -54,6 +59,6 @@ func (l *myHumanAvatarDownloader) ObjectName(ctx context.Context, path string) (
return domain.GetHumanAvatarAssetPath(authz.GetCtxData(ctx).UserID), nil
}
func (l *myHumanAvatarDownloader) BucketName(ctx context.Context, id string) string {
return authz.GetCtxData(ctx).OrgID
func (l *myHumanAvatarDownloader) ResourceOwner(ctx context.Context, _ string) string {
return authz.GetCtxData(ctx).ResourceOwner
}