feat: bulk scim v2 endpoint (#9256)

# Which Problems Are Solved
* Adds support for the bulk SCIM v2 endpoint

# How the Problems Are Solved
* Adds support for the bulk SCIM v2 endpoint under `POST
/scim/v2/{orgID}/Bulk`

# Additional Context
Part of #8140

Co-authored-by: Stefan Benz <46600784+stebenz@users.noreply.github.com>
This commit is contained in:
Lars
2025-01-29 15:23:56 +01:00
committed by GitHub
parent accfb7525a
commit df8bac8a28
23 changed files with 1746 additions and 126 deletions

View File

@@ -0,0 +1,239 @@
package resources
import (
"bytes"
"context"
"encoding/json"
"io"
"math"
"net/http"
"strconv"
"strings"
"github.com/zitadel/logging"
scim_config "github.com/zitadel/zitadel/internal/api/scim/config"
"github.com/zitadel/zitadel/internal/api/scim/metadata"
"github.com/zitadel/zitadel/internal/api/scim/schemas"
"github.com/zitadel/zitadel/internal/api/scim/serrors"
"github.com/zitadel/zitadel/internal/zerrors"
)
type BulkHandler struct {
cfg *scim_config.BulkConfig
handlersByPluralResourceName map[schemas.ScimResourceTypePlural]RawResourceHandlerAdapter
}
type BulkRequest struct {
Schemas []schemas.ScimSchemaType `json:"schemas"`
FailOnErrors *int `json:"failOnErrors"`
Operations []*BulkRequestOperation `json:"Operations"`
}
type BulkRequestOperation struct {
Method string `json:"method"`
BulkID string `json:"bulkId"`
Path string `json:"path"`
Data json.RawMessage `json:"data"`
}
type BulkResponse struct {
Schemas []schemas.ScimSchemaType `json:"schemas"`
Operations []*BulkResponseOperation `json:"Operations"`
}
type BulkResponseOperation struct {
Method string `json:"method"`
BulkID string `json:"bulkId,omitempty"`
Location string `json:"location,omitempty"`
Error *serrors.ScimError `json:"response,omitempty"`
Status string `json:"status"`
}
func (r *BulkRequest) GetSchemas() []schemas.ScimSchemaType {
return r.Schemas
}
func NewBulkHandler(
cfg scim_config.BulkConfig,
handlers ...RawResourceHandlerAdapter,
) *BulkHandler {
handlersByPluralResourceName := make(map[schemas.ScimResourceTypePlural]RawResourceHandlerAdapter, len(handlers))
for _, handler := range handlers {
handlersByPluralResourceName[handler.ResourceNamePlural()] = handler
}
return &BulkHandler{
&cfg,
handlersByPluralResourceName,
}
}
func (h *BulkHandler) BulkFromHttp(r *http.Request) (*BulkResponse, error) {
req, err := h.readBulkRequest(r)
if err != nil {
return nil, err
}
return h.processRequest(r.Context(), req)
}
func (h *BulkHandler) readBulkRequest(r *http.Request) (*BulkRequest, error) {
request := new(BulkRequest)
if err := readSchema(r.Body, request, schemas.IdBulkRequest); err != nil {
return nil, err
}
if len(request.Operations) > h.cfg.MaxOperationsCount {
return nil, serrors.ThrowPayloadTooLarge(zerrors.ThrowInvalidArgumentf(nil, "SCIM-BLK19", "Too many bulk operations in one request, max %d allowed.", h.cfg.MaxOperationsCount))
}
return request, nil
}
func (h *BulkHandler) processRequest(ctx context.Context, req *BulkRequest) (*BulkResponse, error) {
errorBudget := math.MaxInt32
if req.FailOnErrors != nil {
errorBudget = *req.FailOnErrors
}
resp := &BulkResponse{
Schemas: []schemas.ScimSchemaType{schemas.IdBulkResponse},
Operations: make([]*BulkResponseOperation, 0, len(req.Operations)),
}
for _, operation := range req.Operations {
opResp := h.processOperation(ctx, operation)
resp.Operations = append(resp.Operations, opResp)
if opResp.Error == nil {
continue
}
errorBudget--
if errorBudget <= 0 {
return resp, nil
}
}
return resp, nil
}
func (h *BulkHandler) processOperation(ctx context.Context, op *BulkRequestOperation) (opResp *BulkResponseOperation) {
var statusCode int
var resourceNamePlural schemas.ScimResourceTypePlural
var resourceID string
var err error
opResp = &BulkResponseOperation{
Method: op.Method,
BulkID: op.BulkID,
}
defer func() {
if r := recover(); r != nil {
logging.WithFields("panic", r).Error("Bulk operation panic")
err = zerrors.ThrowInternal(nil, "SCIM-BLK12", "Internal error while processing bulk operation")
}
if resourceNamePlural != "" && resourceID != "" {
opResp.Location = buildLocation(ctx, resourceNamePlural, resourceID)
}
opResp.Status = strconv.Itoa(statusCode)
if err != nil {
opResp.Error = serrors.MapToScimError(ctx, err)
opResp.Status = opResp.Error.Status
}
}()
resourceNamePlural, resourceID, err = h.parsePath(op.Path)
if err != nil {
return opResp
}
resourceID, err = metadata.ResolveScimBulkIDIfNeeded(ctx, resourceID)
if err != nil {
return opResp
}
resourceHandler, ok := h.handlersByPluralResourceName[resourceNamePlural]
if !ok {
err = zerrors.ThrowInvalidArgumentf(nil, "SCIM-BLK13", "Unknown resource %s", resourceNamePlural)
return opResp
}
switch op.Method {
case http.MethodPatch:
statusCode = http.StatusNoContent
err = h.processPatchOperation(ctx, resourceHandler, resourceID, op.Data)
case http.MethodPut:
statusCode = http.StatusOK
err = h.processPutOperation(ctx, resourceHandler, resourceID, op)
case http.MethodPost:
statusCode = http.StatusCreated
resourceID, err = h.processPostOperation(ctx, resourceHandler, resourceID, op)
case http.MethodDelete:
statusCode = http.StatusNoContent
err = h.processDeleteOperation(ctx, resourceHandler, resourceID)
default:
err = zerrors.ThrowInvalidArgumentf(nil, "SCIM-BLK14", "Unsupported operation %s", op.Method)
}
return opResp
}
func (h *BulkHandler) processPutOperation(ctx context.Context, resourceHandler RawResourceHandlerAdapter, resourceID string, op *BulkRequestOperation) error {
data := io.NopCloser(bytes.NewReader(op.Data))
_, err := resourceHandler.Replace(ctx, resourceID, data)
return err
}
func (h *BulkHandler) processPostOperation(ctx context.Context, resourceHandler RawResourceHandlerAdapter, resourceID string, op *BulkRequestOperation) (string, error) {
if resourceID != "" {
return "", zerrors.ThrowInvalidArgumentf(nil, "SCIM-BLK56", "Cannot post with a resourceID")
}
data := io.NopCloser(bytes.NewReader(op.Data))
createdResource, err := resourceHandler.Create(ctx, data)
if err != nil {
return "", err
}
id := createdResource.GetResource().ID
if op.BulkID != "" {
metadata.SetScimBulkIDMapping(ctx, op.BulkID, id)
}
return id, nil
}
func (h *BulkHandler) processPatchOperation(ctx context.Context, resourceHandler RawResourceHandlerAdapter, resourceID string, data json.RawMessage) error {
if resourceID == "" {
return zerrors.ThrowInvalidArgumentf(nil, "SCIM-BLK16", "To patch a resource, a resourceID is required")
}
return resourceHandler.Update(ctx, resourceID, io.NopCloser(bytes.NewReader(data)))
}
func (h *BulkHandler) processDeleteOperation(ctx context.Context, resourceHandler RawResourceHandlerAdapter, resourceID string) error {
if resourceID == "" {
return zerrors.ThrowInvalidArgumentf(nil, "SCIM-BLK17", "To delete a resource, a resourceID is required")
}
return resourceHandler.Delete(ctx, resourceID)
}
func (h *BulkHandler) parsePath(path string) (schemas.ScimResourceTypePlural, string, error) {
if !strings.HasPrefix(path, "/") {
return "", "", zerrors.ThrowInvalidArgumentf(nil, "SCIM-BLK19", "Invalid path: has to start with a /")
}
// part 0 is always an empty string due to the leading /
pathParts := strings.Split(path, "/")
switch len(pathParts) {
case 2:
return schemas.ScimResourceTypePlural(pathParts[1]), "", nil
case 3:
return schemas.ScimResourceTypePlural(pathParts[1]), pathParts[2], nil
default:
return "", "", zerrors.ThrowInvalidArgumentf(nil, "SCIM-BLK20", "Invalid resource path")
}
}

View File

@@ -3,7 +3,6 @@ package patch
import (
"encoding/json"
"reflect"
"slices"
"strings"
"github.com/zitadel/logging"
@@ -46,11 +45,11 @@ type ResourcePatcher interface {
Removed(attributePath []string) error
}
func (req *OperationRequest) Validate() error {
if !slices.Contains(req.Schemas, schemas.IdPatchOperation) {
return serrors.ThrowInvalidSyntax(zerrors.ThrowInvalidArgumentf(nil, "SCIM-xy1schema", "Expected schema %v is not provided", schemas.IdPatchOperation))
}
func (req *OperationRequest) GetSchemas() []schemas.ScimSchemaType {
return req.Schemas
}
func (req *OperationRequest) Validate() error {
for _, op := range req.Operations {
if err := op.validate(); err != nil {
return err

View File

@@ -14,9 +14,9 @@ import (
)
type ResourceHandler[T ResourceHolder] interface {
SchemaType() schemas.ScimSchemaType
ResourceNameSingular() schemas.ScimResourceTypeSingular
ResourceNamePlural() schemas.ScimResourceTypePlural
SchemaType() schemas.ScimSchemaType
NewResource() T
Create(ctx context.Context, resource T) (T, error)
@@ -28,6 +28,7 @@ type ResourceHandler[T ResourceHolder] interface {
}
type Resource struct {
ID string `json:"-"`
Schemas []schemas.ScimSchemaType `json:"schemas"`
Meta *ResourceMeta `json:"meta"`
}
@@ -41,9 +42,14 @@ type ResourceMeta struct {
}
type ResourceHolder interface {
SchemasHolder
GetResource() *Resource
}
type SchemasHolder interface {
GetSchemas() []schemas.ScimSchemaType
}
func buildResource[T ResourceHolder](ctx context.Context, handler ResourceHandler[T], details *domain.ObjectDetails) *Resource {
created := details.CreationDate.UTC()
if created.IsZero() {
@@ -51,17 +57,18 @@ func buildResource[T ResourceHolder](ctx context.Context, handler ResourceHandle
}
return &Resource{
ID: details.ID,
Schemas: []schemas.ScimSchemaType{handler.SchemaType()},
Meta: &ResourceMeta{
ResourceType: handler.ResourceNameSingular(),
Created: created,
LastModified: details.EventDate.UTC(),
Version: strconv.FormatUint(details.Sequence, 10),
Location: buildLocation(ctx, handler, details.ID),
Location: buildLocation(ctx, handler.ResourceNamePlural(), details.ID),
},
}
}
func buildLocation[T ResourceHolder](ctx context.Context, handler ResourceHandler[T], id string) string {
return http.DomainContext(ctx).Origin() + path.Join(schemas.HandlerPrefix, authz.GetCtxData(ctx).OrgID, string(handler.ResourceNamePlural()), id)
func buildLocation(ctx context.Context, resourceName schemas.ScimResourceTypePlural, id string) string {
return http.DomainContext(ctx).Origin() + path.Join(schemas.HandlerPrefix, authz.GetCtxData(ctx).OrgID, string(resourceName), id)
}

View File

@@ -1,17 +1,32 @@
package resources
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"slices"
"github.com/gorilla/mux"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/scim/resources/patch"
"github.com/zitadel/zitadel/internal/api/scim/schemas"
"github.com/zitadel/zitadel/internal/api/scim/serrors"
"github.com/zitadel/zitadel/internal/zerrors"
)
// RawResourceHandlerAdapter adapts the ResourceHandler[T] without any generics
type RawResourceHandlerAdapter interface {
ResourceNamePlural() schemas.ScimResourceTypePlural
Create(ctx context.Context, data io.ReadCloser) (ResourceHolder, error)
Replace(ctx context.Context, resourceID string, data io.ReadCloser) (ResourceHolder, error)
Update(ctx context.Context, resourceID string, data io.ReadCloser) error
Delete(ctx context.Context, resourceID string) error
}
type ResourceHandlerAdapter[T ResourceHolder] struct {
handler ResourceHandler[T]
}
@@ -22,38 +37,47 @@ func NewResourceHandlerAdapter[T ResourceHolder](handler ResourceHandler[T]) *Re
}
}
func (adapter *ResourceHandlerAdapter[T]) Create(r *http.Request) (T, error) {
entity, err := adapter.readEntityFromBody(r)
func (adapter *ResourceHandlerAdapter[T]) ResourceNamePlural() schemas.ScimResourceTypePlural {
return adapter.handler.ResourceNamePlural()
}
func (adapter *ResourceHandlerAdapter[T]) CreateFromHttp(r *http.Request) (ResourceHolder, error) {
return adapter.Create(r.Context(), r.Body)
}
func (adapter *ResourceHandlerAdapter[T]) Create(ctx context.Context, data io.ReadCloser) (ResourceHolder, error) {
entity, err := adapter.readEntity(data)
if err != nil {
return entity, err
}
return adapter.handler.Create(r.Context(), entity)
return adapter.handler.Create(ctx, entity)
}
func (adapter *ResourceHandlerAdapter[T]) Replace(r *http.Request) (T, error) {
entity, err := adapter.readEntityFromBody(r)
func (adapter *ResourceHandlerAdapter[T]) ReplaceFromHttp(r *http.Request) (ResourceHolder, error) {
return adapter.Replace(r.Context(), mux.Vars(r)["id"], r.Body)
}
func (adapter *ResourceHandlerAdapter[T]) Replace(ctx context.Context, resourceID string, data io.ReadCloser) (ResourceHolder, error) {
entity, err := adapter.readEntity(data)
if err != nil {
return entity, err
}
id := mux.Vars(r)["id"]
return adapter.handler.Replace(r.Context(), id, entity)
return adapter.handler.Replace(ctx, resourceID, entity)
}
func (adapter *ResourceHandlerAdapter[T]) Update(r *http.Request) error {
func (adapter *ResourceHandlerAdapter[T]) UpdateFromHttp(r *http.Request) error {
return adapter.Update(r.Context(), mux.Vars(r)["id"], r.Body)
}
func (adapter *ResourceHandlerAdapter[T]) Update(ctx context.Context, resourceID string, data io.ReadCloser) error {
request := new(patch.OperationRequest)
err := json.NewDecoder(r.Body).Decode(request)
if err != nil {
if zerrors.IsZitadelError(err) {
return err
}
return serrors.ThrowInvalidSyntax(zerrors.ThrowInvalidArgumentf(nil, "SCIM-ucrjson2", "Could not deserialize json: %v", err.Error()))
if err := readSchema(data, request, schemas.IdPatchOperation); err != nil {
return err
}
err = request.Validate()
if err != nil {
if err := request.Validate(); err != nil {
return err
}
@@ -61,17 +85,19 @@ func (adapter *ResourceHandlerAdapter[T]) Update(r *http.Request) error {
return nil
}
id := mux.Vars(r)["id"]
return adapter.handler.Update(r.Context(), id, request.Operations)
return adapter.handler.Update(ctx, resourceID, request.Operations)
}
func (adapter *ResourceHandlerAdapter[T]) Delete(r *http.Request) error {
id := mux.Vars(r)["id"]
return adapter.handler.Delete(r.Context(), id)
func (adapter *ResourceHandlerAdapter[T]) DeleteFromHttp(r *http.Request) error {
return adapter.Delete(r.Context(), mux.Vars(r)["id"])
}
func (adapter *ResourceHandlerAdapter[T]) List(r *http.Request) (*ListResponse[T], error) {
request, err := readListRequest(r)
func (adapter *ResourceHandlerAdapter[T]) Delete(ctx context.Context, resourceID string) error {
return adapter.handler.Delete(ctx, resourceID)
}
func (adapter *ResourceHandlerAdapter[T]) ListFromHttp(r *http.Request) (*ListResponse[T], error) {
request, err := adapter.readListRequest(r)
if err != nil {
return nil, err
}
@@ -79,30 +105,40 @@ func (adapter *ResourceHandlerAdapter[T]) List(r *http.Request) (*ListResponse[T
return adapter.handler.List(r.Context(), request)
}
func (adapter *ResourceHandlerAdapter[T]) Get(r *http.Request) (T, error) {
func (adapter *ResourceHandlerAdapter[T]) GetFromHttp(r *http.Request) (T, error) {
id := mux.Vars(r)["id"]
return adapter.handler.Get(r.Context(), id)
}
func (adapter *ResourceHandlerAdapter[T]) readEntityFromBody(r *http.Request) (T, error) {
func (adapter *ResourceHandlerAdapter[T]) readEntity(data io.ReadCloser) (T, error) {
entity := adapter.handler.NewResource()
err := json.NewDecoder(r.Body).Decode(entity)
return entity, readSchema(data, entity, adapter.handler.SchemaType())
}
func readSchema(data io.ReadCloser, entity SchemasHolder, schema schemas.ScimSchemaType) error {
defer func() {
err := data.Close()
logging.OnError(err).Warn("Failed to close http request body")
}()
err := json.NewDecoder(data).Decode(&entity)
if err != nil {
if serrors.IsScimOrZitadelError(err) {
return entity, err
var maxBytesErr *http.MaxBytesError
if errors.As(err, &maxBytesErr) {
return serrors.ThrowPayloadTooLarge(zerrors.ThrowInvalidArgumentf(err, "SCIM-hmaxb1", "Request payload too large, max %d bytes allowed.", maxBytesErr.Limit))
}
return entity, serrors.ThrowInvalidSyntax(zerrors.ThrowInvalidArgumentf(nil, "SCIM-ucrjson", "Could not deserialize json: %v", err.Error()))
if serrors.IsScimOrZitadelError(err) {
return err
}
return serrors.ThrowInvalidSyntax(zerrors.ThrowInvalidArgumentf(err, "SCIM-ucrjson", "Could not deserialize json"))
}
resource := entity.GetResource()
if resource == nil {
return entity, serrors.ThrowInvalidSyntax(zerrors.ThrowInvalidArgument(nil, "SCIM-xxrjson", "Could not get resource, is the schema correct?"))
providedSchemas := entity.GetSchemas()
if !slices.Contains(providedSchemas, schema) {
return serrors.ThrowInvalidSyntax(zerrors.ThrowInvalidArgumentf(nil, "SCIM-xxrschema", "Expected schema %v is not provided", schema))
}
if !slices.Contains(resource.Schemas, adapter.handler.SchemaType()) {
return entity, serrors.ThrowInvalidSyntax(zerrors.ThrowInvalidArgumentf(nil, "SCIM-xxrschema", "Expected schema %v is not provided", adapter.handler.SchemaType()))
}
return entity, nil
return nil
}

View File

@@ -1,7 +1,6 @@
package resources
import (
"encoding/json"
"net/http"
zhttp "github.com/zitadel/zitadel/internal/api/http"
@@ -13,6 +12,8 @@ import (
)
type ListRequest struct {
Schemas []schemas.ScimSchemaType `json:"schemas"`
// Count An integer indicating the desired maximum number of query results per page.
Count int64 `json:"count" schema:"count"`
@@ -47,6 +48,10 @@ const (
var parser = zhttp.NewParser()
func (r *ListRequest) GetSchemas() []schemas.ScimSchemaType {
return r.Schemas
}
func (o ListRequestSortOrder) isDefined() bool {
switch o {
case ListRequestSortOrderAsc, ListRequestSortOrderDsc:
@@ -70,7 +75,7 @@ func newListResponse[T ResourceHolder](totalResultCount uint64, q query.SearchRe
}
}
func readListRequest(r *http.Request) (*ListRequest, error) {
func (adapter *ResourceHandlerAdapter[T]) readListRequest(r *http.Request) (*ListRequest, error) {
request := &ListRequest{
Count: defaultListCount,
StartIndex: 1,
@@ -89,12 +94,8 @@ func readListRequest(r *http.Request) (*ListRequest, error) {
return nil, zerrors.ThrowInvalidArgument(nil, "SCIM-ullform", "Could not decode form: "+err.Error())
}
case http.MethodPost:
if err := json.NewDecoder(r.Body).Decode(request); err != nil {
if serrors.IsScimOrZitadelError(err) {
return nil, err
}
return nil, zerrors.ThrowInvalidArgument(nil, "SCIM-ulljson", "Could not decode json: "+err.Error())
if err := readSchema(r.Body, request, schemas.IdSearchRequest); err != nil {
return nil, err
}
// json deserialization initializes this field if an empty string is provided

View File

@@ -124,6 +124,13 @@ func (h *UsersHandler) ResourceNamePlural() scim_schemas.ScimResourceTypePlural
func (u *ScimUser) GetResource() *Resource {
return u.Resource
}
func (u *ScimUser) GetSchemas() []scim_schemas.ScimSchemaType {
if u.Resource == nil {
return nil
}
return u.Resource.Schemas
}
func (h *UsersHandler) NewResource() *ScimUser {
return new(ScimUser)

View File

@@ -384,13 +384,14 @@ func (h *UsersHandler) mapAndValidateMetadata(ctx context.Context, user *ScimUse
func (h *UsersHandler) buildResourceForQuery(ctx context.Context, user *query.User) *Resource {
return &Resource{
ID: user.ID,
Schemas: []schemas.ScimSchemaType{schemas.IdUser},
Meta: &ResourceMeta{
ResourceType: schemas.UserResourceType,
Created: user.CreationDate.UTC(),
LastModified: user.ChangeDate.UTC(),
Version: strconv.FormatUint(user.Sequence, 10),
Location: buildLocation(ctx, h, user.ID),
Location: buildLocation(ctx, h.ResourceNamePlural(), user.ID),
},
}
}
@@ -403,7 +404,7 @@ func (h *UsersHandler) buildResourceForWriteModel(ctx context.Context, user *com
Created: user.CreationDate.UTC(),
LastModified: user.ChangeDate.UTC(),
Version: strconv.FormatUint(user.ProcessedSequence, 10),
Location: buildLocation(ctx, h, user.AggregateID),
Location: buildLocation(ctx, h.ResourceNamePlural(), user.AggregateID),
},
}
}