1
0
mirror of https://github.com/zitadel/zitadel.git synced 2024-12-25 09:07:48 +00:00

542 lines
19 KiB
Go
Raw Normal View History

package start
import (
"context"
"crypto/tls"
_ "embed"
"fmt"
"math"
"net/http"
"os"
"os/signal"
"syscall"
"time"
clockpkg "github.com/benbjohnson/clock"
"github.com/common-nighthawk/go-figure"
"github.com/fatih/color"
"github.com/gorilla/mux"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/zitadel/logging"
2023-10-19 12:34:00 +02:00
"github.com/zitadel/oidc/v3/pkg/op"
"github.com/zitadel/saml/pkg/provider"
"golang.org/x/net/http2"
"golang.org/x/net/http2/h2c"
"github.com/zitadel/zitadel/cmd/build"
"github.com/zitadel/zitadel/cmd/key"
cmd_tls "github.com/zitadel/zitadel/cmd/tls"
"github.com/zitadel/zitadel/feature"
"github.com/zitadel/zitadel/internal/actions"
admin_es "github.com/zitadel/zitadel/internal/admin/repository/eventsourcing"
"github.com/zitadel/zitadel/internal/api"
"github.com/zitadel/zitadel/internal/api/assets"
internal_authz "github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/api/grpc/admin"
"github.com/zitadel/zitadel/internal/api/grpc/auth"
"github.com/zitadel/zitadel/internal/api/grpc/management"
oidc_v2 "github.com/zitadel/zitadel/internal/api/grpc/oidc/v2"
"github.com/zitadel/zitadel/internal/api/grpc/org/v2"
"github.com/zitadel/zitadel/internal/api/grpc/session/v2"
"github.com/zitadel/zitadel/internal/api/grpc/settings/v2"
"github.com/zitadel/zitadel/internal/api/grpc/system"
user_v2 "github.com/zitadel/zitadel/internal/api/grpc/user/v2"
http_util "github.com/zitadel/zitadel/internal/api/http"
"github.com/zitadel/zitadel/internal/api/http/middleware"
"github.com/zitadel/zitadel/internal/api/idp"
"github.com/zitadel/zitadel/internal/api/oidc"
"github.com/zitadel/zitadel/internal/api/robots_txt"
"github.com/zitadel/zitadel/internal/api/saml"
"github.com/zitadel/zitadel/internal/api/ui/console"
"github.com/zitadel/zitadel/internal/api/ui/login"
auth_es "github.com/zitadel/zitadel/internal/auth/repository/eventsourcing"
"github.com/zitadel/zitadel/internal/authz"
authz_repo "github.com/zitadel/zitadel/internal/authz/repository"
authz_es "github.com/zitadel/zitadel/internal/authz/repository/eventsourcing/eventstore"
"github.com/zitadel/zitadel/internal/command"
"github.com/zitadel/zitadel/internal/crypto"
cryptoDB "github.com/zitadel/zitadel/internal/crypto/database"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/domain"
"github.com/zitadel/zitadel/internal/eventstore"
old_es "github.com/zitadel/zitadel/internal/eventstore/repository/sql"
new_es "github.com/zitadel/zitadel/internal/eventstore/v3"
"github.com/zitadel/zitadel/internal/id"
"github.com/zitadel/zitadel/internal/logstore"
"github.com/zitadel/zitadel/internal/logstore/emitters/access"
"github.com/zitadel/zitadel/internal/logstore/emitters/execution"
"github.com/zitadel/zitadel/internal/logstore/emitters/stdout"
perf: project quotas and usages (#6441) * project quota added * project quota removed * add periods table * make log record generic * accumulate usage * query usage * count action run seconds * fix filter in ReportQuotaUsage * fix existing tests * fix logstore tests * fix typo * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * move notifications into debouncer and improve limit querying * cleanup * comment * fix: add quota unit tests command side * fix remaining quota usage query * implement InmemLogStorage * cleanup and linting * improve test * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * action notifications and fixes for notifications query * revert console prefix * fix: add quota unit tests command side * fix: add quota integration tests * improve accountable requests * improve accountable requests * fix: add quota integration tests * fix: add quota integration tests * fix: add quota integration tests * comment * remove ability to store logs in db and other changes requested from review * changes requested from review * changes requested from review * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * tests: fix quotas integration tests * improve incrementUsageStatement * linting * fix: delete e2e tests as intergation tests cover functionality * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * backup * fix conflict * create rc * create prerelease * remove issue release labeling * fix tracing --------- Co-authored-by: Livio Spring <livio.a@gmail.com> Co-authored-by: Stefan Benz <stefan@caos.ch> Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2023-09-15 16:58:45 +02:00
"github.com/zitadel/zitadel/internal/logstore/record"
"github.com/zitadel/zitadel/internal/net"
"github.com/zitadel/zitadel/internal/notification"
"github.com/zitadel/zitadel/internal/query"
"github.com/zitadel/zitadel/internal/static"
"github.com/zitadel/zitadel/internal/webauthn"
"github.com/zitadel/zitadel/openapi"
)
func New(server chan<- *Server) *cobra.Command {
start := &cobra.Command{
Use: "start",
Short: "starts ZITADEL instance",
Long: `starts ZITADEL.
Requirements:
- cockroachdb`,
RunE: func(cmd *cobra.Command, args []string) error {
err := cmd_tls.ModeFromFlag(cmd)
if err != nil {
return err
}
feat(cli): setup (#3267) * commander * commander * selber! * move to packages * fix(errors): implement Is interface * test: command * test: commands * add init steps * setup tenant * add default step yaml * possibility to set password * merge v2 into v2-commander * fix: rename iam command side to instance * fix: rename iam command side to instance * fix: rename iam command side to instance * fix: rename iam command side to instance * fix: search query builder can filter events in memory * fix: filters for add member * fix(setup): add `ExternalSecure` to config * chore: name iam to instance * fix: matching * remove unsued func * base url * base url * test(command): filter funcs * test: commands * fix: rename orgiampolicy to domain policy * start from init * commands * config * fix indexes and add constraints * fixes * fix: merge conflicts * fix: protos * fix: md files * setup * add deprecated org iam policy again * typo * fix search query * fix filter * Apply suggestions from code review * remove custom org from org setup * add todos for verification * change apps creation * simplify package structure * fix error * move preparation helper for tests * fix unique constraints * fix config mapping in setup * fix error handling in encryption_keys.go * fix projection config * fix query from old views to projection * fix setup of mgmt api * set iam project and fix instance projection * imports Co-authored-by: Livio Amstutz <livio.a@gmail.com> Co-authored-by: fabi <fabienne.gerschwiler@gmail.com>
2022-03-28 10:05:09 +02:00
config := MustNewConfig(viper.GetViper())
masterKey, err := key.MasterKey(cmd)
if err != nil {
return err
}
feat(cli): setup (#3267) * commander * commander * selber! * move to packages * fix(errors): implement Is interface * test: command * test: commands * add init steps * setup tenant * add default step yaml * possibility to set password * merge v2 into v2-commander * fix: rename iam command side to instance * fix: rename iam command side to instance * fix: rename iam command side to instance * fix: rename iam command side to instance * fix: search query builder can filter events in memory * fix: filters for add member * fix(setup): add `ExternalSecure` to config * chore: name iam to instance * fix: matching * remove unsued func * base url * base url * test(command): filter funcs * test: commands * fix: rename orgiampolicy to domain policy * start from init * commands * config * fix indexes and add constraints * fixes * fix: merge conflicts * fix: protos * fix: md files * setup * add deprecated org iam policy again * typo * fix search query * fix filter * Apply suggestions from code review * remove custom org from org setup * add todos for verification * change apps creation * simplify package structure * fix error * move preparation helper for tests * fix unique constraints * fix config mapping in setup * fix error handling in encryption_keys.go * fix projection config * fix query from old views to projection * fix setup of mgmt api * set iam project and fix instance projection * imports Co-authored-by: Livio Amstutz <livio.a@gmail.com> Co-authored-by: fabi <fabienne.gerschwiler@gmail.com>
2022-03-28 10:05:09 +02:00
return startZitadel(config, masterKey, server)
},
}
feat(cli): setup (#3267) * commander * commander * selber! * move to packages * fix(errors): implement Is interface * test: command * test: commands * add init steps * setup tenant * add default step yaml * possibility to set password * merge v2 into v2-commander * fix: rename iam command side to instance * fix: rename iam command side to instance * fix: rename iam command side to instance * fix: rename iam command side to instance * fix: search query builder can filter events in memory * fix: filters for add member * fix(setup): add `ExternalSecure` to config * chore: name iam to instance * fix: matching * remove unsued func * base url * base url * test(command): filter funcs * test: commands * fix: rename orgiampolicy to domain policy * start from init * commands * config * fix indexes and add constraints * fixes * fix: merge conflicts * fix: protos * fix: md files * setup * add deprecated org iam policy again * typo * fix search query * fix filter * Apply suggestions from code review * remove custom org from org setup * add todos for verification * change apps creation * simplify package structure * fix error * move preparation helper for tests * fix unique constraints * fix config mapping in setup * fix error handling in encryption_keys.go * fix projection config * fix query from old views to projection * fix setup of mgmt api * set iam project and fix instance projection * imports Co-authored-by: Livio Amstutz <livio.a@gmail.com> Co-authored-by: fabi <fabienne.gerschwiler@gmail.com>
2022-03-28 10:05:09 +02:00
startFlags(start)
return start
}
type Server struct {
Config *Config
DB *database.DB
KeyStorage crypto.KeyStorage
Keys *encryptionKeys
Eventstore *eventstore.Eventstore
Queries *query.Queries
AuthzRepo authz_repo.Repository
Storage static.Storage
Commands *command.Commands
Router *mux.Router
TLSConfig *tls.Config
Shutdown chan<- os.Signal
}
func startZitadel(config *Config, masterKey string, server chan<- *Server) error {
showBasicInformation(config)
ctx := context.Background()
zitadelDBClient, err := database.Connect(config.Database, false, false)
if err != nil {
return fmt.Errorf("cannot start client for projection: %w", err)
}
esPusherDBClient, err := database.Connect(config.Database, false, true)
if err != nil {
return fmt.Errorf("cannot start client for projection: %w", err)
}
keyStorage, err := cryptoDB.NewKeyStorage(zitadelDBClient, masterKey)
if err != nil {
return fmt.Errorf("cannot start key storage: %w", err)
}
keys, err := ensureEncryptionKeys(config.EncryptionKeys, keyStorage)
if err != nil {
return err
}
config.Eventstore.Pusher = new_es.NewEventstore(esPusherDBClient)
config.Eventstore.Querier = old_es.NewCRDB(zitadelDBClient)
eventstoreClient := eventstore.NewEventstore(config.Eventstore)
sessionTokenVerifier := internal_authz.SessionTokenVerifier(keys.OIDC)
queries, err := query.StartQueries(
ctx,
eventstoreClient,
zitadelDBClient,
config.Projections,
config.SystemDefaults,
keys.IDPConfig,
keys.OTP,
keys.OIDC,
keys.SAML,
config.InternalAuthZ.RolePermissionMappings,
sessionTokenVerifier,
func(q *query.Queries) domain.PermissionCheck {
return func(ctx context.Context, permission, orgID, resourceID string) (err error) {
return internal_authz.CheckPermission(ctx, &authz_es.UserMembershipRepo{Queries: q}, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
}
},
config.AuditLogRetention,
config.SystemAPIUsers,
)
if err != nil {
return fmt.Errorf("cannot start queries: %w", err)
}
authZRepo, err := authz.Start(queries, eventstoreClient, zitadelDBClient, keys.OIDC, config.ExternalSecure)
if err != nil {
return fmt.Errorf("error starting authz repo: %w", err)
}
permissionCheck := func(ctx context.Context, permission, orgID, resourceID string) (err error) {
return internal_authz.CheckPermission(ctx, authZRepo, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
}
storage, err := config.AssetStorage.NewStorage(zitadelDBClient.DB)
if err != nil {
return fmt.Errorf("cannot start asset storage client: %w", err)
}
webAuthNConfig := &webauthn.Config{
DisplayName: config.WebAuthNName,
ExternalSecure: config.ExternalSecure,
}
commands, err := command.StartCommands(
eventstoreClient,
config.SystemDefaults,
config.InternalAuthZ.RolePermissionMappings,
storage,
webAuthNConfig,
config.ExternalDomain,
config.ExternalSecure,
config.ExternalPort,
keys.IDPConfig,
keys.OTP,
keys.SMTP,
keys.SMS,
keys.User,
keys.DomainVerification,
keys.OIDC,
keys.SAML,
&http.Client{},
permissionCheck,
sessionTokenVerifier,
config.OIDC.DefaultAccessTokenLifetime,
config.OIDC.DefaultRefreshTokenExpiration,
config.OIDC.DefaultRefreshTokenIdleExpiration,
config.DefaultInstance.SecretGenerators,
)
if err != nil {
return fmt.Errorf("cannot start commands: %w", err)
}
clock := clockpkg.New()
perf: project quotas and usages (#6441) * project quota added * project quota removed * add periods table * make log record generic * accumulate usage * query usage * count action run seconds * fix filter in ReportQuotaUsage * fix existing tests * fix logstore tests * fix typo * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * move notifications into debouncer and improve limit querying * cleanup * comment * fix: add quota unit tests command side * fix remaining quota usage query * implement InmemLogStorage * cleanup and linting * improve test * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * action notifications and fixes for notifications query * revert console prefix * fix: add quota unit tests command side * fix: add quota integration tests * improve accountable requests * improve accountable requests * fix: add quota integration tests * fix: add quota integration tests * fix: add quota integration tests * comment * remove ability to store logs in db and other changes requested from review * changes requested from review * changes requested from review * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * tests: fix quotas integration tests * improve incrementUsageStatement * linting * fix: delete e2e tests as intergation tests cover functionality * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * backup * fix conflict * create rc * create prerelease * remove issue release labeling * fix tracing --------- Co-authored-by: Livio Spring <livio.a@gmail.com> Co-authored-by: Stefan Benz <stefan@caos.ch> Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2023-09-15 16:58:45 +02:00
actionsExecutionStdoutEmitter, err := logstore.NewEmitter[*record.ExecutionLog](ctx, clock, &logstore.EmitterConfig{Enabled: config.LogStore.Execution.Stdout.Enabled}, stdout.NewStdoutEmitter[*record.ExecutionLog]())
if err != nil {
return err
}
actionsExecutionDBEmitter, err := logstore.NewEmitter[*record.ExecutionLog](ctx, clock, config.Quotas.Execution, execution.NewDatabaseLogStorage(zitadelDBClient, commands, queries))
if err != nil {
return err
}
perf: project quotas and usages (#6441) * project quota added * project quota removed * add periods table * make log record generic * accumulate usage * query usage * count action run seconds * fix filter in ReportQuotaUsage * fix existing tests * fix logstore tests * fix typo * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * move notifications into debouncer and improve limit querying * cleanup * comment * fix: add quota unit tests command side * fix remaining quota usage query * implement InmemLogStorage * cleanup and linting * improve test * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * action notifications and fixes for notifications query * revert console prefix * fix: add quota unit tests command side * fix: add quota integration tests * improve accountable requests * improve accountable requests * fix: add quota integration tests * fix: add quota integration tests * fix: add quota integration tests * comment * remove ability to store logs in db and other changes requested from review * changes requested from review * changes requested from review * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * tests: fix quotas integration tests * improve incrementUsageStatement * linting * fix: delete e2e tests as intergation tests cover functionality * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * backup * fix conflict * create rc * create prerelease * remove issue release labeling * fix tracing --------- Co-authored-by: Livio Spring <livio.a@gmail.com> Co-authored-by: Stefan Benz <stefan@caos.ch> Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2023-09-15 16:58:45 +02:00
actionsLogstoreSvc := logstore.New(queries, actionsExecutionDBEmitter, actionsExecutionStdoutEmitter)
actions.SetLogstoreService(actionsLogstoreSvc)
notification.Start(
ctx,
config.Projections.Customizations["notifications"],
config.Projections.Customizations["notificationsquotas"],
config.Projections.Customizations["telemetry"],
*config.Telemetry,
config.ExternalDomain,
config.ExternalPort,
config.ExternalSecure,
commands,
queries,
eventstoreClient,
config.Login.DefaultOTPEmailURLV2,
config.SystemDefaults.Notifications.FileSystemPath,
keys.User,
keys.SMTP,
keys.SMS,
)
router := mux.NewRouter()
tlsConfig, err := config.TLS.Config()
if err != nil {
return err
}
err = startAPIs(
ctx,
clock,
router,
commands,
queries,
eventstoreClient,
zitadelDBClient,
config,
storage,
authZRepo,
keys,
permissionCheck,
)
if err != nil {
return err
}
shutdown := make(chan os.Signal, 1)
signal.Notify(shutdown, os.Interrupt, syscall.SIGTERM)
if server != nil {
server <- &Server{
Config: config,
DB: zitadelDBClient,
KeyStorage: keyStorage,
Keys: keys,
Eventstore: eventstoreClient,
Queries: queries,
AuthzRepo: authZRepo,
Storage: storage,
Commands: commands,
Router: router,
TLSConfig: tlsConfig,
Shutdown: shutdown,
}
close(server)
}
return listen(ctx, router, config.Port, tlsConfig, shutdown)
}
func startAPIs(
ctx context.Context,
clock clockpkg.Clock,
router *mux.Router,
commands *command.Commands,
queries *query.Queries,
eventstore *eventstore.Eventstore,
dbClient *database.DB,
config *Config,
store static.Storage,
authZRepo authz_repo.Repository,
keys *encryptionKeys,
permissionCheck domain.PermissionCheck,
) error {
repo := struct {
authz_repo.Repository
*query.Queries
}{
authZRepo,
queries,
}
oidcPrefixes := []string{"/.well-known/openid-configuration", "/oidc/v1", "/oauth/v2"}
// always set the origin in the context if available in the http headers, no matter for what protocol
router.Use(middleware.OriginHandler)
// adds used HTTPPathPattern and RequestMethod to context
router.Use(middleware.ActivityHandler)
systemTokenVerifier, err := internal_authz.StartSystemTokenVerifierFromConfig(http_util.BuildHTTP(config.ExternalDomain, config.ExternalPort, config.ExternalSecure), config.SystemAPIUsers)
if err != nil {
return err
}
accessTokenVerifer := internal_authz.StartAccessTokenVerifierFromRepo(repo)
verifier := internal_authz.StartAPITokenVerifier(repo, accessTokenVerifer, systemTokenVerifier)
tlsConfig, err := config.TLS.Config()
if err != nil {
return err
}
perf: project quotas and usages (#6441) * project quota added * project quota removed * add periods table * make log record generic * accumulate usage * query usage * count action run seconds * fix filter in ReportQuotaUsage * fix existing tests * fix logstore tests * fix typo * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * move notifications into debouncer and improve limit querying * cleanup * comment * fix: add quota unit tests command side * fix remaining quota usage query * implement InmemLogStorage * cleanup and linting * improve test * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * action notifications and fixes for notifications query * revert console prefix * fix: add quota unit tests command side * fix: add quota integration tests * improve accountable requests * improve accountable requests * fix: add quota integration tests * fix: add quota integration tests * fix: add quota integration tests * comment * remove ability to store logs in db and other changes requested from review * changes requested from review * changes requested from review * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * tests: fix quotas integration tests * improve incrementUsageStatement * linting * fix: delete e2e tests as intergation tests cover functionality * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * backup * fix conflict * create rc * create prerelease * remove issue release labeling * fix tracing --------- Co-authored-by: Livio Spring <livio.a@gmail.com> Co-authored-by: Stefan Benz <stefan@caos.ch> Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2023-09-15 16:58:45 +02:00
accessStdoutEmitter, err := logstore.NewEmitter[*record.AccessLog](ctx, clock, &logstore.EmitterConfig{Enabled: config.LogStore.Access.Stdout.Enabled}, stdout.NewStdoutEmitter[*record.AccessLog]())
if err != nil {
return err
}
perf: project quotas and usages (#6441) * project quota added * project quota removed * add periods table * make log record generic * accumulate usage * query usage * count action run seconds * fix filter in ReportQuotaUsage * fix existing tests * fix logstore tests * fix typo * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * move notifications into debouncer and improve limit querying * cleanup * comment * fix: add quota unit tests command side * fix remaining quota usage query * implement InmemLogStorage * cleanup and linting * improve test * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * action notifications and fixes for notifications query * revert console prefix * fix: add quota unit tests command side * fix: add quota integration tests * improve accountable requests * improve accountable requests * fix: add quota integration tests * fix: add quota integration tests * fix: add quota integration tests * comment * remove ability to store logs in db and other changes requested from review * changes requested from review * changes requested from review * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * tests: fix quotas integration tests * improve incrementUsageStatement * linting * fix: delete e2e tests as intergation tests cover functionality * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * backup * fix conflict * create rc * create prerelease * remove issue release labeling * fix tracing --------- Co-authored-by: Livio Spring <livio.a@gmail.com> Co-authored-by: Stefan Benz <stefan@caos.ch> Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2023-09-15 16:58:45 +02:00
accessDBEmitter, err := logstore.NewEmitter[*record.AccessLog](ctx, clock, &config.Quotas.Access.EmitterConfig, access.NewDatabaseLogStorage(dbClient, commands, queries))
if err != nil {
return err
}
perf: project quotas and usages (#6441) * project quota added * project quota removed * add periods table * make log record generic * accumulate usage * query usage * count action run seconds * fix filter in ReportQuotaUsage * fix existing tests * fix logstore tests * fix typo * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * move notifications into debouncer and improve limit querying * cleanup * comment * fix: add quota unit tests command side * fix remaining quota usage query * implement InmemLogStorage * cleanup and linting * improve test * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * action notifications and fixes for notifications query * revert console prefix * fix: add quota unit tests command side * fix: add quota integration tests * improve accountable requests * improve accountable requests * fix: add quota integration tests * fix: add quota integration tests * fix: add quota integration tests * comment * remove ability to store logs in db and other changes requested from review * changes requested from review * changes requested from review * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * tests: fix quotas integration tests * improve incrementUsageStatement * linting * fix: delete e2e tests as intergation tests cover functionality * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * backup * fix conflict * create rc * create prerelease * remove issue release labeling * fix tracing --------- Co-authored-by: Livio Spring <livio.a@gmail.com> Co-authored-by: Stefan Benz <stefan@caos.ch> Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2023-09-15 16:58:45 +02:00
accessSvc := logstore.New[*record.AccessLog](queries, accessDBEmitter, accessStdoutEmitter)
exhaustedCookieHandler := http_util.NewCookieHandler(
http_util.WithUnsecure(),
http_util.WithNonHttpOnly(),
http_util.WithMaxAge(int(math.Floor(config.Quotas.Access.ExhaustedCookieMaxAge.Seconds()))),
)
perf: project quotas and usages (#6441) * project quota added * project quota removed * add periods table * make log record generic * accumulate usage * query usage * count action run seconds * fix filter in ReportQuotaUsage * fix existing tests * fix logstore tests * fix typo * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * move notifications into debouncer and improve limit querying * cleanup * comment * fix: add quota unit tests command side * fix remaining quota usage query * implement InmemLogStorage * cleanup and linting * improve test * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * action notifications and fixes for notifications query * revert console prefix * fix: add quota unit tests command side * fix: add quota integration tests * improve accountable requests * improve accountable requests * fix: add quota integration tests * fix: add quota integration tests * fix: add quota integration tests * comment * remove ability to store logs in db and other changes requested from review * changes requested from review * changes requested from review * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * tests: fix quotas integration tests * improve incrementUsageStatement * linting * fix: delete e2e tests as intergation tests cover functionality * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * backup * fix conflict * create rc * create prerelease * remove issue release labeling * fix tracing --------- Co-authored-by: Livio Spring <livio.a@gmail.com> Co-authored-by: Stefan Benz <stefan@caos.ch> Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2023-09-15 16:58:45 +02:00
limitingAccessInterceptor := middleware.NewAccessInterceptor(accessSvc, exhaustedCookieHandler, &config.Quotas.Access.AccessConfig)
apis, err := api.New(ctx, config.Port, router, queries, verifier, config.InternalAuthZ, tlsConfig, config.HTTP2HostHeader, config.HTTP1HostHeader, limitingAccessInterceptor)
if err != nil {
return fmt.Errorf("error creating api %w", err)
}
config.Auth.Spooler.Client = dbClient
config.Auth.Spooler.Eventstore = eventstore
authRepo, err := auth_es.Start(ctx, config.Auth, config.SystemDefaults, commands, queries, dbClient, eventstore, keys.OIDC, keys.User)
if err != nil {
return fmt.Errorf("error starting auth repo: %w", err)
}
config.Admin.Spooler.Client = dbClient
config.Admin.Spooler.Eventstore = eventstore
err = admin_es.Start(ctx, config.Admin, store, dbClient)
if err != nil {
return fmt.Errorf("error starting admin repo: %w", err)
}
if err := apis.RegisterServer(ctx, system.CreateServer(commands, queries, config.Database.DatabaseName(), config.DefaultInstance, config.ExternalDomain), tlsConfig); err != nil {
return err
}
if err := apis.RegisterServer(ctx, admin.CreateServer(config.Database.DatabaseName(), commands, queries, config.SystemDefaults, config.ExternalSecure, keys.User, config.AuditLogRetention), tlsConfig); err != nil {
return err
}
if err := apis.RegisterServer(ctx, management.CreateServer(commands, queries, config.SystemDefaults, keys.User, config.ExternalSecure), tlsConfig); err != nil {
return err
}
if err := apis.RegisterServer(ctx, auth.CreateServer(commands, queries, authRepo, config.SystemDefaults, keys.User, config.ExternalSecure), tlsConfig); err != nil {
return err
}
if err := apis.RegisterService(ctx, user_v2.CreateServer(commands, queries, keys.User, keys.IDPConfig, idp.CallbackURL(config.ExternalSecure), idp.SAMLRootURL(config.ExternalSecure))); err != nil {
return err
}
if err := apis.RegisterService(ctx, session.CreateServer(commands, queries, permissionCheck)); err != nil {
return err
}
if err := apis.RegisterService(ctx, settings.CreateServer(commands, queries, config.ExternalSecure)); err != nil {
return err
}
if err := apis.RegisterService(ctx, org.CreateServer(commands, queries, permissionCheck)); err != nil {
return err
}
instanceInterceptor := middleware.InstanceInterceptor(queries, config.HTTP1HostHeader, login.IgnoreInstanceEndpoints...)
assetsCache := middleware.AssetsCacheInterceptor(config.AssetStorage.Cache.MaxAge, config.AssetStorage.Cache.SharedMaxAge)
apis.RegisterHandlerOnPrefix(assets.HandlerPrefix, assets.NewHandler(commands, verifier, config.InternalAuthZ, id.SonyFlakeGenerator(), store, queries, middleware.CallDurationHandler, instanceInterceptor.Handler, assetsCache.Handler, limitingAccessInterceptor.Handle))
apis.RegisterHandlerOnPrefix(idp.HandlerPrefix, idp.NewHandler(commands, queries, keys.IDPConfig, config.ExternalSecure, instanceInterceptor.Handler))
userAgentInterceptor, err := middleware.NewUserAgentHandler(config.UserAgentCookie, keys.UserAgentCookieKey, id.SonyFlakeGenerator(), config.ExternalSecure, login.EndpointResources, login.EndpointExternalLoginCallbackFormPost, login.EndpointSAMLACS)
if err != nil {
return err
}
// robots.txt handler
robotsTxtHandler, err := robots_txt.Start()
if err != nil {
return fmt.Errorf("unable to start robots txt handler: %w", err)
}
apis.RegisterHandlerOnPrefix(robots_txt.HandlerPrefix, robotsTxtHandler)
// TODO: Record openapi access logs?
openAPIHandler, err := openapi.Start()
if err != nil {
return fmt.Errorf("unable to start openapi handler: %w", err)
}
apis.RegisterHandlerOnPrefix(openapi.HandlerPrefix, openAPIHandler)
oidcServer, err := oidc.NewServer(config.OIDC, login.DefaultLoggedOutPath, config.ExternalSecure, commands, queries, authRepo, keys.OIDC, keys.OIDCKey, eventstore, dbClient, userAgentInterceptor, instanceInterceptor.Handler, limitingAccessInterceptor, config.Log.Slog())
if err != nil {
return fmt.Errorf("unable to start oidc provider: %w", err)
}
apis.RegisterHandlerPrefixes(oidcServer, oidcPrefixes...)
perf: project quotas and usages (#6441) * project quota added * project quota removed * add periods table * make log record generic * accumulate usage * query usage * count action run seconds * fix filter in ReportQuotaUsage * fix existing tests * fix logstore tests * fix typo * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * move notifications into debouncer and improve limit querying * cleanup * comment * fix: add quota unit tests command side * fix remaining quota usage query * implement InmemLogStorage * cleanup and linting * improve test * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * action notifications and fixes for notifications query * revert console prefix * fix: add quota unit tests command side * fix: add quota integration tests * improve accountable requests * improve accountable requests * fix: add quota integration tests * fix: add quota integration tests * fix: add quota integration tests * comment * remove ability to store logs in db and other changes requested from review * changes requested from review * changes requested from review * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * tests: fix quotas integration tests * improve incrementUsageStatement * linting * fix: delete e2e tests as intergation tests cover functionality * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * backup * fix conflict * create rc * create prerelease * remove issue release labeling * fix tracing --------- Co-authored-by: Livio Spring <livio.a@gmail.com> Co-authored-by: Stefan Benz <stefan@caos.ch> Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2023-09-15 16:58:45 +02:00
samlProvider, err := saml.NewProvider(config.SAML, config.ExternalSecure, commands, queries, authRepo, keys.OIDC, keys.SAML, eventstore, dbClient, instanceInterceptor.Handler, userAgentInterceptor, limitingAccessInterceptor)
if err != nil {
return fmt.Errorf("unable to start saml provider: %w", err)
}
apis.RegisterHandlerOnPrefix(saml.HandlerPrefix, samlProvider.HttpHandler())
c, err := console.Start(config.Console, config.ExternalSecure, oidcServer.IssuerFromRequest, middleware.CallDurationHandler, instanceInterceptor.Handler, limitingAccessInterceptor, config.CustomerPortal)
if err != nil {
return fmt.Errorf("unable to start console: %w", err)
}
apis.RegisterHandlerOnPrefix(console.HandlerPrefix, c)
l, err := login.CreateLogin(
config.Login,
commands,
queries,
authRepo,
store,
console.HandlerPrefix+"/",
oidcServer.AuthCallbackURL(),
provider.AuthCallbackURL(samlProvider),
config.ExternalSecure,
userAgentInterceptor,
op.NewIssuerInterceptor(oidcServer.IssuerFromRequest).Handler,
provider.NewIssuerInterceptor(samlProvider.IssuerFromRequest).Handler,
instanceInterceptor.Handler,
assetsCache.Handler,
limitingAccessInterceptor.WithoutLimiting().Handle,
keys.User,
keys.IDPConfig,
keys.CSRFCookieKey,
feature.NewCheck(eventstore),
)
if err != nil {
return fmt.Errorf("unable to start login: %w", err)
}
apis.RegisterHandlerOnPrefix(login.HandlerPrefix, l.Handler())
apis.HandleFunc(login.EndpointDeviceAuth, login.RedirectDeviceAuthToPrefix)
// After OIDC provider so that the callback endpoint can be used
if err := apis.RegisterService(ctx, oidc_v2.CreateServer(commands, queries, oidcServer, config.ExternalSecure)); err != nil {
return err
}
// handle grpc at last to be able to handle the root, because grpc and gateway require a lot of different prefixes
apis.RouteGRPC()
return nil
}
func listen(ctx context.Context, router *mux.Router, port uint16, tlsConfig *tls.Config, shutdown <-chan os.Signal) error {
http2Server := &http2.Server{}
http1Server := &http.Server{Handler: h2c.NewHandler(router, http2Server), TLSConfig: tlsConfig}
2023-05-02 19:24:24 +03:00
perf: project quotas and usages (#6441) * project quota added * project quota removed * add periods table * make log record generic * accumulate usage * query usage * count action run seconds * fix filter in ReportQuotaUsage * fix existing tests * fix logstore tests * fix typo * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * move notifications into debouncer and improve limit querying * cleanup * comment * fix: add quota unit tests command side * fix remaining quota usage query * implement InmemLogStorage * cleanup and linting * improve test * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * fix: add quota unit tests command side * action notifications and fixes for notifications query * revert console prefix * fix: add quota unit tests command side * fix: add quota integration tests * improve accountable requests * improve accountable requests * fix: add quota integration tests * fix: add quota integration tests * fix: add quota integration tests * comment * remove ability to store logs in db and other changes requested from review * changes requested from review * changes requested from review * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * tests: fix quotas integration tests * improve incrementUsageStatement * linting * fix: delete e2e tests as intergation tests cover functionality * Update internal/api/http/middleware/access_interceptor.go Co-authored-by: Silvan <silvan.reusser@gmail.com> * backup * fix conflict * create rc * create prerelease * remove issue release labeling * fix tracing --------- Co-authored-by: Livio Spring <livio.a@gmail.com> Co-authored-by: Stefan Benz <stefan@caos.ch> Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
2023-09-15 16:58:45 +02:00
lc := net.ListenConfig()
2023-05-02 19:24:24 +03:00
lis, err := lc.Listen(ctx, "tcp", fmt.Sprintf(":%d", port))
if err != nil {
return fmt.Errorf("tcp listener on %d failed: %w", port, err)
}
errCh := make(chan error)
go func() {
logging.Infof("server is listening on %s", lis.Addr().String())
if tlsConfig != nil {
//we don't need to pass the files here, because we already initialized the TLS config on the server
errCh <- http1Server.ServeTLS(lis, "", "")
} else {
errCh <- http1Server.Serve(lis)
}
}()
select {
case err := <-errCh:
return fmt.Errorf("error starting server: %w", err)
case <-shutdown:
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
return shutdownServer(ctx, http1Server)
case <-ctx.Done():
return shutdownServer(ctx, http1Server)
}
}
func shutdownServer(ctx context.Context, server *http.Server) error {
err := server.Shutdown(ctx)
if err != nil {
return fmt.Errorf("could not shutdown gracefully: %w", err)
}
logging.New().Info("server shutdown gracefully")
return nil
}
func showBasicInformation(startConfig *Config) {
fmt.Println(color.MagentaString(figure.NewFigure("ZITADEL", "", true).String()))
http := "http"
if startConfig.TLS.Enabled || startConfig.ExternalSecure {
http = "https"
}
consoleURL := fmt.Sprintf("%s://%s:%v/ui/console\n", http, startConfig.ExternalDomain, startConfig.ExternalPort)
healthCheckURL := fmt.Sprintf("%s://%s:%v/debug/healthz\n", http, startConfig.ExternalDomain, startConfig.ExternalPort)
insecure := !startConfig.TLS.Enabled && !startConfig.ExternalSecure
fmt.Printf(" ===============================================================\n\n")
fmt.Printf(" Version : %s\n", build.Version())
fmt.Printf(" TLS enabled : %v\n", startConfig.TLS.Enabled)
fmt.Printf(" External Secure : %v\n", startConfig.ExternalSecure)
fmt.Printf(" Console URL : %s", color.BlueString(consoleURL))
fmt.Printf(" Health Check URL : %s", color.BlueString(healthCheckURL))
if insecure {
fmt.Printf("\n %s: you're using plain http without TLS. Be aware this is \n", color.RedString("Warning"))
fmt.Printf(" not a secure setup and should only be used for test systems. \n")
fmt.Printf(" Visit: %s \n", color.CyanString("https://zitadel.com/docs/self-hosting/manage/tls_modes"))
}
fmt.Printf("\n ===============================================================\n\n")
}