diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 90dbb52ba7..5d49f92cf4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -7,9 +7,7 @@ "ghcr.io/devcontainers/features/go:1": { "version": "1.22" }, - "ghcr.io/devcontainers/features/node:1": { - "version": "18" - }, + "ghcr.io/devcontainers/features/node:1": {}, "ghcr.io/guiyomh/features/golangci-lint:0": {}, "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}, "ghcr.io/devcontainers/features/github-cli:1": {}, @@ -21,4 +19,4 @@ 8080 ], "onCreateCommand": "npm install -g sass@1.64.1" -} \ No newline at end of file +} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a8f673c2e8..ac7d909589 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,9 +50,19 @@ jobs: console_cache_path: ${{ needs.console.outputs.cache_path }} version: ${{ needs.version.outputs.version }} - core-test: + core-unit-test: needs: core - uses: ./.github/workflows/core-test.yml + uses: ./.github/workflows/core-unit-test.yml + with: + go_version: "1.22" + core_cache_key: ${{ needs.core.outputs.cache_key }} + core_cache_path: ${{ needs.core.outputs.cache_path }} + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + core-integration-test: + needs: core + uses: ./.github/workflows/core-integration-test.yml with: go_version: "1.22" core_cache_key: ${{ needs.core.outputs.cache_key }} @@ -93,7 +103,7 @@ jobs: issues: write pull-requests: write needs: - [version, core-test, lint, container, e2e] + [version, core-unit-test, core-integration-test, lint, container, e2e] if: ${{ github.event_name == 'workflow_dispatch' }} secrets: GCR_JSON_KEY_BASE64: ${{ secrets.GCR_JSON_KEY_BASE64 }} diff --git a/.github/workflows/core-test.yml b/.github/workflows/core-integration-test.yml similarity index 94% rename from .github/workflows/core-test.yml rename to .github/workflows/core-integration-test.yml index 4d8d978b60..2673d4addf 100644 --- a/.github/workflows/core-test.yml +++ b/.github/workflows/core-integration-test.yml @@ -18,11 +18,8 @@ on: jobs: postgres: - runs-on: ubuntu-latest - # TODO: use runner group as soon as integration tests run in parallel - # Currently it only consumes time and adds no value - # runs-on: - # group: zitadel-public + runs-on: + group: zitadel-public services: postgres: image: postgres @@ -77,6 +74,15 @@ jobs: ZITADEL_MASTERKEY: MasterkeyNeedsToHave32Characters INTEGRATION_DB_FLAVOR: postgres run: make core_integration_test + - + name: upload server logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: integration-test-server-logs + path: | + tmp/zitadel.log + tmp/race.log.* - name: publish coverage uses: codecov/codecov-action@v4.3.0 diff --git a/.github/workflows/core-unit-test.yml b/.github/workflows/core-unit-test.yml new file mode 100644 index 0000000000..0b1467ff5d --- /dev/null +++ b/.github/workflows/core-unit-test.yml @@ -0,0 +1,76 @@ +name: Unit test core + +on: + workflow_call: + inputs: + go_version: + required: true + type: string + core_cache_key: + required: true + type: string + core_cache_path: + required: true + type: string + crdb_version: + required: false + type: string + secrets: + CODECOV_TOKEN: + required: true + +jobs: + test: + runs-on: + group: zitadel-public + steps: + - + uses: actions/checkout@v3 + - + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go_version }} + - + uses: actions/cache/restore@v4 + timeout-minutes: 1 + name: restore core + id: restore-core + with: + path: ${{ inputs.core_cache_path }} + key: ${{ inputs.core_cache_key }} + fail-on-cache-miss: true + - + id: go-cache-path + name: set cache path + run: echo "GO_CACHE_PATH=$(go env GOCACHE)" >> $GITHUB_OUTPUT + - + uses: actions/cache/restore@v4 + id: cache + timeout-minutes: 1 + continue-on-error: true + name: restore previous results + with: + key: unit-test-${{ inputs.core_cache_key }} + restore-keys: | + unit-test-core- + path: ${{ steps.go-cache-path.outputs.GO_CACHE_PATH }} + - + name: test + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + run: make core_unit_test + - + name: publish coverage + uses: codecov/codecov-action@v4.3.0 + with: + file: profile.cov + name: core-unit-tests + flags: core-unit-tests + token: ${{ secrets.CODECOV_TOKEN }} + - + uses: actions/cache/save@v4 + name: cache results + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + with: + key: unit-test-${{ inputs.core_cache_key }} + path: ${{ steps.go-cache-path.outputs.GO_CACHE_PATH }} + \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4d58908786..e56ca307d1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -205,16 +205,50 @@ make core_unit_test #### Run Local Integration Tests -To test the database-connected gRPC API, run PostgreSQL and CockroachDB, set up a ZITADEL instance and run the tests including integration tests: +Integration tests are run as gRPC clients against a running ZITADEL server binary. +The server binary is typically [build with coverage enabled](https://go.dev/doc/build-cover). +It is also possible to run a ZITADEL sever in a debugger and run the integrations tests like that. In order to run the server, a database is required. + +The database flavor can **optionally** be set in the environment to `cockroach` or `postgres`. The default is `postgres`. ```bash -export INTEGRATION_DB_FLAVOR="cockroach" ZITADEL_MASTERKEY="MasterkeyNeedsToHave32Characters" -docker compose -f internal/integration/config/docker-compose.yaml up --pull always --wait ${INTEGRATION_DB_FLAVOR} -make core_integration_test -docker compose -f internal/integration/config/docker-compose.yaml down +export INTEGRATION_DB_FLAVOR="cockroach" ``` -Repeat the above with `INTEGRATION_DB_FLAVOR="postgres"`. +In order to prepare the local system, the following will bring up the database, builds a coverage binary, initializes the database and starts the sever. + +```bash +make core_integration_db_up core_integration_server_start +``` + +When this job is finished, you can run individual package integration test through your IDE or command-line. The actual integration test clients reside in the `integration_test` subdirectory of the package they aim to test. Integration test files use the `integration` build tag, in order to be excluded from regular unit tests. +Because of the server-client split, Go is usually unaware of changes in server code and tends to cache test results. Pas `-count 1` to disable test caching. + +Example command to run a single package integration test: + +```bash +go test -count 1 -tags integration ./internal/api/grpc/management/integration_test +``` + +To run all available integration tests: + +```bash +make core_integration_test_packages +``` + +When you change any ZITADEL server code, be sure to rebuild and restart the server before the next test run. + +```bash +make core_integration_server_stop core_integration_server_start +``` + +To cleanup after testing (deletes the database!): + +```bash +make core_integration_server_stop core_integration_db_down +``` + +The test binary has the race detector enabled. `core_core_integration_server_stop` checks for any race logs reported by Go and will print them along a `66` exit code when found. Note that the actual race condition may have happened anywhere during the server lifetime, including start, stop or serving gRPC requests during tests. #### Run Local End-to-End Tests diff --git a/Makefile b/Makefile index 8ef48623f6..17e1bbd9b7 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,12 @@ VERSION ?= development-$(now) COMMIT_SHA ?= $(shell git rev-parse HEAD) ZITADEL_IMAGE ?= zitadel:local +GOCOVERDIR = tmp/coverage +INTEGRATION_DB_FLAVOR ?= postgres +ZITADEL_MASTERKEY ?= MasterkeyNeedsToHave32Characters + +export GOCOVERDIR INTEGRATION_DB_FLAVOR ZITADEL_MASTERKEY + .PHONY: compile compile: core_build console_build compile_pipeline @@ -99,25 +105,54 @@ clean: $(RM) -r .artifacts/grpc $(RM) $(gen_authopt_path) $(RM) $(gen_zitadel_path) + $(RM) -r tmp/ .PHONY: core_unit_test core_unit_test: - go test -race -coverprofile=profile.cov ./... + go test -race -coverprofile=profile.cov -coverpkg=./internal/... ./... + +.PHONY: core_integration_db_up +core_integration_db_up: + docker compose -f internal/integration/config/docker-compose.yaml up --pull always --wait $${INTEGRATION_DB_FLAVOR} + +.PHONY: core_integration_db_down +core_integration_db_down: + docker compose -f internal/integration/config/docker-compose.yaml down .PHONY: core_integration_setup core_integration_setup: - go build -o zitadel main.go - ./zitadel init --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml - ./zitadel setup --masterkeyFromEnv --init-projections --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml --steps internal/integration/config/zitadel.yaml --steps internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml - $(RM) zitadel + go build -cover -race -tags integration -o zitadel.test main.go + mkdir -p $${GOCOVERDIR} + GORACE="halt_on_error=1" ./zitadel.test init --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml + GORACE="halt_on_error=1" ./zitadel.test setup --masterkeyFromEnv --init-projections --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml --steps internal/integration/config/steps.yaml + +.PHONY: core_integration_server_start +core_integration_server_start: core_integration_setup + GORACE="log_path=tmp/race.log" \ + ./zitadel.test start --masterkeyFromEnv --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml \ + > tmp/zitadel.log 2>&1 \ + & printf $$! > tmp/zitadel.pid + +.PHONY: core_integration_test_packages +core_integration_test_packages: + go test -count 1 -tags integration -timeout 30m $$(go list -tags integration ./... | grep "integration_test") + +.PHONY: core_integration_server_stop +core_integration_server_stop: + pid=$$(cat tmp/zitadel.pid); \ + $(RM) tmp/zitadel.pid; \ + kill $$pid; \ + if [ -s tmp/race.log.$$pid ]; then \ + cat tmp/race.log.$$pid; \ + exit 66; \ + fi + +.PHONY: core_integration_reports +core_integration_reports: + go tool covdata textfmt -i=tmp/coverage -pkg=github.com/zitadel/zitadel/internal/...,github.com/zitadel/zitadel/cmd/... -o profile.cov .PHONY: core_integration_test -core_integration_test: core_integration_setup - go test -tags=integration -race -p 1 -coverprofile=profile.cov -coverpkg=./internal/...,./cmd/... ./... - -.PHONY: core_integration_test_fast -core_integration_test_fast: core_integration_setup - go test -tags=integration -p 1 ./... +core_integration_test: core_integration_server_start core_integration_test_packages core_integration_server_stop core_integration_reports .PHONY: console_lint console_lint: diff --git a/README.md b/README.md index 1840218eb7..27c66dbf81 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,11 @@ See all guides [here](https://zitadel.com/docs/self-hosting/deploy/overview) ### Setup ZITADEL Cloud (SaaS) If you want to experience a hands-free ZITADEL, you should use [ZITADEL Cloud](https://zitadel.com). +Available data regions are: +* 🇺🇸 United States +* 🇪🇺 European Union +* 🇦🇺 Australia +* 🇨🇭 Switzerland ZITADEL Cloud comes with a free tier, providing you with all the same features as the open-source version. Learn more about the [pay-as-you-go pricing](https://zitadel.com/pricing). @@ -149,11 +154,12 @@ Deployment - [Zero Downtime Updates](https://zitadel.com/docs/concepts/architecture/solution#zero-downtime-updates) - [High scalability](https://zitadel.com/docs/self-hosting/manage/production) -Track upcoming features on our [roadmap](https://zitadel.com/roadmap). +Track upcoming features on our [roadmap](https://zitadel.com/roadmap) and follow our [changelog](https://zitadel.com/changelog) for recent updates. ## How To Contribute -Find details about how you can contribute in our [Contribution Guide](./CONTRIBUTING.md) +Find details about how you can contribute in our [Contribution Guide](./CONTRIBUTING.md). +Join our [Discord Chat](https://zitadel.com/chat) to get help. ## Contributors diff --git a/cmd/defaults.yaml b/cmd/defaults.yaml index 76f2fb7fbb..a81a1ff126 100644 --- a/cmd/defaults.yaml +++ b/cmd/defaults.yaml @@ -712,6 +712,13 @@ DefaultInstance: IncludeUpperLetters: false # ZITADEL_DEFAULTINSTANCE_SECRETGENERATORS_OTPEMAIL_INCLUDEUPPERLETTERS IncludeDigits: true # ZITADEL_DEFAULTINSTANCE_SECRETGENERATORS_OTPEMAIL_INCLUDEDIGITS IncludeSymbols: false # ZITADEL_DEFAULTINSTANCE_SECRETGENERATORS_OTPEMAIL_INCLUDESYMBOLS + InviteCode: + Length: 6 # ZITADEL_DEFAULTINSTANCE_SECRETGENERATORS_INITIALIZEUSERCODE_LENGTH + Expiry: "72h" # ZITADEL_DEFAULTINSTANCE_SECRETGENERATORS_INITIALIZEUSERCODE_EXPIRY + IncludeLowerLetters: false # ZITADEL_DEFAULTINSTANCE_SECRETGENERATORS_INITIALIZEUSERCODE_INCLUDELOWERLETTERS + IncludeUpperLetters: true # ZITADEL_DEFAULTINSTANCE_SECRETGENERATORS_INITIALIZEUSERCODE_INCLUDEUPPERLETTERS + IncludeDigits: true # ZITADEL_DEFAULTINSTANCE_SECRETGENERATORS_INITIALIZEUSERCODE_INCLUDEDIGITS + IncludeSymbols: false # ZITADEL_DEFAULTINSTANCE_SECRETGENERATORS_INITIALIZEUSERCODE_INCLUDESYMBOLS PasswordComplexityPolicy: MinLength: 8 # ZITADEL_DEFAULTINSTANCE_PASSWORDCOMPLEXITYPOLICY_MINLENGTH HasLowercase: true # ZITADEL_DEFAULTINSTANCE_PASSWORDCOMPLEXITYPOLICY_HASLOWERCASE @@ -1033,6 +1040,8 @@ InternalAuthZ: - "iam.web_key.write" - "iam.web_key.delete" - "iam.web_key.read" + - "iam.debug.write" + - "iam.debug.read" - "org.read" - "org.global.read" - "org.create" @@ -1110,6 +1119,7 @@ InternalAuthZ: - "iam.restrictions.read" - "iam.feature.read" - "iam.web_key.read" + - "iam.debug.read" - "org.read" - "org.member.read" - "org.idp.read" diff --git a/cmd/mirror/event.go b/cmd/mirror/event.go index 2bb0d52f45..af0ac25e27 100644 --- a/cmd/mirror/event.go +++ b/cmd/mirror/event.go @@ -3,6 +3,8 @@ package mirror import ( "context" + "github.com/shopspring/decimal" + "github.com/zitadel/zitadel/internal/v2/eventstore" "github.com/zitadel/zitadel/internal/v2/projection" "github.com/zitadel/zitadel/internal/v2/readmodel" @@ -30,12 +32,12 @@ func queryLastSuccessfulMigration(ctx context.Context, destinationES *eventstore return lastSuccess, nil } -func writeMigrationStart(ctx context.Context, sourceES *eventstore.EventStore, id string, destination string) (_ float64, err error) { +func writeMigrationStart(ctx context.Context, sourceES *eventstore.EventStore, id string, destination string) (_ decimal.Decimal, err error) { var cmd *eventstore.Command if len(instanceIDs) > 0 { cmd, err = mirror_event.NewStartedInstancesCommand(destination, instanceIDs) if err != nil { - return 0, err + return decimal.Decimal{}, err } } else { cmd = mirror_event.NewStartedSystemCommand(destination) @@ -58,12 +60,12 @@ func writeMigrationStart(ctx context.Context, sourceES *eventstore.EventStore, i ), ) if err != nil { - return 0, err + return decimal.Decimal{}, err } return position.Position, nil } -func writeMigrationSucceeded(ctx context.Context, destinationES *eventstore.EventStore, id, source string, position float64) error { +func writeMigrationSucceeded(ctx context.Context, destinationES *eventstore.EventStore, id, source string, position decimal.Decimal) error { return destinationES.Push( ctx, eventstore.NewPushIntent( diff --git a/cmd/mirror/event_store.go b/cmd/mirror/event_store.go index 23145bdc37..2eab4eb0da 100644 --- a/cmd/mirror/event_store.go +++ b/cmd/mirror/event_store.go @@ -9,6 +9,7 @@ import ( "time" "github.com/jackc/pgx/v5/stdlib" + "github.com/shopspring/decimal" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/zitadel/logging" @@ -180,7 +181,7 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) { logging.WithFields("took", time.Since(start), "count", eventCount).Info("events migrated") } -func writeCopyEventsDone(ctx context.Context, es *eventstore.EventStore, id, source string, position float64, errs <-chan error) { +func writeCopyEventsDone(ctx context.Context, es *eventstore.EventStore, id, source string, position decimal.Decimal, errs <-chan error) { joinedErrs := make([]error, 0, len(errs)) for err := range errs { joinedErrs = append(joinedErrs, err) diff --git a/cmd/start/start.go b/cmd/start/start.go index 47bc33ca42..5fc4ba936a 100644 --- a/cmd/start/start.go +++ b/cmd/start/start.go @@ -45,6 +45,7 @@ import ( org_v2 "github.com/zitadel/zitadel/internal/api/grpc/org/v2" org_v2beta "github.com/zitadel/zitadel/internal/api/grpc/org/v2beta" action_v3_alpha "github.com/zitadel/zitadel/internal/api/grpc/resources/action/v3alpha" + "github.com/zitadel/zitadel/internal/api/grpc/resources/debug_events/debug_events" user_v3_alpha "github.com/zitadel/zitadel/internal/api/grpc/resources/user/v3alpha" userschema_v3_alpha "github.com/zitadel/zitadel/internal/api/grpc/resources/userschema/v3alpha" "github.com/zitadel/zitadel/internal/api/grpc/resources/webkey/v3" @@ -79,6 +80,7 @@ import ( new_es "github.com/zitadel/zitadel/internal/eventstore/v3" "github.com/zitadel/zitadel/internal/i18n" "github.com/zitadel/zitadel/internal/id" + "github.com/zitadel/zitadel/internal/integration/sink" "github.com/zitadel/zitadel/internal/logstore" "github.com/zitadel/zitadel/internal/logstore/emitters/access" "github.com/zitadel/zitadel/internal/logstore/emitters/execution" @@ -138,6 +140,10 @@ type Server struct { func startZitadel(ctx context.Context, config *Config, masterKey string, server chan<- *Server) error { showBasicInformation(config) + // sink Server is stubbed out in production builds, see function's godoc. + closeSink := sink.StartServer() + defer closeSink() + i18n.MustLoadSupportedLanguagesFromDir() queryDBClient, err := database.Connect(config.Database, false, dialect.DBPurposeQuery) @@ -454,6 +460,9 @@ func startAPIs( if err := apis.RegisterService(ctx, webkey.CreateServer(commands, queries)); err != nil { return nil, err } + if err := apis.RegisterService(ctx, debug_events.CreateServer(commands, queries)); err != nil { + return nil, err + } instanceInterceptor := middleware.InstanceInterceptor(queries, config.ExternalDomain, login.IgnoreInstanceEndpoints...) assetsCache := middleware.AssetsCacheInterceptor(config.AssetStorage.Cache.MaxAge, config.AssetStorage.Cache.SharedMaxAge) apis.RegisterHandlerOnPrefix(assets.HandlerPrefix, assets.NewHandler(commands, verifier, config.InternalAuthZ, id.SonyFlakeGenerator(), store, queries, middleware.CallDurationHandler, instanceInterceptor.Handler, assetsCache.Handler, limitingAccessInterceptor.Handle)) diff --git a/console/src/app/modules/changes/changes.component.html b/console/src/app/modules/changes/changes.component.html index 1cf65b3db8..e78fb0fbb3 100644 --- a/console/src/app/modules/changes/changes.component.html +++ b/console/src/app/modules/changes/changes.component.html @@ -28,7 +28,7 @@ {{ action.localizedMessage }} {{ dayelement.dates[j] | timestampToDate | localizedDate: 'HH:mm' }} diff --git a/console/src/app/modules/header/header.component.html b/console/src/app/modules/header/header.component.html index 2a167731ec..e4ef76dd2a 100644 --- a/console/src/app/modules/header/header.component.html +++ b/console/src/app/modules/header/header.component.html @@ -172,11 +172,11 @@ {{ pP.customLinkText }} - - - {{ 'MENU.DOCUMENTATION' | translate }} - + + {{ 'MENU.DOCUMENTATION' | translate }} + +