From e167365548529fd1f5bdea7a269339966aa88975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Mon, 24 Apr 2023 20:40:31 +0300 Subject: [PATCH 01/16] chore: test server for direct resource access --- cmd/admin/admin.go | 4 +-- cmd/start/start.go | 55 ++++++++++++++++++++++++++++++----- cmd/start/start_from_init.go | 4 +-- cmd/start/start_from_setup.go | 4 +-- cmd/zitadel.go | 37 ++++++++++++++++++++--- cmd/zitadel_test.go | 13 +++++++++ main.go | 2 +- 7 files changed, 100 insertions(+), 19 deletions(-) create mode 100644 cmd/zitadel_test.go diff --git a/cmd/admin/admin.go b/cmd/admin/admin.go index 785ed1abf0..04741a4a88 100644 --- a/cmd/admin/admin.go +++ b/cmd/admin/admin.go @@ -26,8 +26,8 @@ func New() *cobra.Command { adminCMD.AddCommand( initialise.New(), setup.New(), - start.New(), - start.NewStartFromInit(), + start.New(nil), + start.NewStartFromInit(nil), key.New(), ) diff --git a/cmd/start/start.go b/cmd/start/start.go index c3ba766fea..361619536b 100644 --- a/cmd/start/start.go +++ b/cmd/start/start.go @@ -45,6 +45,7 @@ import ( "github.com/zitadel/zitadel/internal/authz" authz_repo "github.com/zitadel/zitadel/internal/authz/repository" "github.com/zitadel/zitadel/internal/command" + "github.com/zitadel/zitadel/internal/crypto" cryptoDB "github.com/zitadel/zitadel/internal/crypto/database" "github.com/zitadel/zitadel/internal/database" "github.com/zitadel/zitadel/internal/eventstore" @@ -60,7 +61,7 @@ import ( "github.com/zitadel/zitadel/openapi" ) -func New() *cobra.Command { +func New(server chan<- *Server) *cobra.Command { start := &cobra.Command{ Use: "start", Short: "starts ZITADEL instance", @@ -78,7 +79,7 @@ Requirements: return err } - return startZitadel(config, masterKey) + return startZitadel(config, masterKey, server) }, } @@ -87,7 +88,24 @@ Requirements: return start } -func startZitadel(config *Config, masterKey string) error { +type Server struct { + background context.Context + Config *Config + DB *database.DB + KeyStorage crypto.KeyStorage + Keys *encryptionKeys + Eventstore *eventstore.Eventstore + Queries *query.Queries + AuthzRepo authz_repo.Repository + Storage static.Storage + Commands *command.Commands + LogStore *logstore.Service + Router *mux.Router + TLSConfig *tls.Config + Shutdown chan<- os.Signal +} + +func startZitadel(config *Config, masterKey string, server chan<- *Server) error { ctx := context.Background() dbClient, err := database.Connect(config.Database, false) @@ -179,7 +197,31 @@ func startZitadel(config *Config, masterKey string) error { if err != nil { return err } - return listen(ctx, router, config.Port, tlsConfig) + + shutdown := make(chan os.Signal, 1) + signal.Notify(shutdown, os.Interrupt, syscall.SIGTERM) + + if server != nil { + server <- &Server{ + background: ctx, + Config: config, + DB: dbClient, + KeyStorage: keyStorage, + Keys: keys, + Eventstore: eventstoreClient, + Queries: queries, + AuthzRepo: authZRepo, + Storage: storage, + Commands: commands, + LogStore: actionsLogstoreSvc, + Router: router, + TLSConfig: tlsConfig, + Shutdown: shutdown, + } + close(server) + } + + return listen(ctx, router, config.Port, tlsConfig, shutdown) } func startAPIs( @@ -300,7 +342,7 @@ func startAPIs( return nil } -func listen(ctx context.Context, router *mux.Router, port uint16, tlsConfig *tls.Config) error { +func listen(ctx context.Context, router *mux.Router, port uint16, tlsConfig *tls.Config, shutdown <-chan os.Signal) error { http2Server := &http2.Server{} http1Server := &http.Server{Handler: h2c.NewHandler(router, http2Server), TLSConfig: tlsConfig} lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) @@ -320,9 +362,6 @@ func listen(ctx context.Context, router *mux.Router, port uint16, tlsConfig *tls } }() - shutdown := make(chan os.Signal, 1) - signal.Notify(shutdown, os.Interrupt, syscall.SIGTERM) - select { case err := <-errCh: return fmt.Errorf("error starting server: %w", err) diff --git a/cmd/start/start_from_init.go b/cmd/start/start_from_init.go index 89c74d5592..940efb4e84 100644 --- a/cmd/start/start_from_init.go +++ b/cmd/start/start_from_init.go @@ -11,7 +11,7 @@ import ( "github.com/zitadel/zitadel/cmd/tls" ) -func NewStartFromInit() *cobra.Command { +func NewStartFromInit(server chan<- *Server) *cobra.Command { cmd := &cobra.Command{ Use: "start-from-init", Short: "cold starts zitadel", @@ -37,7 +37,7 @@ Requirements: startConfig := MustNewConfig(viper.GetViper()) - err = startZitadel(startConfig, masterKey) + err = startZitadel(startConfig, masterKey, server) logging.OnError(err).Fatal("unable to start zitadel") }, } diff --git a/cmd/start/start_from_setup.go b/cmd/start/start_from_setup.go index 75c2347c2b..0be315fae9 100644 --- a/cmd/start/start_from_setup.go +++ b/cmd/start/start_from_setup.go @@ -10,7 +10,7 @@ import ( "github.com/zitadel/zitadel/cmd/tls" ) -func NewStartFromSetup() *cobra.Command { +func NewStartFromSetup(server chan<- *Server) *cobra.Command { cmd := &cobra.Command{ Use: "start-from-setup", Short: "cold starts zitadel", @@ -35,7 +35,7 @@ Requirements: startConfig := MustNewConfig(viper.GetViper()) - err = startZitadel(startConfig, masterKey) + err = startZitadel(startConfig, masterKey, server) logging.OnError(err).Fatal("unable to start zitadel") }, } diff --git a/cmd/zitadel.go b/cmd/zitadel.go index 9822e1da05..8de2aaf5b4 100644 --- a/cmd/zitadel.go +++ b/cmd/zitadel.go @@ -5,7 +5,9 @@ import ( _ "embed" "errors" "io" + "os" "strings" + "sync" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -26,7 +28,7 @@ var ( defaultConfig []byte ) -func New(out io.Writer, in io.Reader, args []string) *cobra.Command { +func New(out io.Writer, in io.Reader, args []string, server chan<- *start.Server) *cobra.Command { cmd := &cobra.Command{ Use: "zitadel", Short: "The ZITADEL CLI lets you interact with ZITADEL", @@ -51,9 +53,9 @@ func New(out io.Writer, in io.Reader, args []string) *cobra.Command { admin.New(), //is now deprecated, remove later on initialise.New(), setup.New(), - start.New(), - start.NewStartFromInit(), - start.NewStartFromSetup(), + start.New(server), + start.NewStartFromInit(server), + start.NewStartFromSetup(server), key.New(), ) @@ -69,3 +71,30 @@ func initConfig() { logging.WithFields("file", file).OnError(err).Warn("unable to read config file") } } + +type TestServer struct { + *start.Server + wg sync.WaitGroup +} + +func (s *TestServer) Done() { + s.Shutdown <- os.Interrupt + s.wg.Wait() +} + +func NewTestServer(args []string) *TestServer { + testServer := new(TestServer) + server := make(chan *start.Server, 1) + + testServer.wg.Add(1) + go func(wg *sync.WaitGroup) { + defer wg.Done() + + cmd := New(os.Stdout, os.Stdin, args, server) + cmd.SetArgs(args) + logging.OnError(cmd.Execute()).Fatal() + }(&testServer.wg) + + testServer.Server = <-server + return testServer +} diff --git a/cmd/zitadel_test.go b/cmd/zitadel_test.go new file mode 100644 index 0000000000..7302469d3d --- /dev/null +++ b/cmd/zitadel_test.go @@ -0,0 +1,13 @@ +package cmd + +import ( + "strings" + "testing" +) + +const commandLine = `start-from-init --masterkey MasterkeyNeedsToHave32Characters --tlsMode disabled --config ../e2e/config/localhost/zitadel.yaml --steps ../e2e/config/localhost/zitadel.yaml` + +func TestNewTestServer(t *testing.T) { + s := NewTestServer(strings.Split(commandLine, " ")) + defer s.Done() +} diff --git a/main.go b/main.go index 0f3264080e..97689bbe41 100644 --- a/main.go +++ b/main.go @@ -10,6 +10,6 @@ import ( func main() { args := os.Args[1:] - rootCmd := cmd.New(os.Stdout, os.Stdin /*, int(os.Stdin.Fd())*/, args) + rootCmd := cmd.New(os.Stdout, os.Stdin, args, nil) cobra.CheckErr(rootCmd.Execute()) } From a22b58f1c04a7ec7f58c41a5422ce117ea79c819 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Tue, 25 Apr 2023 15:04:35 +0300 Subject: [PATCH 02/16] simple test of a health endpoint --- cmd/zitadel.go | 29 --------- cmd/zitadel_test.go | 13 ---- internal/api/grpc/admin/admin_test.go | 29 +++++++++ internal/api/grpc/admin/information_test.go | 16 +++++ internal/integration/integration.go | 68 +++++++++++++++++++++ internal/integration/integration_test.go | 18 ++++++ 6 files changed, 131 insertions(+), 42 deletions(-) delete mode 100644 cmd/zitadel_test.go create mode 100644 internal/api/grpc/admin/admin_test.go create mode 100644 internal/api/grpc/admin/information_test.go create mode 100644 internal/integration/integration.go create mode 100644 internal/integration/integration_test.go diff --git a/cmd/zitadel.go b/cmd/zitadel.go index 8de2aaf5b4..0c839f2678 100644 --- a/cmd/zitadel.go +++ b/cmd/zitadel.go @@ -5,9 +5,7 @@ import ( _ "embed" "errors" "io" - "os" "strings" - "sync" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -71,30 +69,3 @@ func initConfig() { logging.WithFields("file", file).OnError(err).Warn("unable to read config file") } } - -type TestServer struct { - *start.Server - wg sync.WaitGroup -} - -func (s *TestServer) Done() { - s.Shutdown <- os.Interrupt - s.wg.Wait() -} - -func NewTestServer(args []string) *TestServer { - testServer := new(TestServer) - server := make(chan *start.Server, 1) - - testServer.wg.Add(1) - go func(wg *sync.WaitGroup) { - defer wg.Done() - - cmd := New(os.Stdout, os.Stdin, args, server) - cmd.SetArgs(args) - logging.OnError(cmd.Execute()).Fatal() - }(&testServer.wg) - - testServer.Server = <-server - return testServer -} diff --git a/cmd/zitadel_test.go b/cmd/zitadel_test.go deleted file mode 100644 index 7302469d3d..0000000000 --- a/cmd/zitadel_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package cmd - -import ( - "strings" - "testing" -) - -const commandLine = `start-from-init --masterkey MasterkeyNeedsToHave32Characters --tlsMode disabled --config ../e2e/config/localhost/zitadel.yaml --steps ../e2e/config/localhost/zitadel.yaml` - -func TestNewTestServer(t *testing.T) { - s := NewTestServer(strings.Split(commandLine, " ")) - defer s.Done() -} diff --git a/internal/api/grpc/admin/admin_test.go b/internal/api/grpc/admin/admin_test.go new file mode 100644 index 0000000000..9ca65d819c --- /dev/null +++ b/internal/api/grpc/admin/admin_test.go @@ -0,0 +1,29 @@ +package admin_test + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/zitadel/zitadel/internal/integration" +) + +const commandLine = `start-from-init --masterkey MasterkeyNeedsToHave32Characters --tlsMode disabled --config ../../e2e/config/localhost/zitadel.yaml --steps ../../e2e/config/localhost/zitadel.yaml` + +var ( + Tester *integration.Tester +) + +func TestMain(m *testing.M) { + os.Exit(func() int { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + Tester = integration.NewTester(ctx, strings.Split(commandLine, " ")) + defer Tester.Done() + + return m.Run() + }()) +} diff --git a/internal/api/grpc/admin/information_test.go b/internal/api/grpc/admin/information_test.go new file mode 100644 index 0000000000..e2867805be --- /dev/null +++ b/internal/api/grpc/admin/information_test.go @@ -0,0 +1,16 @@ +package admin_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/zitadel/zitadel/pkg/grpc/admin" +) + +func TestServer_Healthz(t *testing.T) { + client := admin.NewAdminServiceClient(Tester.ClientConn) + _, err := client.Healthz(context.TODO(), &admin.HealthzRequest{}) + require.NoError(t, err) +} diff --git a/internal/integration/integration.go b/internal/integration/integration.go new file mode 100644 index 0000000000..e5a91f786b --- /dev/null +++ b/internal/integration/integration.go @@ -0,0 +1,68 @@ +// Package integration provides helpers for integration testing. +package integration + +import ( + "context" + "fmt" + "os" + "sync" + + "github.com/zitadel/logging" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + "github.com/zitadel/zitadel/cmd" + "github.com/zitadel/zitadel/cmd/start" +) + +type Tester struct { + *start.Server + ClientConn *grpc.ClientConn + + wg sync.WaitGroup // used for shutdown +} + +func (s *Tester) createClientConn(ctx context.Context) { + target := fmt.Sprintf("localhost:%d", s.Config.Port) + cc, err := grpc.DialContext(ctx, target, + grpc.WithBlock(), grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + s.Shutdown <- os.Interrupt + s.wg.Wait() + } + logging.OnError(err).Fatal("integration tester client dial") + logging.New().WithField("target", target).Info("finished dialing grpc client conn") + + s.ClientConn = cc +} + +func (s *Tester) Done() { + err := s.ClientConn.Close() + logging.OnError(err).Error("integration tester client close") + + s.Shutdown <- os.Interrupt + s.wg.Wait() +} + +func NewTester(ctx context.Context, args []string) *Tester { + tester := new(Tester) + sc := make(chan *start.Server) + tester.wg.Add(1) + go func(wg *sync.WaitGroup) { + defer wg.Done() + + cmd := cmd.New(os.Stdout, os.Stdin, args, sc) + cmd.SetArgs(args) + logging.OnError(cmd.Execute()).Fatal() + }(&tester.wg) + + select { + case tester.Server = <-sc: + case <-ctx.Done(): + logging.OnError(ctx.Err()).Fatal("waiting for integration tester server") + } + tester.createClientConn(ctx) + + return tester +} diff --git a/internal/integration/integration_test.go b/internal/integration/integration_test.go new file mode 100644 index 0000000000..4da09350fb --- /dev/null +++ b/internal/integration/integration_test.go @@ -0,0 +1,18 @@ +package integration + +import ( + "context" + "strings" + "testing" + "time" +) + +const commandLine = `start-from-init --masterkey MasterkeyNeedsToHave32Characters --tlsMode disabled --config ../../e2e/config/localhost/zitadel.yaml --steps ../../e2e/config/localhost/zitadel.yaml` + +func TestNewTester(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + s := NewTester(ctx, strings.Split(commandLine, " ")) + defer s.Done() +} From 79084089eaedb5ee8b9a4eb55adb338757be333f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Tue, 25 Apr 2023 18:11:04 +0300 Subject: [PATCH 03/16] add github action for integration tests --- .github/workflows/integration.yml | 35 ++++++++++++++++ e2e/config/integration/docker-compose.yaml | 24 +++++++++++ .../grpc/admin/failed_event_converter_test.go | 2 + .../grpc/admin/iam_member_converter_test.go | 2 + internal/api/grpc/admin/idp_converter_test.go | 2 + ...est.go => information_integration_test.go} | 18 +++++--- internal/api/grpc/admin/information_test.go | 16 ------- internal/api/grpc/errors/caos_errors_test.go | 2 + internal/api/grpc/header_test.go | 2 + .../api/grpc/management/idp_converter_test.go | 2 + .../middleware/auth_interceptor_test.go | 2 + .../middleware/error_interceptor_test.go | 2 + .../middleware/instance_interceptor_test.go | 2 + internal/api/grpc/server/probes_test.go | 2 + .../system/failed_event_converter_test.go | 2 + internal/integration/config/cockroach.yaml | 10 +++++ internal/integration/config/postgres.yaml | 15 +++++++ internal/integration/config/zitadel.yaml | 38 +++++++++++++++++ internal/integration/integration.go | 42 ++++++++++++++++--- internal/integration/integration_test.go | 7 ++-- 20 files changed, 196 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/integration.yml create mode 100644 e2e/config/integration/docker-compose.yaml rename internal/api/grpc/admin/{admin_test.go => information_integration_test.go} (50%) delete mode 100644 internal/api/grpc/admin/information_test.go create mode 100644 internal/integration/config/cockroach.yaml create mode 100644 internal/integration/config/postgres.yaml create mode 100644 internal/integration/config/zitadel.yaml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000000..6c80e53723 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,35 @@ +on: + push: + +jobs: + integration-tests: + strategy: + matrix: + db: [cockroach, postgres] + runs-on: ubuntu-20.04 + env: + DOCKER_BUILDKIT: 1 + steps: + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.19 + - name: Source checkout + uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + driver: docker + install: true + - name: Generate gRPC definitions + run: docker build -f build/grpc/Dockerfile -t zitadel-base:local . + - name: Copy gRPC definitions + run: docker build -f build/zitadel/Dockerfile . -t zitadel-go-base --target go-copy -o . + - name: Download Go modules + run: go mod download + - name: Start ${{ matrix.db }} database + run: docker compose -f e2e/config/integration/docker-compose.yaml up --wait ${{ matrix.db }} + - name: Run integration test + env: + INTEGRATION_DB_FLAVOR: ${{ matrix.db }} + run: go test -tags=integration -v ./internal/integration ./internal/api/grpc/... diff --git a/e2e/config/integration/docker-compose.yaml b/e2e/config/integration/docker-compose.yaml new file mode 100644 index 0000000000..85d68e4a2a --- /dev/null +++ b/e2e/config/integration/docker-compose.yaml @@ -0,0 +1,24 @@ +version: '3.8' + +services: + cockroach: + extends: + file: '../localhost/docker-compose.yaml' + service: 'db' + + postgres: + restart: 'always' + image: 'postgres:15' + environment: + - POSTGRES_USER=zitadel + - PGUSER=zitadel + - POSTGRES_DB=zitadel + - POSTGRES_HOST_AUTH_METHOD=trust + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: '10s' + timeout: '30s' + retries: 5 + start_period: '20s' + ports: + - 5432:5432 diff --git a/internal/api/grpc/admin/failed_event_converter_test.go b/internal/api/grpc/admin/failed_event_converter_test.go index 6edc36c1e2..cb6c6bddab 100644 --- a/internal/api/grpc/admin/failed_event_converter_test.go +++ b/internal/api/grpc/admin/failed_event_converter_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package admin import ( diff --git a/internal/api/grpc/admin/iam_member_converter_test.go b/internal/api/grpc/admin/iam_member_converter_test.go index 74dd329ee1..f2b90031db 100644 --- a/internal/api/grpc/admin/iam_member_converter_test.go +++ b/internal/api/grpc/admin/iam_member_converter_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package admin import ( diff --git a/internal/api/grpc/admin/idp_converter_test.go b/internal/api/grpc/admin/idp_converter_test.go index 23d378b888..0eb04efcb1 100644 --- a/internal/api/grpc/admin/idp_converter_test.go +++ b/internal/api/grpc/admin/idp_converter_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package admin import ( diff --git a/internal/api/grpc/admin/admin_test.go b/internal/api/grpc/admin/information_integration_test.go similarity index 50% rename from internal/api/grpc/admin/admin_test.go rename to internal/api/grpc/admin/information_integration_test.go index 9ca65d819c..65f2dd93b7 100644 --- a/internal/api/grpc/admin/admin_test.go +++ b/internal/api/grpc/admin/information_integration_test.go @@ -1,16 +1,18 @@ +//go:build integration + package admin_test import ( "context" "os" - "strings" "testing" "time" - "github.com/zitadel/zitadel/internal/integration" -) + "github.com/stretchr/testify/require" -const commandLine = `start-from-init --masterkey MasterkeyNeedsToHave32Characters --tlsMode disabled --config ../../e2e/config/localhost/zitadel.yaml --steps ../../e2e/config/localhost/zitadel.yaml` + "github.com/zitadel/zitadel/internal/integration" + "github.com/zitadel/zitadel/pkg/grpc/admin" +) var ( Tester *integration.Tester @@ -21,9 +23,15 @@ func TestMain(m *testing.M) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - Tester = integration.NewTester(ctx, strings.Split(commandLine, " ")) + Tester = integration.NewTester(ctx) defer Tester.Done() return m.Run() }()) } + +func TestServer_Healthz(t *testing.T) { + client := admin.NewAdminServiceClient(Tester.ClientConn) + _, err := client.Healthz(context.TODO(), &admin.HealthzRequest{}) + require.NoError(t, err) +} diff --git a/internal/api/grpc/admin/information_test.go b/internal/api/grpc/admin/information_test.go deleted file mode 100644 index e2867805be..0000000000 --- a/internal/api/grpc/admin/information_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package admin_test - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/zitadel/zitadel/pkg/grpc/admin" -) - -func TestServer_Healthz(t *testing.T) { - client := admin.NewAdminServiceClient(Tester.ClientConn) - _, err := client.Healthz(context.TODO(), &admin.HealthzRequest{}) - require.NoError(t, err) -} diff --git a/internal/api/grpc/errors/caos_errors_test.go b/internal/api/grpc/errors/caos_errors_test.go index 3a14f1bbfc..6efe37b588 100644 --- a/internal/api/grpc/errors/caos_errors_test.go +++ b/internal/api/grpc/errors/caos_errors_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package errors import ( diff --git a/internal/api/grpc/header_test.go b/internal/api/grpc/header_test.go index e304900073..085473328e 100644 --- a/internal/api/grpc/header_test.go +++ b/internal/api/grpc/header_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package grpc import ( diff --git a/internal/api/grpc/management/idp_converter_test.go b/internal/api/grpc/management/idp_converter_test.go index 999be2ea06..ae30b09c89 100644 --- a/internal/api/grpc/management/idp_converter_test.go +++ b/internal/api/grpc/management/idp_converter_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package management import ( diff --git a/internal/api/grpc/server/middleware/auth_interceptor_test.go b/internal/api/grpc/server/middleware/auth_interceptor_test.go index abe50606f9..d51de2ff7a 100644 --- a/internal/api/grpc/server/middleware/auth_interceptor_test.go +++ b/internal/api/grpc/server/middleware/auth_interceptor_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package middleware import ( diff --git a/internal/api/grpc/server/middleware/error_interceptor_test.go b/internal/api/grpc/server/middleware/error_interceptor_test.go index 82dc35f64a..3ce69e0d34 100644 --- a/internal/api/grpc/server/middleware/error_interceptor_test.go +++ b/internal/api/grpc/server/middleware/error_interceptor_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package middleware import ( diff --git a/internal/api/grpc/server/middleware/instance_interceptor_test.go b/internal/api/grpc/server/middleware/instance_interceptor_test.go index 513dfe96cc..194fafbdde 100644 --- a/internal/api/grpc/server/middleware/instance_interceptor_test.go +++ b/internal/api/grpc/server/middleware/instance_interceptor_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package middleware import ( diff --git a/internal/api/grpc/server/probes_test.go b/internal/api/grpc/server/probes_test.go index 15c889e24c..1d743178c8 100644 --- a/internal/api/grpc/server/probes_test.go +++ b/internal/api/grpc/server/probes_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package server import ( diff --git a/internal/api/grpc/system/failed_event_converter_test.go b/internal/api/grpc/system/failed_event_converter_test.go index f91c2d1b4f..27caeff918 100644 --- a/internal/api/grpc/system/failed_event_converter_test.go +++ b/internal/api/grpc/system/failed_event_converter_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package system_test import ( diff --git a/internal/integration/config/cockroach.yaml b/internal/integration/config/cockroach.yaml new file mode 100644 index 0000000000..920e3cd6ec --- /dev/null +++ b/internal/integration/config/cockroach.yaml @@ -0,0 +1,10 @@ +Database: + cockroach: + Host: localhost + Port: 26257 + Database: zitadel + Options: "" + User: + Username: zitadel + Admin: + Username: root diff --git a/internal/integration/config/postgres.yaml b/internal/integration/config/postgres.yaml new file mode 100644 index 0000000000..0ef4739e25 --- /dev/null +++ b/internal/integration/config/postgres.yaml @@ -0,0 +1,15 @@ +Database: + postgres: + Host: localhost + Port: 5432 + Database: zitadel + MaxOpenConns: 20 + MaxIdleConns: 10 + User: + Username: zitadel + SSL: + Mode: disable + Admin: + Username: zitadel + SSL: + Mode: disable diff --git a/internal/integration/config/zitadel.yaml b/internal/integration/config/zitadel.yaml new file mode 100644 index 0000000000..de4eb8aa1d --- /dev/null +++ b/internal/integration/config/zitadel.yaml @@ -0,0 +1,38 @@ +Log: + Level: debug + +TLS: + Enabled: false + +FirstInstance: + Org: + Human: + PasswordChangeRequired: false + +LogStore: + Access: + Database: + Enabled: true + Debounce: + MinFrequency: 0s + MaxBulkSize: 0 + Execution: + Database: + Enabled: true + Stdout: + Enabled: true + +Quotas: + Access: + ExhaustedCookieKey: "zitadel.quota.limiting" + ExhaustedCookieMaxAge: "60s" + +Projections: + Customizations: + NotificationsQuotas: + RequeueEvery: 1s + +DefaultInstance: + LoginPolicy: + MfaInitSkipLifetime: "0" + diff --git a/internal/integration/integration.go b/internal/integration/integration.go index e5a91f786b..2cba61c2bd 100644 --- a/internal/integration/integration.go +++ b/internal/integration/integration.go @@ -2,11 +2,15 @@ package integration import ( + "bytes" "context" + _ "embed" "fmt" "os" + "strings" "sync" + "github.com/spf13/viper" "github.com/zitadel/logging" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -15,6 +19,15 @@ import ( "github.com/zitadel/zitadel/cmd/start" ) +var ( + //go:embed config/zitadel.yaml + zitadelYAML []byte + //go:embed config/cockroach.yaml + cockroachYAML []byte + //go:embed config/postgres.yaml + postgresYAML []byte +) + type Tester struct { *start.Server ClientConn *grpc.ClientConn @@ -22,6 +35,8 @@ type Tester struct { wg sync.WaitGroup // used for shutdown } +const commandLine = `start-from-init --masterkey MasterkeyNeedsToHave32Characters --tlsMode disabled` + func (s *Tester) createClientConn(ctx context.Context) { target := fmt.Sprintf("localhost:%d", s.Config.Port) cc, err := grpc.DialContext(ctx, target, @@ -45,16 +60,31 @@ func (s *Tester) Done() { s.wg.Wait() } -func NewTester(ctx context.Context, args []string) *Tester { - tester := new(Tester) +func NewTester(ctx context.Context) *Tester { + args := strings.Split(commandLine, " ") + sc := make(chan *start.Server) + cmd := cmd.New(os.Stdout, os.Stdin, args, sc) + cmd.SetArgs(args) + err := viper.MergeConfig(bytes.NewBuffer(zitadelYAML)) + logging.OnError(err).Fatal() + + flavor := os.Getenv("INTEGRATION_DB_FLAVOR") + switch flavor { + case "cockroach": + err = viper.MergeConfig(bytes.NewBuffer(cockroachYAML)) + case "postgres": + err = viper.MergeConfig(bytes.NewBuffer(postgresYAML)) + default: + logging.New().WithField("flavor", flavor).Fatal("unknown db flavor set in INTEGRATION_DB_FLAVOR") + } + logging.OnError(err).Fatal() + + tester := new(Tester) tester.wg.Add(1) go func(wg *sync.WaitGroup) { - defer wg.Done() - - cmd := cmd.New(os.Stdout, os.Stdin, args, sc) - cmd.SetArgs(args) logging.OnError(cmd.Execute()).Fatal() + wg.Done() }(&tester.wg) select { diff --git a/internal/integration/integration_test.go b/internal/integration/integration_test.go index 4da09350fb..bb9b12f97e 100644 --- a/internal/integration/integration_test.go +++ b/internal/integration/integration_test.go @@ -1,18 +1,17 @@ +//go:build integration + package integration import ( "context" - "strings" "testing" "time" ) -const commandLine = `start-from-init --masterkey MasterkeyNeedsToHave32Characters --tlsMode disabled --config ../../e2e/config/localhost/zitadel.yaml --steps ../../e2e/config/localhost/zitadel.yaml` - func TestNewTester(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - s := NewTester(ctx, strings.Split(commandLine, " ")) + s := NewTester(ctx) defer s.Done() } From 68200dcbe96f5c05c00327f74aaced4dd9eae849 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Tue, 25 Apr 2023 18:11:53 +0300 Subject: [PATCH 04/16] temporarily remove other workflow for fast running --- .github/workflows/codeql-analysis.yml | 41 ------------ .github/workflows/e2e.yml | 66 ------------------- .github/workflows/issues.yml | 20 ------ .github/workflows/release-channels.yml | 52 --------------- .github/workflows/test-code.yml | 75 ---------------------- .github/workflows/test-docs.yml | 21 ------ .github/workflows/zitadel.yml | 88 -------------------------- 7 files changed, 363 deletions(-) delete mode 100644 .github/workflows/codeql-analysis.yml delete mode 100644 .github/workflows/e2e.yml delete mode 100644 .github/workflows/issues.yml delete mode 100644 .github/workflows/release-channels.yml delete mode 100644 .github/workflows/test-code.yml delete mode 100644 .github/workflows/test-docs.yml delete mode 100644 .github/workflows/zitadel.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 6ca453c32c..0000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: "Code scanning - action" - -on: - push: - branches: - - 'main' - paths-ignore: - - 'docs/**' - pull_request: - # The branches below must be a subset of the branches above - branches: - - 'main' - paths-ignore: - - 'docs/**' - -jobs: - CodeQL-Build: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v3 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - # Override language selection by uncommenting this and choosing your languages - with: - languages: go, javascript - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - #- run: | - # make bootstrap - # make release - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml deleted file mode 100644 index 50f1ce85c6..0000000000 --- a/.github/workflows/e2e.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: "ZITADEL e2e Tests" - -on: - workflow_run: - workflows: [ZITADEL Release] - types: - - completed - workflow_dispatch: - inputs: - releaseversion: - description: 'Release version to test' - required: true - default: 'latest' - -jobs: - test: - strategy: - matrix: - browser: [firefox, chrome] - runs-on: ubuntu-20.04 - env: - ZITADEL_IMAGE_REGISTRY: 'ghcr.io/zitadel/zitadel' - steps: - - name: Checkout Repository - uses: actions/checkout@v3 - - name: Set TAG env manual trigger - if: github.event_name == 'workflow_dispatch' - run: echo "ZITADEL_IMAGE=${ZITADEL_IMAGE_REGISTRY}:${{ github.event.inputs.releaseversion }}" >> $GITHUB_ENV - - name: get latest tag - uses: actions-ecosystem/action-get-latest-tag@v1 - id: get-latest-tag - with: - semver_only: true - - name: Set TAG env on ZITADEL release - if: github.event_name == 'workflow_run' - run: echo "ZITADEL_IMAGE=${ZITADEL_IMAGE_REGISTRY}:${{ steps.get-latest-tag.outputs.tag }}" >> $GITHUB_ENV - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - driver: docker - install: true - - name: Test ${{ matrix.browser }} - run: docker compose run --service-ports e2e --browser ${{ matrix.browser }} - working-directory: e2e/config/host.docker.internal - - name: Ensure Artifacts Directory Exists - run: mkdir -p ./.artifacts - - name: Save ZITADEL Logs - if: always() - run: docker compose logs zitadel > ../../../.artifacts/e2e-compose-zitadel.log - working-directory: e2e/config/host.docker.internal - - name: Save Prepare Logs - if: always() - run: docker compose logs prepare > ../../../.artifacts/e2e-compose-prepare.log - working-directory: e2e/config/host.docker.internal - - name: Archive production tests ${{ matrix.browser }} - if: always() - uses: actions/upload-artifact@v3 - with: - name: production-tests-${{ matrix.browser }} - path: | - e2e/cypress/results - e2e/cypress/videos - e2e/cypress/screenshots - .artifacts/e2e-compose-zitadel.log - .artifacts/e2e-compose-prepare.log - retention-days: 30 diff --git a/.github/workflows/issues.yml b/.github/workflows/issues.yml deleted file mode 100644 index cc350f8e76..0000000000 --- a/.github/workflows/issues.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Add new issues to product management project - -on: - issues: - types: - - opened - -jobs: - add-to-project: - name: Add issue to project - runs-on: ubuntu-latest - steps: - - uses: actions/add-to-project@v0.3.0 - with: - # You can target a repository in a different organization - # to the issue - project-url: https://github.com/orgs/zitadel/projects/2 - github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} - labeled: OKR - label-operator: NOT diff --git a/.github/workflows/release-channels.yml b/.github/workflows/release-channels.yml deleted file mode 100644 index 864a0a7778..0000000000 --- a/.github/workflows/release-channels.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: ZITADEL Release tags - -on: - push: - branches: - - "main" - paths: - - 'release-channels.yaml' - workflow_dispatch: - -permissions: - contents: write - packages: write - -jobs: - Build: - runs-on: ubuntu-20.04 - env: - DOCKER_BUILDKIT: 1 - steps: - - name: Source checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: get stable tag - run: echo STABLE_RELEASE=$(yq eval '.stable' release-channels.yaml) >> $GITHUB_ENV - - name: checkout stable tag - uses: actions/checkout@v3 - with: - fetch-depth: 0 - ref: ${{ env.STABLE_RELEASE }} - - name: GitHub Container Registry Login - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - uses: tibdex/github-app-token@v1 - id: generate-token - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - - name: Google Artifact Registry Login - uses: docker/login-action@v2 - with: - registry: europe-docker.pkg.dev - username: _json_key_base64 - password: ${{ secrets.GCR_JSON_KEY_BASE64 }} - - name: copy release to stable - run: | - skopeo --version - skopeo copy --all docker://ghcr.io/zitadel/zitadel:$STABLE_RELEASE docker://ghcr.io/zitadel/zitadel:stable diff --git a/.github/workflows/test-code.yml b/.github/workflows/test-code.yml deleted file mode 100644 index 7a4793dbe0..0000000000 --- a/.github/workflows/test-code.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: ZITADEL PR - -on: - pull_request: - paths-ignore: - - 'docs/**' - - 'guides/**' - - '**.md' - - 'release-channels.yaml' - -jobs: - Build-ZITADEL: - runs-on: ubuntu-20.04 - env: - DOCKER_BUILDKIT: 1 - steps: - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: 1.19 - - name: Source checkout - uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - driver: docker - install: true - - name: Install GoReleaser - uses: goreleaser/goreleaser-action@v3 - with: - install-only: true - version: v1.10.3 - - name: Build and Unit Test - run: GOOS="linux" GOARCH="amd64" goreleaser build --id prod --snapshot --single-target --rm-dist --output .artifacts/zitadel/zitadel - - name: linting - uses: golangci/golangci-lint-action@v3 - with: - version: v1.52 - only-new-issues: true - skip-pkg-cache: true - - name: Publish go coverage - uses: codecov/codecov-action@v3.1.0 - with: - file: .artifacts/codecov/profile.cov - name: go-codecov - # As goreleaser doesn't build a dockerfile in snapshot mode, we have to build it here - - name: Build Docker Image - run: docker build -t zitadel:pr --file build/Dockerfile .artifacts/zitadel - - name: Run E2E Tests - run: docker compose run --service-ports e2e --browser chrome - working-directory: e2e/config/host.docker.internal - env: - ZITADEL_IMAGE: zitadel:pr - - name: Save ZITADEL Logs - if: always() - run: docker compose logs zitadel > ../../../.artifacts/e2e-compose-zitadel.log - working-directory: e2e/config/host.docker.internal - - name: Save Prepare Logs - if: always() - run: docker compose logs prepare > ../../../.artifacts/e2e-compose-prepare.log - working-directory: e2e/config/host.docker.internal - - name: Archive Test Results - if: always() - uses: actions/upload-artifact@v3 - with: - name: pull-request-tests - path: | - e2e/cypress/results - e2e/cypress/videos - e2e/cypress/screenshots - .artifacts/e2e-compose-zitadel.log - .artifacts/e2e-compose-prepare.log - retention-days: 30 diff --git a/.github/workflows/test-docs.yml b/.github/workflows/test-docs.yml deleted file mode 100644 index 7bf3b222b0..0000000000 --- a/.github/workflows/test-docs.yml +++ /dev/null @@ -1,21 +0,0 @@ -# ATTENTION: Although this workflow doesn't do much, it is still important. -# It is complementary to the workflow in the file test-code.yml. -# It enables to exclude files for the workflow and still mark the Test job as required without having pending PRs. -# GitHub recommends this solution here: -# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks - -name: ZITADEL PR - -on: - pull_request: - paths: - - 'docs/**' - - 'guides/**' - - '**.md' - - 'release-channels.yaml' - -jobs: - Build-ZITADEL: - runs-on: ubuntu-20.04 - steps: - - run: 'echo "No tests for docs are implemented, yet"' diff --git a/.github/workflows/zitadel.yml b/.github/workflows/zitadel.yml deleted file mode 100644 index 2691109f80..0000000000 --- a/.github/workflows/zitadel.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: ZITADEL Release - -on: - push: - tags-ignore: - - "*" - workflow_dispatch: - -permissions: - contents: write - packages: write - -jobs: - Build: - runs-on: ubuntu-20.04 - env: - DOCKER_BUILDKIT: 1 - steps: - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: 1.19 - - name: Source checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Fetch all tags - run: git fetch --force --tags - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - driver: docker - install: true - - name: Tag - id: semantic - uses: cycjimmy/semantic-release-action@v2 - with: - dry_run: false - semantic_version: 19.0.2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: GitHub Container Registry Login - if: steps.semantic.outputs.new_release_published == 'true' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - uses: tibdex/github-app-token@v1 - id: generate-token - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - - name: Google Artifact Registry Login - if: steps.semantic.outputs.new_release_published == 'true' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' - uses: docker/login-action@v2 - with: - registry: europe-docker.pkg.dev - username: _json_key_base64 - password: ${{ secrets.GCR_JSON_KEY_BASE64 }} - - uses: goreleaser/goreleaser-action@v3 - name: Publish ZITADEL - if: steps.semantic.outputs.new_release_published == 'true' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' - with: - distribution: goreleaser - version: v1.11.0 - args: release --timeout 50m - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GORELEASER_TOKEN_TAP: ${{ steps.generate-token.outputs.token }} - RELEASE_VERSION: ${{ steps.semantic.outputs.release-version }} # I think this line is not needed. Nevertheless, it's explicit - DISCORD_WEBHOOK_ID: "976058224484687932" - DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" - - name: Publish go coverage - uses: codecov/codecov-action@v3.1.0 - with: - file: .artifacts/codecov/profile.cov - name: go-codecov - - name: Bump Chart Version - uses: peter-evans/repository-dispatch@v2 - if: steps.semantic.outputs.new_release_published == 'true' && github.ref == 'refs/heads/main' - with: - token: ${{ steps.generate-token.outputs.token }} - repository: zitadel/zitadel-charts - event-type: zitadel-released - client-payload: '{"semanticoutputs": "${{ steps.semantic.outputs }}"}' From 234186c60c933fe8f76de0a0c974a513b08a38bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Tue, 25 Apr 2023 18:30:08 +0300 Subject: [PATCH 05/16] do not run tests in parallel --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 6c80e53723..f87938d827 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -32,4 +32,4 @@ jobs: - name: Run integration test env: INTEGRATION_DB_FLAVOR: ${{ matrix.db }} - run: go test -tags=integration -v ./internal/integration ./internal/api/grpc/... + run: go test -tags=integration -parallel 1 -v ./internal/integration ./internal/api/grpc/... From 90ba3a8d92a1479d31ff6e1b5a4b579a4d361f16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Wed, 26 Apr 2023 19:54:47 +0300 Subject: [PATCH 06/16] poll on test start --- .github/workflows/integration.yml | 11 +++-- cmd/start/start.go | 2 - .../admin/information_integration_test.go | 3 +- internal/integration/integration.go | 45 ++++++++++++++++--- internal/integration/integration_test.go | 2 +- 5 files changed, 47 insertions(+), 16 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f87938d827..90edd89bfa 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -9,6 +9,7 @@ jobs: runs-on: ubuntu-20.04 env: DOCKER_BUILDKIT: 1 + INTEGRATION_DB_FLAVOR: ${{ matrix.db }} steps: - name: Set up Go uses: actions/setup-go@v3 @@ -28,8 +29,10 @@ jobs: - name: Download Go modules run: go mod download - name: Start ${{ matrix.db }} database - run: docker compose -f e2e/config/integration/docker-compose.yaml up --wait ${{ matrix.db }} - - name: Run integration test - env: - INTEGRATION_DB_FLAVOR: ${{ matrix.db }} + run: docker compose -f e2e/config/integration/docker-compose.yaml up --wait ${INTEGRATION_DB_FLAVOR} + - name: Run zitadel init and setup + run: | + go run main.go init --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml + go run main.go setup --masterkey MasterkeyNeedsToHave32Characters --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml + - name: Run integration tests run: go test -tags=integration -parallel 1 -v ./internal/integration ./internal/api/grpc/... diff --git a/cmd/start/start.go b/cmd/start/start.go index 361619536b..3d6e4746c3 100644 --- a/cmd/start/start.go +++ b/cmd/start/start.go @@ -89,7 +89,6 @@ Requirements: } type Server struct { - background context.Context Config *Config DB *database.DB KeyStorage crypto.KeyStorage @@ -203,7 +202,6 @@ func startZitadel(config *Config, masterKey string, server chan<- *Server) error if server != nil { server <- &Server{ - background: ctx, Config: config, DB: dbClient, KeyStorage: keyStorage, diff --git a/internal/api/grpc/admin/information_integration_test.go b/internal/api/grpc/admin/information_integration_test.go index 65f2dd93b7..cac2c952aa 100644 --- a/internal/api/grpc/admin/information_integration_test.go +++ b/internal/api/grpc/admin/information_integration_test.go @@ -22,7 +22,6 @@ func TestMain(m *testing.M) { os.Exit(func() int { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - Tester = integration.NewTester(ctx) defer Tester.Done() @@ -31,7 +30,7 @@ func TestMain(m *testing.M) { } func TestServer_Healthz(t *testing.T) { - client := admin.NewAdminServiceClient(Tester.ClientConn) + client := admin.NewAdminServiceClient(Tester.GRPCClientConn) _, err := client.Healthz(context.TODO(), &admin.HealthzRequest{}) require.NoError(t, err) } diff --git a/internal/integration/integration.go b/internal/integration/integration.go index 2cba61c2bd..0b8fa21792 100644 --- a/internal/integration/integration.go +++ b/internal/integration/integration.go @@ -9,6 +9,7 @@ import ( "os" "strings" "sync" + "time" "github.com/spf13/viper" "github.com/zitadel/logging" @@ -17,6 +18,7 @@ import ( "github.com/zitadel/zitadel/cmd" "github.com/zitadel/zitadel/cmd/start" + "github.com/zitadel/zitadel/pkg/grpc/admin" ) var ( @@ -30,12 +32,11 @@ var ( type Tester struct { *start.Server - ClientConn *grpc.ClientConn - - wg sync.WaitGroup // used for shutdown + GRPCClientConn *grpc.ClientConn + wg sync.WaitGroup // used for shutdown } -const commandLine = `start-from-init --masterkey MasterkeyNeedsToHave32Characters --tlsMode disabled` +const commandLine = `start --masterkey MasterkeyNeedsToHave32Characters` func (s *Tester) createClientConn(ctx context.Context) { target := fmt.Sprintf("localhost:%d", s.Config.Port) @@ -49,11 +50,41 @@ func (s *Tester) createClientConn(ctx context.Context) { logging.OnError(err).Fatal("integration tester client dial") logging.New().WithField("target", target).Info("finished dialing grpc client conn") - s.ClientConn = cc + s.GRPCClientConn = cc + err = s.pollHealth(ctx) + logging.OnError(err).Fatal("integration tester health") +} + +// pollHealth waits until a healthy status is reported. +// TODO: remove when we make the setup blocking on all +// projections completed. +func (s *Tester) pollHealth(ctx context.Context) (err error) { + client := admin.NewAdminServiceClient(s.GRPCClientConn) + + for { + err = func(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + _, err := client.Healthz(ctx, &admin.HealthzRequest{}) + return err + }(ctx) + if err == nil { + return nil + } + logging.WithError(err).Info("poll healthz") + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + continue + } + } } func (s *Tester) Done() { - err := s.ClientConn.Close() + err := s.GRPCClientConn.Close() logging.OnError(err).Error("integration tester client close") s.Shutdown <- os.Interrupt @@ -71,7 +102,7 @@ func NewTester(ctx context.Context) *Tester { flavor := os.Getenv("INTEGRATION_DB_FLAVOR") switch flavor { - case "cockroach": + case "cockroach", "": err = viper.MergeConfig(bytes.NewBuffer(cockroachYAML)) case "postgres": err = viper.MergeConfig(bytes.NewBuffer(postgresYAML)) diff --git a/internal/integration/integration_test.go b/internal/integration/integration_test.go index bb9b12f97e..a01d8cc7ff 100644 --- a/internal/integration/integration_test.go +++ b/internal/integration/integration_test.go @@ -9,7 +9,7 @@ import ( ) func TestNewTester(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() s := NewTester(ctx) From 1dc46b16b0b3bc5796d1c85ddc2f6cc2ff3cf227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Wed, 26 Apr 2023 19:55:13 +0300 Subject: [PATCH 07/16] remove negated integration tags --- internal/api/grpc/admin/failed_event_converter_test.go | 2 -- internal/api/grpc/admin/iam_member_converter_test.go | 2 -- internal/api/grpc/admin/idp_converter_test.go | 2 -- internal/api/grpc/errors/caos_errors_test.go | 2 -- internal/api/grpc/header_test.go | 2 -- internal/api/grpc/management/idp_converter_test.go | 2 -- internal/api/grpc/server/middleware/auth_interceptor_test.go | 2 -- internal/api/grpc/server/middleware/error_interceptor_test.go | 2 -- .../api/grpc/server/middleware/instance_interceptor_test.go | 2 -- internal/api/grpc/server/probes_test.go | 2 -- internal/api/grpc/system/failed_event_converter_test.go | 2 -- 11 files changed, 22 deletions(-) diff --git a/internal/api/grpc/admin/failed_event_converter_test.go b/internal/api/grpc/admin/failed_event_converter_test.go index cb6c6bddab..6edc36c1e2 100644 --- a/internal/api/grpc/admin/failed_event_converter_test.go +++ b/internal/api/grpc/admin/failed_event_converter_test.go @@ -1,5 +1,3 @@ -//go:build !integration - package admin import ( diff --git a/internal/api/grpc/admin/iam_member_converter_test.go b/internal/api/grpc/admin/iam_member_converter_test.go index f2b90031db..74dd329ee1 100644 --- a/internal/api/grpc/admin/iam_member_converter_test.go +++ b/internal/api/grpc/admin/iam_member_converter_test.go @@ -1,5 +1,3 @@ -//go:build !integration - package admin import ( diff --git a/internal/api/grpc/admin/idp_converter_test.go b/internal/api/grpc/admin/idp_converter_test.go index 0eb04efcb1..23d378b888 100644 --- a/internal/api/grpc/admin/idp_converter_test.go +++ b/internal/api/grpc/admin/idp_converter_test.go @@ -1,5 +1,3 @@ -//go:build !integration - package admin import ( diff --git a/internal/api/grpc/errors/caos_errors_test.go b/internal/api/grpc/errors/caos_errors_test.go index 6efe37b588..3a14f1bbfc 100644 --- a/internal/api/grpc/errors/caos_errors_test.go +++ b/internal/api/grpc/errors/caos_errors_test.go @@ -1,5 +1,3 @@ -//go:build !integration - package errors import ( diff --git a/internal/api/grpc/header_test.go b/internal/api/grpc/header_test.go index 085473328e..e304900073 100644 --- a/internal/api/grpc/header_test.go +++ b/internal/api/grpc/header_test.go @@ -1,5 +1,3 @@ -//go:build !integration - package grpc import ( diff --git a/internal/api/grpc/management/idp_converter_test.go b/internal/api/grpc/management/idp_converter_test.go index ae30b09c89..999be2ea06 100644 --- a/internal/api/grpc/management/idp_converter_test.go +++ b/internal/api/grpc/management/idp_converter_test.go @@ -1,5 +1,3 @@ -//go:build !integration - package management import ( diff --git a/internal/api/grpc/server/middleware/auth_interceptor_test.go b/internal/api/grpc/server/middleware/auth_interceptor_test.go index d51de2ff7a..abe50606f9 100644 --- a/internal/api/grpc/server/middleware/auth_interceptor_test.go +++ b/internal/api/grpc/server/middleware/auth_interceptor_test.go @@ -1,5 +1,3 @@ -//go:build !integration - package middleware import ( diff --git a/internal/api/grpc/server/middleware/error_interceptor_test.go b/internal/api/grpc/server/middleware/error_interceptor_test.go index 3ce69e0d34..82dc35f64a 100644 --- a/internal/api/grpc/server/middleware/error_interceptor_test.go +++ b/internal/api/grpc/server/middleware/error_interceptor_test.go @@ -1,5 +1,3 @@ -//go:build !integration - package middleware import ( diff --git a/internal/api/grpc/server/middleware/instance_interceptor_test.go b/internal/api/grpc/server/middleware/instance_interceptor_test.go index 194fafbdde..513dfe96cc 100644 --- a/internal/api/grpc/server/middleware/instance_interceptor_test.go +++ b/internal/api/grpc/server/middleware/instance_interceptor_test.go @@ -1,5 +1,3 @@ -//go:build !integration - package middleware import ( diff --git a/internal/api/grpc/server/probes_test.go b/internal/api/grpc/server/probes_test.go index 1d743178c8..15c889e24c 100644 --- a/internal/api/grpc/server/probes_test.go +++ b/internal/api/grpc/server/probes_test.go @@ -1,5 +1,3 @@ -//go:build !integration - package server import ( diff --git a/internal/api/grpc/system/failed_event_converter_test.go b/internal/api/grpc/system/failed_event_converter_test.go index 27caeff918..f91c2d1b4f 100644 --- a/internal/api/grpc/system/failed_event_converter_test.go +++ b/internal/api/grpc/system/failed_event_converter_test.go @@ -1,5 +1,3 @@ -//go:build !integration - package system_test import ( From e1655c3fbb59101abdb395d6a6eef2bbfb1302ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Wed, 26 Apr 2023 20:03:40 +0300 Subject: [PATCH 08/16] Revert "temporarily remove other workflow for fast running" This reverts commit 68200dcbe96f5c05c00327f74aaced4dd9eae849. --- .github/workflows/codeql-analysis.yml | 41 ++++++++++++ .github/workflows/e2e.yml | 66 +++++++++++++++++++ .github/workflows/issues.yml | 20 ++++++ .github/workflows/release-channels.yml | 52 +++++++++++++++ .github/workflows/test-code.yml | 75 ++++++++++++++++++++++ .github/workflows/test-docs.yml | 21 ++++++ .github/workflows/zitadel.yml | 88 ++++++++++++++++++++++++++ 7 files changed, 363 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/e2e.yml create mode 100644 .github/workflows/issues.yml create mode 100644 .github/workflows/release-channels.yml create mode 100644 .github/workflows/test-code.yml create mode 100644 .github/workflows/test-docs.yml create mode 100644 .github/workflows/zitadel.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000000..6ca453c32c --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,41 @@ +name: "Code scanning - action" + +on: + push: + branches: + - 'main' + paths-ignore: + - 'docs/**' + pull_request: + # The branches below must be a subset of the branches above + branches: + - 'main' + paths-ignore: + - 'docs/**' + +jobs: + CodeQL-Build: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + # Override language selection by uncommenting this and choosing your languages + with: + languages: go, javascript + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + #- run: | + # make bootstrap + # make release + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000000..50f1ce85c6 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,66 @@ +name: "ZITADEL e2e Tests" + +on: + workflow_run: + workflows: [ZITADEL Release] + types: + - completed + workflow_dispatch: + inputs: + releaseversion: + description: 'Release version to test' + required: true + default: 'latest' + +jobs: + test: + strategy: + matrix: + browser: [firefox, chrome] + runs-on: ubuntu-20.04 + env: + ZITADEL_IMAGE_REGISTRY: 'ghcr.io/zitadel/zitadel' + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + - name: Set TAG env manual trigger + if: github.event_name == 'workflow_dispatch' + run: echo "ZITADEL_IMAGE=${ZITADEL_IMAGE_REGISTRY}:${{ github.event.inputs.releaseversion }}" >> $GITHUB_ENV + - name: get latest tag + uses: actions-ecosystem/action-get-latest-tag@v1 + id: get-latest-tag + with: + semver_only: true + - name: Set TAG env on ZITADEL release + if: github.event_name == 'workflow_run' + run: echo "ZITADEL_IMAGE=${ZITADEL_IMAGE_REGISTRY}:${{ steps.get-latest-tag.outputs.tag }}" >> $GITHUB_ENV + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + driver: docker + install: true + - name: Test ${{ matrix.browser }} + run: docker compose run --service-ports e2e --browser ${{ matrix.browser }} + working-directory: e2e/config/host.docker.internal + - name: Ensure Artifacts Directory Exists + run: mkdir -p ./.artifacts + - name: Save ZITADEL Logs + if: always() + run: docker compose logs zitadel > ../../../.artifacts/e2e-compose-zitadel.log + working-directory: e2e/config/host.docker.internal + - name: Save Prepare Logs + if: always() + run: docker compose logs prepare > ../../../.artifacts/e2e-compose-prepare.log + working-directory: e2e/config/host.docker.internal + - name: Archive production tests ${{ matrix.browser }} + if: always() + uses: actions/upload-artifact@v3 + with: + name: production-tests-${{ matrix.browser }} + path: | + e2e/cypress/results + e2e/cypress/videos + e2e/cypress/screenshots + .artifacts/e2e-compose-zitadel.log + .artifacts/e2e-compose-prepare.log + retention-days: 30 diff --git a/.github/workflows/issues.yml b/.github/workflows/issues.yml new file mode 100644 index 0000000000..cc350f8e76 --- /dev/null +++ b/.github/workflows/issues.yml @@ -0,0 +1,20 @@ +name: Add new issues to product management project + +on: + issues: + types: + - opened + +jobs: + add-to-project: + name: Add issue to project + runs-on: ubuntu-latest + steps: + - uses: actions/add-to-project@v0.3.0 + with: + # You can target a repository in a different organization + # to the issue + project-url: https://github.com/orgs/zitadel/projects/2 + github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} + labeled: OKR + label-operator: NOT diff --git a/.github/workflows/release-channels.yml b/.github/workflows/release-channels.yml new file mode 100644 index 0000000000..864a0a7778 --- /dev/null +++ b/.github/workflows/release-channels.yml @@ -0,0 +1,52 @@ +name: ZITADEL Release tags + +on: + push: + branches: + - "main" + paths: + - 'release-channels.yaml' + workflow_dispatch: + +permissions: + contents: write + packages: write + +jobs: + Build: + runs-on: ubuntu-20.04 + env: + DOCKER_BUILDKIT: 1 + steps: + - name: Source checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: get stable tag + run: echo STABLE_RELEASE=$(yq eval '.stable' release-channels.yaml) >> $GITHUB_ENV + - name: checkout stable tag + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ env.STABLE_RELEASE }} + - name: GitHub Container Registry Login + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: tibdex/github-app-token@v1 + id: generate-token + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + - name: Google Artifact Registry Login + uses: docker/login-action@v2 + with: + registry: europe-docker.pkg.dev + username: _json_key_base64 + password: ${{ secrets.GCR_JSON_KEY_BASE64 }} + - name: copy release to stable + run: | + skopeo --version + skopeo copy --all docker://ghcr.io/zitadel/zitadel:$STABLE_RELEASE docker://ghcr.io/zitadel/zitadel:stable diff --git a/.github/workflows/test-code.yml b/.github/workflows/test-code.yml new file mode 100644 index 0000000000..7a4793dbe0 --- /dev/null +++ b/.github/workflows/test-code.yml @@ -0,0 +1,75 @@ +name: ZITADEL PR + +on: + pull_request: + paths-ignore: + - 'docs/**' + - 'guides/**' + - '**.md' + - 'release-channels.yaml' + +jobs: + Build-ZITADEL: + runs-on: ubuntu-20.04 + env: + DOCKER_BUILDKIT: 1 + steps: + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.19 + - name: Source checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + driver: docker + install: true + - name: Install GoReleaser + uses: goreleaser/goreleaser-action@v3 + with: + install-only: true + version: v1.10.3 + - name: Build and Unit Test + run: GOOS="linux" GOARCH="amd64" goreleaser build --id prod --snapshot --single-target --rm-dist --output .artifacts/zitadel/zitadel + - name: linting + uses: golangci/golangci-lint-action@v3 + with: + version: v1.52 + only-new-issues: true + skip-pkg-cache: true + - name: Publish go coverage + uses: codecov/codecov-action@v3.1.0 + with: + file: .artifacts/codecov/profile.cov + name: go-codecov + # As goreleaser doesn't build a dockerfile in snapshot mode, we have to build it here + - name: Build Docker Image + run: docker build -t zitadel:pr --file build/Dockerfile .artifacts/zitadel + - name: Run E2E Tests + run: docker compose run --service-ports e2e --browser chrome + working-directory: e2e/config/host.docker.internal + env: + ZITADEL_IMAGE: zitadel:pr + - name: Save ZITADEL Logs + if: always() + run: docker compose logs zitadel > ../../../.artifacts/e2e-compose-zitadel.log + working-directory: e2e/config/host.docker.internal + - name: Save Prepare Logs + if: always() + run: docker compose logs prepare > ../../../.artifacts/e2e-compose-prepare.log + working-directory: e2e/config/host.docker.internal + - name: Archive Test Results + if: always() + uses: actions/upload-artifact@v3 + with: + name: pull-request-tests + path: | + e2e/cypress/results + e2e/cypress/videos + e2e/cypress/screenshots + .artifacts/e2e-compose-zitadel.log + .artifacts/e2e-compose-prepare.log + retention-days: 30 diff --git a/.github/workflows/test-docs.yml b/.github/workflows/test-docs.yml new file mode 100644 index 0000000000..7bf3b222b0 --- /dev/null +++ b/.github/workflows/test-docs.yml @@ -0,0 +1,21 @@ +# ATTENTION: Although this workflow doesn't do much, it is still important. +# It is complementary to the workflow in the file test-code.yml. +# It enables to exclude files for the workflow and still mark the Test job as required without having pending PRs. +# GitHub recommends this solution here: +# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks + +name: ZITADEL PR + +on: + pull_request: + paths: + - 'docs/**' + - 'guides/**' + - '**.md' + - 'release-channels.yaml' + +jobs: + Build-ZITADEL: + runs-on: ubuntu-20.04 + steps: + - run: 'echo "No tests for docs are implemented, yet"' diff --git a/.github/workflows/zitadel.yml b/.github/workflows/zitadel.yml new file mode 100644 index 0000000000..2691109f80 --- /dev/null +++ b/.github/workflows/zitadel.yml @@ -0,0 +1,88 @@ +name: ZITADEL Release + +on: + push: + tags-ignore: + - "*" + workflow_dispatch: + +permissions: + contents: write + packages: write + +jobs: + Build: + runs-on: ubuntu-20.04 + env: + DOCKER_BUILDKIT: 1 + steps: + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.19 + - name: Source checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Fetch all tags + run: git fetch --force --tags + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + driver: docker + install: true + - name: Tag + id: semantic + uses: cycjimmy/semantic-release-action@v2 + with: + dry_run: false + semantic_version: 19.0.2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: GitHub Container Registry Login + if: steps.semantic.outputs.new_release_published == 'true' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: tibdex/github-app-token@v1 + id: generate-token + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + - name: Google Artifact Registry Login + if: steps.semantic.outputs.new_release_published == 'true' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' + uses: docker/login-action@v2 + with: + registry: europe-docker.pkg.dev + username: _json_key_base64 + password: ${{ secrets.GCR_JSON_KEY_BASE64 }} + - uses: goreleaser/goreleaser-action@v3 + name: Publish ZITADEL + if: steps.semantic.outputs.new_release_published == 'true' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' + with: + distribution: goreleaser + version: v1.11.0 + args: release --timeout 50m + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GORELEASER_TOKEN_TAP: ${{ steps.generate-token.outputs.token }} + RELEASE_VERSION: ${{ steps.semantic.outputs.release-version }} # I think this line is not needed. Nevertheless, it's explicit + DISCORD_WEBHOOK_ID: "976058224484687932" + DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" + - name: Publish go coverage + uses: codecov/codecov-action@v3.1.0 + with: + file: .artifacts/codecov/profile.cov + name: go-codecov + - name: Bump Chart Version + uses: peter-evans/repository-dispatch@v2 + if: steps.semantic.outputs.new_release_published == 'true' && github.ref == 'refs/heads/main' + with: + token: ${{ steps.generate-token.outputs.token }} + repository: zitadel/zitadel-charts + event-type: zitadel-released + client-payload: '{"semanticoutputs": "${{ steps.semantic.outputs }}"}' From ec8f741e58f4031d12d599010ffe97ff3a8874a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Wed, 26 Apr 2023 20:12:58 +0300 Subject: [PATCH 09/16] coverage reports --- .github/workflows/integration.yml | 11 ++++++++++- .github/workflows/test-code.yml | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 90edd89bfa..e334cb9d06 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,5 +1,9 @@ +name: Integration tests + on: push: + tags-ignore: + - "*" jobs: integration-tests: @@ -35,4 +39,9 @@ jobs: go run main.go init --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml go run main.go setup --masterkey MasterkeyNeedsToHave32Characters --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml - name: Run integration tests - run: go test -tags=integration -parallel 1 -v ./internal/integration ./internal/api/grpc/... + run: go test -tags=integration -parallel 1 -v -coverprofile=profile.cov -coverpkg=./pkg/... ./internal/integration ./internal/api/grpc/... + - name: Publish go coverage + uses: codecov/codecov-action@v3.1.0 + with: + file: profile.cov + name: integration-tests diff --git a/.github/workflows/test-code.yml b/.github/workflows/test-code.yml index 7a4793dbe0..b681d3b2da 100644 --- a/.github/workflows/test-code.yml +++ b/.github/workflows/test-code.yml @@ -44,7 +44,7 @@ jobs: uses: codecov/codecov-action@v3.1.0 with: file: .artifacts/codecov/profile.cov - name: go-codecov + name: unit-tests # As goreleaser doesn't build a dockerfile in snapshot mode, we have to build it here - name: Build Docker Image run: docker build -t zitadel:pr --file build/Dockerfile .artifacts/zitadel From 03789d43e55612c31c5c00a2e14c5d5b73fac817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Wed, 26 Apr 2023 20:29:43 +0300 Subject: [PATCH 10/16] re-enable PR build --- .github/workflows/integration.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e334cb9d06..4d273d29b3 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -3,7 +3,10 @@ name: Integration tests on: push: tags-ignore: - - "*" + - '**' + pull_request: + branches: + - '**' jobs: integration-tests: From 596900aba6b39d00c790cc9f883c74690f718933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Wed, 26 Apr 2023 20:49:33 +0300 Subject: [PATCH 11/16] nolint contextcheck in NewTester --- internal/integration/integration.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/integration/integration.go b/internal/integration/integration.go index 0b8fa21792..f594bb743b 100644 --- a/internal/integration/integration.go +++ b/internal/integration/integration.go @@ -95,6 +95,7 @@ func NewTester(ctx context.Context) *Tester { args := strings.Split(commandLine, " ") sc := make(chan *start.Server) + //nolint:contextcheck cmd := cmd.New(os.Stdout, os.Stdin, args, sc) cmd.SetArgs(args) err := viper.MergeConfig(bytes.NewBuffer(zitadelYAML)) From bd3820cc6c40c1c2fd40bfc21a55d1752e11c64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Thu, 27 Apr 2023 12:03:57 +0300 Subject: [PATCH 12/16] correct the coverpkg path --- .github/workflows/integration.yml | 2 +- build/zitadel/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 4d273d29b3..619379e1e0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -42,7 +42,7 @@ jobs: go run main.go init --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml go run main.go setup --masterkey MasterkeyNeedsToHave32Characters --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml - name: Run integration tests - run: go test -tags=integration -parallel 1 -v -coverprofile=profile.cov -coverpkg=./pkg/... ./internal/integration ./internal/api/grpc/... + run: go test -tags=integration -race -parallel 1 -v -coverprofile=profile.cov -coverpkg=./... ./internal/integration ./internal/api/grpc/... - name: Publish go coverage uses: codecov/codecov-action@v3.1.0 with: diff --git a/build/zitadel/Dockerfile b/build/zitadel/Dockerfile index 1ed0e34d7b..5de3dd652e 100644 --- a/build/zitadel/Dockerfile +++ b/build/zitadel/Dockerfile @@ -98,7 +98,7 @@ RUN rm -r cockroach-${COCKROACH_VERSION}.linux-amd64 # Migrations for cockroach-secure RUN go install github.com/rakyll/statik \ - && go test -race -v -coverprofile=profile.cov $(go list ./... | grep -v /operator/) + && go test -race -v -coverprofile=profile.cov -coverpkg=./... $(go list ./... | grep -v /operator/) ####################### ## Go test results From 4b7f5ae186b43317f343a2548d1041e1378e909e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Fri, 28 Apr 2023 14:39:53 +0300 Subject: [PATCH 13/16] AddHumanUser tests --- build/zitadel/Dockerfile | 2 +- .../admin/information_integration_test.go | 2 +- .../api/grpc/user/v2/user_integration_test.go | 317 ++++++++++++++++++ internal/integration/assert.go | 25 ++ internal/integration/config/zitadel.yaml | 1 - internal/integration/integration.go | 94 ++++++ internal/integration/integration_test.go | 3 +- internal/integration/usertype_string.go | 24 ++ 8 files changed, 463 insertions(+), 5 deletions(-) create mode 100644 internal/api/grpc/user/v2/user_integration_test.go create mode 100644 internal/integration/assert.go create mode 100644 internal/integration/usertype_string.go diff --git a/build/zitadel/Dockerfile b/build/zitadel/Dockerfile index ba2c3cccde..7c0f7bdcad 100644 --- a/build/zitadel/Dockerfile +++ b/build/zitadel/Dockerfile @@ -97,7 +97,7 @@ RUN rm -r cockroach-${COCKROACH_VERSION}.linux-amd64 # Migrations for cockroach-secure RUN go install github.com/rakyll/statik \ - && go test -race -v -coverprofile=profile.cov -coverpkg=./... $(go list ./... | grep -v /operator/) + && go test -race -v -coverprofile=profile.cov $(go list ./... | grep -v /operator/) ####################### ## Go test results diff --git a/internal/api/grpc/admin/information_integration_test.go b/internal/api/grpc/admin/information_integration_test.go index cac2c952aa..8eb9818241 100644 --- a/internal/api/grpc/admin/information_integration_test.go +++ b/internal/api/grpc/admin/information_integration_test.go @@ -20,7 +20,7 @@ var ( func TestMain(m *testing.M) { os.Exit(func() int { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + ctx, _, cancel := integration.Contexts(time.Minute) defer cancel() Tester = integration.NewTester(ctx) defer Tester.Done() diff --git a/internal/api/grpc/user/v2/user_integration_test.go b/internal/api/grpc/user/v2/user_integration_test.go new file mode 100644 index 0000000000..157aa00fb2 --- /dev/null +++ b/internal/api/grpc/user/v2/user_integration_test.go @@ -0,0 +1,317 @@ +//go:build integration + +package user_test + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/muhlemmer/gu" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zitadel/zitadel/internal/integration" + object "github.com/zitadel/zitadel/pkg/grpc/object/v2alpha" + user "github.com/zitadel/zitadel/pkg/grpc/user/v2alpha" + "google.golang.org/protobuf/types/known/timestamppb" +) + +var ( + CTX context.Context + ErrCTX context.Context + Tester *integration.Tester + Client user.UserServiceClient +) + +func TestMain(m *testing.M) { + os.Exit(func() int { + ctx, errCtx, cancel := integration.Contexts(time.Hour) + defer cancel() + + Tester = integration.NewTester(ctx) + defer Tester.Done() + + CTX, ErrCTX = Tester.WithSystemAuthorization(ctx, integration.OrgOwner), errCtx + Client = user.NewUserServiceClient(Tester.GRPCClientConn) + return m.Run() + }()) +} + +func TestServer_AddHumanUser(t *testing.T) { + type args struct { + ctx context.Context + req *user.AddHumanUserRequest + } + tests := []struct { + name string + args args + want *user.AddHumanUserResponse + wantErr bool + }{ + { + name: "default verification", + args: args{ + CTX, + &user.AddHumanUserRequest{ + Organisation: &object.Organisation{ + Org: &object.Organisation_OrgId{ + OrgId: "211137963315232910", + }, + }, + Profile: &user.SetHumanProfile{ + FirstName: "Donald", + LastName: "Duck", + NickName: gu.Ptr("Dukkie"), + DisplayName: gu.Ptr("Donald Duck"), + PreferredLanguage: gu.Ptr("en"), + Gender: user.Gender_GENDER_DIVERSE.Enum(), + }, + Email: &user.SetHumanEmail{}, + Metadata: []*user.SetMetadataEntry{ + { + Key: "somekey", + Value: []byte("somevalue"), + }, + }, + PasswordType: &user.AddHumanUserRequest_Password{ + Password: &user.Password{ + Password: "DifficultPW666!", + ChangeRequired: true, + }, + }, + }, + }, + want: &user.AddHumanUserResponse{ + Details: &object.Details{ + ChangeDate: timestamppb.Now(), + ResourceOwner: Tester.Organisation.ID, + }, + }, + }, + { + name: "return verification code", + args: args{ + CTX, + &user.AddHumanUserRequest{ + Organisation: &object.Organisation{ + Org: &object.Organisation_OrgId{ + OrgId: "211137963315232910", + }, + }, + Profile: &user.SetHumanProfile{ + FirstName: "Donald", + LastName: "Duck", + NickName: gu.Ptr("Dukkie"), + DisplayName: gu.Ptr("Donald Duck"), + PreferredLanguage: gu.Ptr("en"), + Gender: user.Gender_GENDER_DIVERSE.Enum(), + }, + Email: &user.SetHumanEmail{ + Verification: &user.SetHumanEmail_ReturnCode{ + ReturnCode: &user.ReturnEmailVerificationCode{}, + }, + }, + Metadata: []*user.SetMetadataEntry{ + { + Key: "somekey", + Value: []byte("somevalue"), + }, + }, + PasswordType: &user.AddHumanUserRequest_Password{ + Password: &user.Password{ + Password: "DifficultPW666!", + ChangeRequired: true, + }, + }, + }, + }, + want: &user.AddHumanUserResponse{ + Details: &object.Details{ + ChangeDate: timestamppb.Now(), + ResourceOwner: Tester.Organisation.ID, + }, + EmailCode: gu.Ptr("something"), + }, + }, + { + name: "custom template", + args: args{ + CTX, + &user.AddHumanUserRequest{ + Organisation: &object.Organisation{ + Org: &object.Organisation_OrgId{ + OrgId: "211137963315232910", + }, + }, + Profile: &user.SetHumanProfile{ + FirstName: "Donald", + LastName: "Duck", + NickName: gu.Ptr("Dukkie"), + DisplayName: gu.Ptr("Donald Duck"), + PreferredLanguage: gu.Ptr("en"), + Gender: user.Gender_GENDER_DIVERSE.Enum(), + }, + Email: &user.SetHumanEmail{ + Verification: &user.SetHumanEmail_SendCode{ + SendCode: &user.SendEmailVerificationCode{ + UrlTemplate: gu.Ptr("https://example.com/email/verify?userID={{.UserID}}&code={{.Code}}&orgID={{.OrgID}}"), + }, + }, + }, + Metadata: []*user.SetMetadataEntry{ + { + Key: "somekey", + Value: []byte("somevalue"), + }, + }, + PasswordType: &user.AddHumanUserRequest_Password{ + Password: &user.Password{ + Password: "DifficultPW666!", + ChangeRequired: true, + }, + }, + }, + }, + want: &user.AddHumanUserResponse{ + Details: &object.Details{ + ChangeDate: timestamppb.Now(), + ResourceOwner: Tester.Organisation.ID, + }, + }, + }, + { + name: "custom template error", + args: args{ + CTX, + &user.AddHumanUserRequest{ + Organisation: &object.Organisation{ + Org: &object.Organisation_OrgId{ + OrgId: "211137963315232910", + }, + }, + Profile: &user.SetHumanProfile{ + FirstName: "Donald", + LastName: "Duck", + NickName: gu.Ptr("Dukkie"), + DisplayName: gu.Ptr("Donald Duck"), + PreferredLanguage: gu.Ptr("en"), + Gender: user.Gender_GENDER_DIVERSE.Enum(), + }, + Email: &user.SetHumanEmail{ + Verification: &user.SetHumanEmail_SendCode{ + SendCode: &user.SendEmailVerificationCode{ + UrlTemplate: gu.Ptr("{{"), + }, + }, + }, + Metadata: []*user.SetMetadataEntry{ + { + Key: "somekey", + Value: []byte("somevalue"), + }, + }, + PasswordType: &user.AddHumanUserRequest_Password{ + Password: &user.Password{ + Password: "DifficultPW666!", + ChangeRequired: true, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "missing REQUIRED profile", + args: args{ + CTX, + &user.AddHumanUserRequest{ + Organisation: &object.Organisation{ + Org: &object.Organisation_OrgId{ + OrgId: "211137963315232910", + }, + }, + Email: &user.SetHumanEmail{ + Verification: &user.SetHumanEmail_ReturnCode{ + ReturnCode: &user.ReturnEmailVerificationCode{}, + }, + }, + Metadata: []*user.SetMetadataEntry{ + { + Key: "somekey", + Value: []byte("somevalue"), + }, + }, + PasswordType: &user.AddHumanUserRequest_Password{ + Password: &user.Password{ + Password: "DifficultPW666!", + ChangeRequired: true, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "missing REQUIRED email", + args: args{ + CTX, + &user.AddHumanUserRequest{ + Organisation: &object.Organisation{ + Org: &object.Organisation_OrgId{ + OrgId: "211137963315232910", + }, + }, + Profile: &user.SetHumanProfile{ + FirstName: "Donald", + LastName: "Duck", + NickName: gu.Ptr("Dukkie"), + DisplayName: gu.Ptr("Donald Duck"), + PreferredLanguage: gu.Ptr("en"), + Gender: user.Gender_GENDER_DIVERSE.Enum(), + }, + Metadata: []*user.SetMetadataEntry{ + { + Key: "somekey", + Value: []byte("somevalue"), + }, + }, + PasswordType: &user.AddHumanUserRequest_Password{ + Password: &user.Password{ + Password: "DifficultPW666!", + ChangeRequired: true, + }, + }, + }, + }, + wantErr: true, + }, + } + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + userID := fmt.Sprint(time.Now().UnixNano() + int64(i)) + tt.args.req.UserId = &userID + if email := tt.args.req.GetEmail(); email != nil { + email.Email = fmt.Sprintf("%s@me.now", userID) + } + + if tt.want != nil { + tt.want.UserId = userID + } + + got, err := Client.AddHumanUser(tt.args.ctx, tt.args.req) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + assert.Equal(t, tt.want.GetUserId(), got.GetUserId()) + if tt.want.GetEmailCode() != "" { + assert.NotEmpty(t, got.GetEmailCode()) + } + integration.AssertDetails(t, tt.want, got) + }) + } +} diff --git a/internal/integration/assert.go b/internal/integration/assert.go new file mode 100644 index 0000000000..512c006ad9 --- /dev/null +++ b/internal/integration/assert.go @@ -0,0 +1,25 @@ +package integration + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + object "github.com/zitadel/zitadel/pkg/grpc/object/v2alpha" +) + +type DetailsMsg interface { + GetDetails() *object.Details +} + +func AssertDetails[D DetailsMsg](t testing.TB, exptected, actual D) { + wantDetails, gotDetails := exptected.GetDetails(), actual.GetDetails() + + if wantDetails != nil { + assert.NotZero(t, gotDetails.GetSequence()) + } + wantCD, gotCD := wantDetails.GetChangeDate().AsTime(), gotDetails.GetChangeDate().AsTime() + assert.WithinRange(t, gotCD, wantCD, wantCD.Add(time.Minute)) + assert.Equal(t, wantDetails.GetResourceOwner(), gotDetails.GetResourceOwner()) +} diff --git a/internal/integration/config/zitadel.yaml b/internal/integration/config/zitadel.yaml index de4eb8aa1d..60389d6de0 100644 --- a/internal/integration/config/zitadel.yaml +++ b/internal/integration/config/zitadel.yaml @@ -35,4 +35,3 @@ Projections: DefaultInstance: LoginPolicy: MfaInitSkipLifetime: "0" - diff --git a/internal/integration/integration.go b/internal/integration/integration.go index f594bb743b..545066f056 100644 --- a/internal/integration/integration.go +++ b/internal/integration/integration.go @@ -4,7 +4,9 @@ package integration import ( "bytes" "context" + "database/sql" _ "embed" + "errors" "fmt" "os" "strings" @@ -13,11 +15,20 @@ import ( "github.com/spf13/viper" "github.com/zitadel/logging" + "github.com/zitadel/oidc/v2/pkg/oidc" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" "github.com/zitadel/zitadel/cmd" "github.com/zitadel/zitadel/cmd/start" + "github.com/zitadel/zitadel/internal/api/authz" + z_oidc "github.com/zitadel/zitadel/internal/api/oidc" + "github.com/zitadel/zitadel/internal/command" + "github.com/zitadel/zitadel/internal/domain" + caos_errs "github.com/zitadel/zitadel/internal/errors" + "github.com/zitadel/zitadel/internal/eventstore/v1/models" + "github.com/zitadel/zitadel/internal/query" "github.com/zitadel/zitadel/pkg/grpc/admin" ) @@ -30,8 +41,26 @@ var ( postgresYAML []byte ) +type UserType int + +//go:generate stringer -type=UserType +const ( + Unspecified UserType = iota + OrgOwner +) + +type User struct { + *query.User + Token string +} + type Tester struct { *start.Server + + Instance authz.Instance + Organisation *query.Org + Users map[UserType]User + GRPCClientConn *grpc.ClientConn wg sync.WaitGroup // used for shutdown } @@ -83,6 +112,63 @@ func (s *Tester) pollHealth(ctx context.Context) (err error) { } } +const ( + SystemUser = "integration1" +) + +func (s *Tester) createSystemUser(ctx context.Context) { + var err error + + s.Instance, err = s.Queries.InstanceByHost(ctx, "localhost:8080") + logging.OnError(err).Fatal("query instance") + ctx = authz.WithInstance(ctx, s.Instance) + + s.Organisation, err = s.Queries.OrgByID(ctx, true, s.Instance.DefaultOrganisationID()) + logging.OnError(err).Fatal("query organisation") + + query, err := query.NewUserUsernameSearchQuery(SystemUser, query.TextEquals) + logging.OnError(err).Fatal("user query") + user, err := s.Queries.GetUser(ctx, true, true, query) + + if errors.Is(err, sql.ErrNoRows) { + _, err = s.Commands.AddMachine(ctx, &command.Machine{ + ObjectRoot: models.ObjectRoot{ + ResourceOwner: s.Organisation.ID, + }, + Username: SystemUser, + Name: SystemUser, + Description: "who cares?", + AccessTokenType: domain.OIDCTokenTypeJWT, + }) + logging.OnError(err).Fatal("add machine user") + user, err = s.Queries.GetUser(ctx, true, true, query) + + } + logging.OnError(err).Fatal("get user") + + _, err = s.Commands.AddOrgMember(ctx, s.Organisation.ID, user.ID, "ORG_OWNER") + target := new(caos_errs.AlreadyExistsError) + if !errors.As(err, &target) { + logging.OnError(err).Fatal("add org member") + } + + scopes := []string{oidc.ScopeOpenID, z_oidc.ScopeUserMetaData, z_oidc.ScopeResourceOwner} + pat := command.NewPersonalAccessToken(user.ResourceOwner, user.ID, time.Now().Add(time.Hour), scopes, domain.UserTypeMachine) + _, err = s.Commands.AddPersonalAccessToken(ctx, pat) + logging.OnError(err).Fatal("add pat") + + s.Users = map[UserType]User{ + OrgOwner: { + User: user, + Token: pat.Token, + }, + } +} + +func (s *Tester) WithSystemAuthorization(ctx context.Context, u UserType) context.Context { + return metadata.AppendToOutgoingContext(ctx, "Authorization", fmt.Sprintf("Bearer %s", s.Users[u].Token)) +} + func (s *Tester) Done() { err := s.GRPCClientConn.Close() logging.OnError(err).Error("integration tester client close") @@ -125,6 +211,14 @@ func NewTester(ctx context.Context) *Tester { logging.OnError(ctx.Err()).Fatal("waiting for integration tester server") } tester.createClientConn(ctx) + tester.createSystemUser(ctx) return tester } + +func Contexts(timeout time.Duration) (ctx, errCtx context.Context, cancel context.CancelFunc) { + errCtx, cancel = context.WithCancel(context.Background()) + cancel() + ctx, cancel = context.WithTimeout(context.Background(), timeout) + return ctx, errCtx, cancel +} diff --git a/internal/integration/integration_test.go b/internal/integration/integration_test.go index a01d8cc7ff..416602ea25 100644 --- a/internal/integration/integration_test.go +++ b/internal/integration/integration_test.go @@ -3,13 +3,12 @@ package integration import ( - "context" "testing" "time" ) func TestNewTester(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + ctx, _, cancel := Contexts(time.Hour) defer cancel() s := NewTester(ctx) diff --git a/internal/integration/usertype_string.go b/internal/integration/usertype_string.go new file mode 100644 index 0000000000..3f5db98d72 --- /dev/null +++ b/internal/integration/usertype_string.go @@ -0,0 +1,24 @@ +// Code generated by "stringer -type=UserType"; DO NOT EDIT. + +package integration + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[Unspecified-0] + _ = x[OrgOwner-1] +} + +const _UserType_name = "UnspecifiedOrgOwner" + +var _UserType_index = [...]uint8{0, 11, 19} + +func (i UserType) String() string { + if i < 0 || i >= UserType(len(_UserType_index)-1) { + return "UserType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _UserType_name[_UserType_index[i]:_UserType_index[i+1]] +} From 498c4436ae7a02e53c8e7fb297edc3370a81aedc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Fri, 28 Apr 2023 17:44:59 +0300 Subject: [PATCH 14/16] inegration tests for user email --- internal/api/grpc/user/v2/email.go | 6 +- .../grpc/user/v2/email_integration_test.go | 210 ++++++++++++++++++ 2 files changed, 211 insertions(+), 5 deletions(-) create mode 100644 internal/api/grpc/user/v2/email_integration_test.go diff --git a/internal/api/grpc/user/v2/email.go b/internal/api/grpc/user/v2/email.go index 95928e2e61..d1c91333e0 100644 --- a/internal/api/grpc/user/v2/email.go +++ b/internal/api/grpc/user/v2/email.go @@ -21,11 +21,7 @@ func (s *Server) SetEmail(ctx context.Context, req *user.SetEmailRequest) (resp case *user.SetEmailRequest_ReturnCode: email, err = s.command.ChangeUserEmailReturnCode(ctx, req.GetUserId(), resourceOwner, req.GetEmail(), s.userCodeAlg) case *user.SetEmailRequest_IsVerified: - if v.IsVerified { - email, err = s.command.ChangeUserEmailVerified(ctx, req.GetUserId(), resourceOwner, req.GetEmail()) - } else { - email, err = s.command.ChangeUserEmail(ctx, req.GetUserId(), resourceOwner, req.GetEmail(), s.userCodeAlg) - } + email, err = s.command.ChangeUserEmailVerified(ctx, req.GetUserId(), resourceOwner, req.GetEmail()) case nil: email, err = s.command.ChangeUserEmail(ctx, req.GetUserId(), resourceOwner, req.GetEmail(), s.userCodeAlg) default: diff --git a/internal/api/grpc/user/v2/email_integration_test.go b/internal/api/grpc/user/v2/email_integration_test.go new file mode 100644 index 0000000000..91c4806b29 --- /dev/null +++ b/internal/api/grpc/user/v2/email_integration_test.go @@ -0,0 +1,210 @@ +//go:build integration + +package user_test + +import ( + "fmt" + "testing" + "time" + + "github.com/muhlemmer/gu" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zitadel/zitadel/internal/integration" + object "github.com/zitadel/zitadel/pkg/grpc/object/v2alpha" + user "github.com/zitadel/zitadel/pkg/grpc/user/v2alpha" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func createHumanUser(t *testing.T) *user.AddHumanUserResponse { + resp, err := Client.AddHumanUser(CTX, &user.AddHumanUserRequest{ + Organisation: &object.Organisation{ + Org: &object.Organisation_OrgId{ + OrgId: Tester.Organisation.ID, + }, + }, + Profile: &user.SetHumanProfile{ + FirstName: "Mickey", + LastName: "Mouse", + }, + Email: &user.SetHumanEmail{ + Email: fmt.Sprintf("%d@mouse.com", time.Now().UnixNano()), + Verification: &user.SetHumanEmail_ReturnCode{ + ReturnCode: &user.ReturnEmailVerificationCode{}, + }, + }, + }) + require.NoError(t, err) + require.NotEmpty(t, resp.GetUserId()) + return resp +} + +func TestServer_SetEmail(t *testing.T) { + userID := createHumanUser(t).GetUserId() + + tests := []struct { + name string + req *user.SetEmailRequest + want *user.SetEmailResponse + wantErr bool + }{ + { + name: "default verfication", + req: &user.SetEmailRequest{ + UserId: userID, + Email: "default-verifier@mouse.com", + }, + want: &user.SetEmailResponse{ + Details: &object.Details{ + Sequence: 1, + ChangeDate: timestamppb.Now(), + ResourceOwner: Tester.Organisation.ID, + }, + }, + }, + { + name: "custom url template", + req: &user.SetEmailRequest{ + UserId: userID, + Email: "custom-url@mouse.com", + Verification: &user.SetEmailRequest_SendCode{ + SendCode: &user.SendEmailVerificationCode{ + UrlTemplate: gu.Ptr("https://example.com/email/verify?userID={{.UserID}}&code={{.Code}}&orgID={{.OrgID}}"), + }, + }, + }, + want: &user.SetEmailResponse{ + Details: &object.Details{ + Sequence: 1, + ChangeDate: timestamppb.Now(), + ResourceOwner: Tester.Organisation.ID, + }, + }, + }, + { + name: "template error", + req: &user.SetEmailRequest{ + UserId: userID, + Email: "custom-url@mouse.com", + Verification: &user.SetEmailRequest_SendCode{ + SendCode: &user.SendEmailVerificationCode{ + UrlTemplate: gu.Ptr("{{"), + }, + }, + }, + wantErr: true, + }, + { + name: "return code", + req: &user.SetEmailRequest{ + UserId: userID, + Email: "return-code@mouse.com", + Verification: &user.SetEmailRequest_ReturnCode{ + ReturnCode: &user.ReturnEmailVerificationCode{}, + }, + }, + want: &user.SetEmailResponse{ + Details: &object.Details{ + Sequence: 1, + ChangeDate: timestamppb.Now(), + ResourceOwner: Tester.Organisation.ID, + }, + VerificationCode: gu.Ptr("xxx"), + }, + }, + { + name: "is verified true", + req: &user.SetEmailRequest{ + UserId: userID, + Email: "verified-true@mouse.com", + Verification: &user.SetEmailRequest_IsVerified{ + IsVerified: true, + }, + }, + want: &user.SetEmailResponse{ + Details: &object.Details{ + Sequence: 1, + ChangeDate: timestamppb.Now(), + ResourceOwner: Tester.Organisation.ID, + }, + }, + }, + { + name: "is verified false", + req: &user.SetEmailRequest{ + UserId: userID, + Email: "verified-false@mouse.com", + Verification: &user.SetEmailRequest_IsVerified{ + IsVerified: false, + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Client.SetEmail(CTX, tt.req) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + integration.AssertDetails(t, tt.want, got) + if tt.want.GetVerificationCode() != "" { + assert.NotEmpty(t, got.GetVerificationCode()) + } + }) + } +} + +func TestServer_VerifyEmail(t *testing.T) { + userResp := createHumanUser(t) + tests := []struct { + name string + req *user.VerifyEmailRequest + want *user.VerifyEmailResponse + wantErr bool + }{ + { + name: "wrong code", + req: &user.VerifyEmailRequest{ + UserId: userResp.GetUserId(), + VerificationCode: "xxx", + }, + wantErr: true, + }, + { + name: "wrong user", + req: &user.VerifyEmailRequest{ + UserId: "xxx", + VerificationCode: userResp.GetEmailCode(), + }, + wantErr: true, + }, + { + name: "verify user", + req: &user.VerifyEmailRequest{ + UserId: userResp.GetUserId(), + VerificationCode: userResp.GetEmailCode(), + }, + want: &user.VerifyEmailResponse{ + Details: &object.Details{ + Sequence: 1, + ChangeDate: timestamppb.Now(), + ResourceOwner: Tester.Organisation.ID, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Client.VerifyEmail(CTX, tt.req) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + integration.AssertDetails(t, tt.want, got) + }) + } +} From c839cb3ce02a1687d72f439d1e6a63c56c06f3b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Tue, 2 May 2023 19:24:24 +0300 Subject: [PATCH 15/16] tie loose ends, documentation --- .github/workflows/integration.yml | 4 +- CONTRIBUTING.md | 15 ++++++ cmd/start/start.go | 14 ++++- .../api/grpc/user/v2/user_integration_test.go | 12 ++--- internal/integration/assert.go | 26 ++++++++-- internal/integration/assert_test.go | 51 +++++++++++++++++++ .../integration/config}/docker-compose.yaml | 2 +- internal/integration/integration.go | 24 ++++++++- 8 files changed, 132 insertions(+), 16 deletions(-) create mode 100644 internal/integration/assert_test.go rename {e2e/config/integration => internal/integration/config}/docker-compose.yaml (87%) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 619379e1e0..f3d360b18e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -9,7 +9,7 @@ on: - '**' jobs: - integration-tests: + run: strategy: matrix: db: [cockroach, postgres] @@ -36,7 +36,7 @@ jobs: - name: Download Go modules run: go mod download - name: Start ${{ matrix.db }} database - run: docker compose -f e2e/config/integration/docker-compose.yaml up --wait ${INTEGRATION_DB_FLAVOR} + run: docker compose -f internal/integration/config/docker-compose.yaml up --wait ${INTEGRATION_DB_FLAVOR} - name: Run zitadel init and setup run: | go run main.go init --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 60936beb7d..3e47e7e4d0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -199,6 +199,21 @@ When you are happy with your changes, you can cleanup your environment. docker compose --file ./e2e/config/host.docker.internal/docker-compose.yaml down ``` +#### Integration tests + +In order to run the integrations tests for the gRPC API, PostgreSQL and CockroachDB must be running and initialized. + +```bash +export INTEGRATION_DB_FLAVOR="cockroach" +docker compose -f internal/integration/config/docker-compose.yaml up --wait ${INTEGRATION_DB_FLAVOR} +go run main.go init --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml +go run main.go setup --masterkey MasterkeyNeedsToHave32Characters --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml +go test -tags=integration -race -parallel 1 ./internal/integration ./internal/api/grpc/... +docker compose -f internal/integration/config/docker-compose.yaml down +``` + +The above can be repeated with `INTEGRATION_DB_FLAVOR="postgres"`. + ### Console By executing the commands from this section, you run everything you need to develop the console locally. diff --git a/cmd/start/start.go b/cmd/start/start.go index e51c7e4d69..bda2cc9abe 100644 --- a/cmd/start/start.go +++ b/cmd/start/start.go @@ -21,6 +21,7 @@ import ( "github.com/zitadel/saml/pkg/provider" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" + "golang.org/x/sys/unix" "github.com/zitadel/zitadel/cmd/key" cmd_tls "github.com/zitadel/zitadel/cmd/tls" @@ -341,10 +342,21 @@ func startAPIs( return nil } +func reusePort(network, address string, conn syscall.RawConn) error { + return conn.Control(func(descriptor uintptr) { + err := syscall.SetsockoptInt(int(descriptor), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1) + if err != nil { + panic(err) + } + }) +} + func listen(ctx context.Context, router *mux.Router, port uint16, tlsConfig *tls.Config, shutdown <-chan os.Signal) error { http2Server := &http2.Server{} http1Server := &http.Server{Handler: h2c.NewHandler(router, http2Server), TLSConfig: tlsConfig} - lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + + lc := &net.ListenConfig{Control: reusePort} + lis, err := lc.Listen(ctx, "tcp", fmt.Sprintf(":%d", port)) if err != nil { return fmt.Errorf("tcp listener on %d failed: %w", port, err) } diff --git a/internal/api/grpc/user/v2/user_integration_test.go b/internal/api/grpc/user/v2/user_integration_test.go index 157aa00fb2..c389ee073a 100644 --- a/internal/api/grpc/user/v2/user_integration_test.go +++ b/internal/api/grpc/user/v2/user_integration_test.go @@ -57,7 +57,7 @@ func TestServer_AddHumanUser(t *testing.T) { &user.AddHumanUserRequest{ Organisation: &object.Organisation{ Org: &object.Organisation_OrgId{ - OrgId: "211137963315232910", + OrgId: Tester.Organisation.ID, }, }, Profile: &user.SetHumanProfile{ @@ -97,7 +97,7 @@ func TestServer_AddHumanUser(t *testing.T) { &user.AddHumanUserRequest{ Organisation: &object.Organisation{ Org: &object.Organisation_OrgId{ - OrgId: "211137963315232910", + OrgId: Tester.Organisation.ID, }, }, Profile: &user.SetHumanProfile{ @@ -142,7 +142,7 @@ func TestServer_AddHumanUser(t *testing.T) { &user.AddHumanUserRequest{ Organisation: &object.Organisation{ Org: &object.Organisation_OrgId{ - OrgId: "211137963315232910", + OrgId: Tester.Organisation.ID, }, }, Profile: &user.SetHumanProfile{ @@ -188,7 +188,7 @@ func TestServer_AddHumanUser(t *testing.T) { &user.AddHumanUserRequest{ Organisation: &object.Organisation{ Org: &object.Organisation_OrgId{ - OrgId: "211137963315232910", + OrgId: Tester.Organisation.ID, }, }, Profile: &user.SetHumanProfile{ @@ -229,7 +229,7 @@ func TestServer_AddHumanUser(t *testing.T) { &user.AddHumanUserRequest{ Organisation: &object.Organisation{ Org: &object.Organisation_OrgId{ - OrgId: "211137963315232910", + OrgId: Tester.Organisation.ID, }, }, Email: &user.SetHumanEmail{ @@ -260,7 +260,7 @@ func TestServer_AddHumanUser(t *testing.T) { &user.AddHumanUserRequest{ Organisation: &object.Organisation{ Org: &object.Organisation_OrgId{ - OrgId: "211137963315232910", + OrgId: Tester.Organisation.ID, }, }, Profile: &user.SetHumanProfile{ diff --git a/internal/integration/assert.go b/internal/integration/assert.go index 512c006ad9..3361392987 100644 --- a/internal/integration/assert.go +++ b/internal/integration/assert.go @@ -13,13 +13,29 @@ type DetailsMsg interface { GetDetails() *object.Details } +// AssertDetails asserts values in a message's object Details, +// if the object Details in expected is a non-nil value. +// It targets API v2 messages that have the `GetDetails()` method. +// +// Dynamically generated values are not compared with expected. +// Instead a sanity check is performed. +// For the sequence a non-zero value is expected. +// The change date has to be now, with a tollerance of 1 second. +// +// The resource owner is compared with expected and is +// therefore the only value that has to be set. func AssertDetails[D DetailsMsg](t testing.TB, exptected, actual D) { wantDetails, gotDetails := exptected.GetDetails(), actual.GetDetails() - - if wantDetails != nil { - assert.NotZero(t, gotDetails.GetSequence()) + if wantDetails == nil { + assert.Nil(t, gotDetails) + return } - wantCD, gotCD := wantDetails.GetChangeDate().AsTime(), gotDetails.GetChangeDate().AsTime() - assert.WithinRange(t, gotCD, wantCD, wantCD.Add(time.Minute)) + + assert.NotZero(t, gotDetails.GetSequence()) + + gotCD := gotDetails.GetChangeDate().AsTime() + now := time.Now() + assert.WithinRange(t, gotCD, now.Add(-time.Second), now.Add(time.Second)) + assert.Equal(t, wantDetails.GetResourceOwner(), gotDetails.GetResourceOwner()) } diff --git a/internal/integration/assert_test.go b/internal/integration/assert_test.go new file mode 100644 index 0000000000..49a6949007 --- /dev/null +++ b/internal/integration/assert_test.go @@ -0,0 +1,51 @@ +package integration + +import ( + "testing" + + "google.golang.org/protobuf/types/known/timestamppb" + + object "github.com/zitadel/zitadel/pkg/grpc/object/v2alpha" +) + +type myMsg struct { + details *object.Details +} + +func (m myMsg) GetDetails() *object.Details { + return m.details +} + +func TestAssertDetails(t *testing.T) { + tests := []struct { + name string + exptected myMsg + actual myMsg + }{ + { + name: "nil", + exptected: myMsg{}, + actual: myMsg{}, + }, + { + name: "values", + exptected: myMsg{ + details: &object.Details{ + ResourceOwner: "me", + }, + }, + actual: myMsg{ + details: &object.Details{ + Sequence: 123, + ChangeDate: timestamppb.Now(), + ResourceOwner: "me", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + AssertDetails(t, tt.exptected, tt.actual) + }) + } +} diff --git a/e2e/config/integration/docker-compose.yaml b/internal/integration/config/docker-compose.yaml similarity index 87% rename from e2e/config/integration/docker-compose.yaml rename to internal/integration/config/docker-compose.yaml index 85d68e4a2a..a43cecf335 100644 --- a/e2e/config/integration/docker-compose.yaml +++ b/internal/integration/config/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.8' services: cockroach: extends: - file: '../localhost/docker-compose.yaml' + file: '../../../e2e/config/localhost/docker-compose.yaml' service: 'db' postgres: diff --git a/internal/integration/integration.go b/internal/integration/integration.go index 545066f056..39873f3533 100644 --- a/internal/integration/integration.go +++ b/internal/integration/integration.go @@ -41,6 +41,11 @@ var ( postgresYAML []byte ) +// UserType provides constants that give +// a short explinanation with the purpose +// a serverice user. +// This allows to pre-create users with +// different permissions and reuse them. type UserType int //go:generate stringer -type=UserType @@ -49,11 +54,13 @@ const ( OrgOwner ) +// User information with a Personal Access Token. type User struct { *query.User Token string } +// Tester is a Zitadel server and client with all resources available for testing. type Tester struct { *start.Server @@ -113,7 +120,7 @@ func (s *Tester) pollHealth(ctx context.Context) (err error) { } const ( - SystemUser = "integration1" + SystemUser = "integration" ) func (s *Tester) createSystemUser(ctx context.Context) { @@ -169,6 +176,7 @@ func (s *Tester) WithSystemAuthorization(ctx context.Context, u UserType) contex return metadata.AppendToOutgoingContext(ctx, "Authorization", fmt.Sprintf("Bearer %s", s.Users[u].Token)) } +// Done send an interrupt signal to cleanly shutdown the server. func (s *Tester) Done() { err := s.GRPCClientConn.Close() logging.OnError(err).Error("integration tester client close") @@ -177,6 +185,20 @@ func (s *Tester) Done() { s.wg.Wait() } +// NewTester start a new Zitadel server by passing the default commandline. +// The server will listen on the configured port. +// The database configuration that will be used can be set by the +// INTEGRATION_DB_FLAVOR environment variable and can have the values "cockroach" +// or "postgres". Defaults to "cockroach". +// +// The deault Instance and Organisation are read from the DB and system +// users are created as needed. +// +// After the server is started, a [grpc.ClientConn] will be created and +// the server is polled for it's health status. +// +// Note: the database must already be setup and intialized before +// using NewTester. See the CONTRIBUTING.md document for details. func NewTester(ctx context.Context) *Tester { args := strings.Split(commandLine, " ") From 5f0c1b52902dbffd9b6f945cff1f6e7e88b681a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Wed, 3 May 2023 15:10:27 +0200 Subject: [PATCH 16/16] resolve comments --- .github/workflows/integration.yml | 5 +++-- CONTRIBUTING.md | 6 +++--- internal/integration/integration.go | 10 +++++++--- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f3d360b18e..8f7ba48a11 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -16,7 +16,8 @@ jobs: runs-on: ubuntu-20.04 env: DOCKER_BUILDKIT: 1 - INTEGRATION_DB_FLAVOR: ${{ matrix.db }} + INTEGRATION_DB_FLAVOR: ${{ matrix.db }} + ZITADEL_MASTERKEY: MasterkeyNeedsToHave32Characters steps: - name: Set up Go uses: actions/setup-go@v3 @@ -40,7 +41,7 @@ jobs: - name: Run zitadel init and setup run: | go run main.go init --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml - go run main.go setup --masterkey MasterkeyNeedsToHave32Characters --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml + go run main.go setup --masterkeyFromEnv --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml - name: Run integration tests run: go test -tags=integration -race -parallel 1 -v -coverprofile=profile.cov -coverpkg=./... ./internal/integration ./internal/api/grpc/... - name: Publish go coverage diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3e47e7e4d0..b17d42c0ab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -201,13 +201,13 @@ docker compose --file ./e2e/config/host.docker.internal/docker-compose.yaml down #### Integration tests -In order to run the integrations tests for the gRPC API, PostgreSQL and CockroachDB must be running and initialized. +In order to run the integrations tests for the gRPC API, PostgreSQL and CockroachDB must be started and initialized: ```bash -export INTEGRATION_DB_FLAVOR="cockroach" +export INTEGRATION_DB_FLAVOR="cockroach" ZITADEL_MASTERKEY="MasterkeyNeedsToHave32Characters" docker compose -f internal/integration/config/docker-compose.yaml up --wait ${INTEGRATION_DB_FLAVOR} go run main.go init --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml -go run main.go setup --masterkey MasterkeyNeedsToHave32Characters --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml +go run main.go setup --masterkeyFromEnv --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml go test -tags=integration -race -parallel 1 ./internal/integration ./internal/api/grpc/... docker compose -f internal/integration/config/docker-compose.yaml down ``` diff --git a/internal/integration/integration.go b/internal/integration/integration.go index 39873f3533..67ef9d60b2 100644 --- a/internal/integration/integration.go +++ b/internal/integration/integration.go @@ -72,10 +72,14 @@ type Tester struct { wg sync.WaitGroup // used for shutdown } -const commandLine = `start --masterkey MasterkeyNeedsToHave32Characters` +const commandLine = `start --masterkeyFromEnv` + +func (s *Tester) Host() string { + return fmt.Sprintf("%s:%d", s.Config.ExternalDomain, s.Config.Port) +} func (s *Tester) createClientConn(ctx context.Context) { - target := fmt.Sprintf("localhost:%d", s.Config.Port) + target := s.Host() cc, err := grpc.DialContext(ctx, target, grpc.WithBlock(), grpc.WithTransportCredentials(insecure.NewCredentials()), ) @@ -126,7 +130,7 @@ const ( func (s *Tester) createSystemUser(ctx context.Context) { var err error - s.Instance, err = s.Queries.InstanceByHost(ctx, "localhost:8080") + s.Instance, err = s.Queries.InstanceByHost(ctx, s.Host()) logging.OnError(err).Fatal("query instance") ctx = authz.WithInstance(ctx, s.Instance)