diff --git a/cmd/admin/initialise/config.go b/cmd/admin/initialise/config.go index 9519b15ead..18e843c410 100644 --- a/cmd/admin/initialise/config.go +++ b/cmd/admin/initialise/config.go @@ -4,11 +4,13 @@ import ( "github.com/spf13/viper" "github.com/zitadel/logging" "github.com/zitadel/zitadel/internal/database" + "github.com/zitadel/zitadel/internal/id" ) type Config struct { Database database.Config AdminUser database.User + Machine *id.Config Log *logging.Config } diff --git a/cmd/admin/initialise/init.go b/cmd/admin/initialise/init.go index 385e2c759f..04cf47e0bc 100644 --- a/cmd/admin/initialise/init.go +++ b/cmd/admin/initialise/init.go @@ -9,6 +9,7 @@ import ( "github.com/zitadel/logging" "github.com/zitadel/zitadel/internal/database" + "github.com/zitadel/zitadel/internal/id" ) func New() *cobra.Command { @@ -20,7 +21,7 @@ func New() *cobra.Command { Prereqesits: - cockroachdb -The user provided by flags needs priviledge to +The user provided by flags needs privileges to - create the database if it does not exist - see other users and create a new one if the user does not exist - grant all rights of the ZITADEL database to the user created if not yet set @@ -37,6 +38,7 @@ The user provided by flags needs priviledge to } func InitAll(config *Config) { + id.Configure(config.Machine) err := initialise(config, VerifyUser(config.Database.Username, config.Database.Password), VerifyDatabase(config.Database.Database), diff --git a/cmd/admin/initialise/verify_zitadel.go b/cmd/admin/initialise/verify_zitadel.go index f7b610e72a..82a61409ec 100644 --- a/cmd/admin/initialise/verify_zitadel.go +++ b/cmd/admin/initialise/verify_zitadel.go @@ -46,8 +46,8 @@ var ( func newZitadel() *cobra.Command { return &cobra.Command{ Use: "zitadel", - Short: "initialize ZITADEL internas", - Long: `initialize ZITADEL internas. + Short: "initialize ZITADEL internals", + Long: `initialize ZITADEL internals. Prereqesits: - cockroachdb with user and database diff --git a/cmd/admin/start/start.go b/cmd/admin/start/start.go index 3943389995..99d5cd7838 100644 --- a/cmd/admin/start/start.go +++ b/cmd/admin/start/start.go @@ -175,9 +175,9 @@ func startAPIs(ctx context.Context, router *mux.Router, commands *command.Comman } instanceInterceptor := middleware.InstanceInterceptor(queries, config.HTTP1HostHeader, login.IgnoreInstanceEndpoints...) - authenticatedAPIs.RegisterHandler(assets.HandlerPrefix, assets.NewHandler(commands, verifier, config.InternalAuthZ, id.SonyFlakeGenerator, store, queries, instanceInterceptor.Handler)) + authenticatedAPIs.RegisterHandler(assets.HandlerPrefix, assets.NewHandler(commands, verifier, config.InternalAuthZ, id.SonyFlakeGenerator(), store, queries, instanceInterceptor.Handler)) - userAgentInterceptor, err := middleware.NewUserAgentHandler(config.UserAgentCookie, keys.UserAgentCookieKey, id.SonyFlakeGenerator, config.ExternalSecure, login.EndpointResources) + userAgentInterceptor, err := middleware.NewUserAgentHandler(config.UserAgentCookie, keys.UserAgentCookieKey, id.SonyFlakeGenerator(), config.ExternalSecure, login.EndpointResources) if err != nil { return err } diff --git a/cmd/defaults.yaml b/cmd/defaults.yaml index c2ed2ba105..2e57c810cd 100644 --- a/cmd/defaults.yaml +++ b/cmd/defaults.yaml @@ -38,6 +38,39 @@ AdminUser: Cert: "" Key: "" +Machine: + # Cloud hosted VMs need to specify their metadata endpoint so that the machine can be uniquely identified. + Identification: + # Use private IP to identify machines uniquely + PrivateIp: + Enabled: true + # Use hostname to identify machines uniquely + # You want the process to be identified uniquely, so this works well in k8s where each pod gets its own + # unique host name, but not as well in some other hosting environments. + Hostname: + Enabled: false + # Use a webhook response to identify machines uniquely + # Google Cloud Configuration + Webhook: + Enabled: true + Url: "http://metadata.google.internal/computeMetadata/v1/instance/id" + Headers: + "Metadata-Flavor": "Google" + # + # AWS EC2 IMDSv1 Configuration: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html + # Webhook: + # Url: "http://169.254.169.254/latest/meta-data/ami-id" + # + # AWS ECS v4 Configuration: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v4.html + # Webhook: + # Url: "${ECS_CONTAINER_METADATA_URI_V4}" + # JPath: "$.DockerId" + # + # Azure Configuration: https://docs.microsoft.com/en-us/azure/virtual-machines/windows/instance-metadata-service?tabs=linux + # Webhook: + # Url: "http://169.254.169.254/metadata/instance?api-version=2021-02-01" + # JPath: "$.compute.vmId" + Projections: RequeueEvery: 10s RetryFailedAfter: 1s diff --git a/go.mod b/go.mod index 60c0bd82e9..50a15d2d58 100644 --- a/go.mod +++ b/go.mod @@ -89,6 +89,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect + github.com/drone/envsubst v1.0.3 // indirect github.com/dsoprea/go-exif v0.0.0-20210131231135-d154f10435cc // indirect github.com/dsoprea/go-exif/v2 v2.0.0-20200604193436-ca8584a0e1c4 // indirect github.com/dsoprea/go-iptc v0.0.0-20200609062250-162ae6b44feb // indirect @@ -124,6 +125,7 @@ require ( github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jarcoal/jpath v0.0.0-20140328210829-f76b8b2dbf52 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/go-types v0.0.0-20210723172823-2deba1f80ba7 // indirect diff --git a/go.sum b/go.sum index 8bae227c67..8615bbdbd5 100644 --- a/go.sum +++ b/go.sum @@ -198,6 +198,8 @@ github.com/dop251/goja v0.0.0-20211129110639-4739a1d10a51/go.mod h1:R9ET47fwRVRP github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d h1:W1n4DvpzZGOISgp7wWNtraLcHtnmnTwBlJidqtMIuwQ= github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= +github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= +github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= github.com/dsoprea/go-exif v0.0.0-20210131231135-d154f10435cc h1:AuzYp98IFVOi0NU/WcZyGDQ6vAh/zkCjxGD3kt8aLzA= github.com/dsoprea/go-exif v0.0.0-20210131231135-d154f10435cc/go.mod h1:lOaOt7+UEppOgyvRy749v3do836U/hw0YVJNjoyPaEs= github.com/dsoprea/go-exif/v2 v2.0.0-20200321225314-640175a69fe4/go.mod h1:Lm2lMM2zx8p4a34ZemkaUV95AnMl4ZvLbCUbwOvLC2E= @@ -556,6 +558,8 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jarcoal/jpath v0.0.0-20140328210829-f76b8b2dbf52 h1:jny9eqYPwkG8IVy7foUoRjQmFLcArCSz+uPsL6KS0HQ= +github.com/jarcoal/jpath v0.0.0-20140328210829-f76b8b2dbf52/go.mod h1:RDZ+4PR3mDOtTpVbI0qBE+rdhmtIrtbssiNn38/1OWA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o= github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs= diff --git a/guides/development.md b/guides/development.md index da257f45fe..8a788b65c5 100644 --- a/guides/development.md +++ b/guides/development.md @@ -5,38 +5,40 @@ You should stay in the ZITADEL root directory to execute the statements in the f ## Prerequisite - Buildkit compatible docker installation +- [go](https://go.dev/doc/install) +- [goreleaser](https://goreleaser.com/install/) Minimum resources: - CPU's: 2 - Memory: 4Gb +### Installing goreleaser + +If you get the error `Failed to fetch https://repo.goreleaser.com/apt/Packages Certificate verification failed: The certificate is NOT trusted. The certificate chain uses expired certificate.` while installing goreleaser with `apt`, then ensure that ca-certificates are installed: + +```sh +sudo apt install ca-certificates +``` + ### env variables You can use the default vars provided in [this .env-file](../build/local/local.env) or create your own and update the paths in the [docker compose file](../build/local/docker-compose-local.yml). -## Generate required files +## Local Build -This part is relevant if you start the backend or console without docker compose. +Simply run goreleaser to build locally. This will generate all the required files, such as angular and grpc automatically. -### Console - -This command generates the grpc stub for console into the folder console/src/app/proto/generated for local development. - -```bash -DOCKER_BUILDKIT=1 docker build -f build/zitadel/Dockerfile . -t zitadel:gen-fe --target js-client -o . +```sh +goreleaser build --snapshot --rm-dist --single-target ``` -### Start the Backend +## Production Build & Release -With these commands you can generate the stub for the backend. +Simply use goreleaser: -```bash -# generates grpc stub -DOCKER_BUILDKIT=1 docker build -f build/zitadel/Dockerfile . -t zitadel:gen-be --target go-client -o . -# generates keys for cryptography -COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 \ -&& docker compose -f ./build/local/docker-compose-local.yml --profile backend-stub up --exit-code-from keys +```sh +goreleaser release ``` ## Run diff --git a/guides/production.md b/guides/production.md index 64336be091..5d4c879042 100644 --- a/guides/production.md +++ b/guides/production.md @@ -1,7 +1,13 @@ # Production Build -This can also be run locally! +To create a production build to run locally, create a snapshot release with goreleaser: -```bash -DOCKER_BUILDKIT=1 docker build -f build/dockerfile . -t zitadel:local --build-arg ENV=prod +```sh +goreleaser release --snapshot --rm-dist +``` + +This can be released to production (if you have credentials configured) using gorelease as well: + +```sh +goreleaser release ``` diff --git a/internal/auth/repository/eventsourcing/repository.go b/internal/auth/repository/eventsourcing/repository.go index 006c693376..87d82a9ee6 100644 --- a/internal/auth/repository/eventsourcing/repository.go +++ b/internal/auth/repository/eventsourcing/repository.go @@ -38,7 +38,7 @@ func Start(conf Config, systemDefaults sd.SystemDefaults, command *command.Comma if err != nil { return nil, err } - idGenerator := id.SonyFlakeGenerator + idGenerator := id.SonyFlakeGenerator() view, err := auth_view.StartView(dbClient, oidcEncryption, queries, idGenerator, assetsPrefix) if err != nil { diff --git a/internal/authz/repository/eventsourcing/repository.go b/internal/authz/repository/eventsourcing/repository.go index b5c3113d16..6488184a63 100644 --- a/internal/authz/repository/eventsourcing/repository.go +++ b/internal/authz/repository/eventsourcing/repository.go @@ -32,7 +32,7 @@ func Start(conf Config, systemDefaults sd.SystemDefaults, queries *query.Queries return nil, err } - idGenerator := id.SonyFlakeGenerator + idGenerator := id.SonyFlakeGenerator() view, err := authz_view.StartView(dbClient, idGenerator, queries) if err != nil { return nil, err diff --git a/internal/command/command.go b/internal/command/command.go index 25df7c033b..983f5114ff 100644 --- a/internal/command/command.go +++ b/internal/command/command.go @@ -72,7 +72,7 @@ func StartCommands(es *eventstore.Eventstore, repo = &Commands{ eventstore: es, static: staticStore, - idGenerator: id.SonyFlakeGenerator, + idGenerator: id.SonyFlakeGenerator(), zitadelRoles: zitadelRoles, externalDomain: externalDomain, externalSecure: externalSecure, diff --git a/internal/command/instance.go b/internal/command/instance.go index ad2fd7cf13..03e17d9870 100644 --- a/internal/command/instance.go +++ b/internal/command/instance.go @@ -33,6 +33,7 @@ const ( type InstanceSetup struct { zitadel ZitadelConfig + idGenerator id.Generator InstanceName string CustomDomain string DefaultLanguage language.Tag @@ -113,28 +114,28 @@ type ZitadelConfig struct { consoleAppID string } -func (s *InstanceSetup) generateIDs() (err error) { - s.zitadel.projectID, err = id.SonyFlakeGenerator.Next() +func (s *InstanceSetup) generateIDs(idGenerator id.Generator) (err error) { + s.zitadel.projectID, err = idGenerator.Next() if err != nil { return err } - s.zitadel.mgmtAppID, err = id.SonyFlakeGenerator.Next() + s.zitadel.mgmtAppID, err = idGenerator.Next() if err != nil { return err } - s.zitadel.adminAppID, err = id.SonyFlakeGenerator.Next() + s.zitadel.adminAppID, err = idGenerator.Next() if err != nil { return err } - s.zitadel.authAppID, err = id.SonyFlakeGenerator.Next() + s.zitadel.authAppID, err = idGenerator.Next() if err != nil { return err } - s.zitadel.consoleAppID, err = id.SonyFlakeGenerator.Next() + s.zitadel.consoleAppID, err = idGenerator.Next() if err != nil { return err } @@ -142,7 +143,7 @@ func (s *InstanceSetup) generateIDs() (err error) { } func (c *Commands) SetUpInstance(ctx context.Context, setup *InstanceSetup) (string, *domain.ObjectDetails, error) { - instanceID, err := id.SonyFlakeGenerator.Next() + instanceID, err := c.idGenerator.Next() if err != nil { return "", nil, err } @@ -153,17 +154,17 @@ func (c *Commands) SetUpInstance(ctx context.Context, setup *InstanceSetup) (str ctx = authz.SetCtxData(authz.WithRequestedDomain(authz.WithInstanceID(ctx, instanceID), c.externalDomain), authz.CtxData{OrgID: instanceID, ResourceOwner: instanceID}) - orgID, err := id.SonyFlakeGenerator.Next() + orgID, err := c.idGenerator.Next() if err != nil { return "", nil, err } - userID, err := id.SonyFlakeGenerator.Next() + userID, err := c.idGenerator.Next() if err != nil { return "", nil, err } - if err = setup.generateIDs(); err != nil { + if err = setup.generateIDs(c.idGenerator); err != nil { return "", nil, err } ctx = authz.WithConsole(ctx, setup.zitadel.projectID, setup.zitadel.consoleAppID) @@ -292,7 +293,7 @@ func (c *Commands) SetUpInstance(ctx context.Context, setup *InstanceSetup) (str AddProjectCommand(projectAgg, zitadelProjectName, userID, false, false, false, domain.PrivateLabelingSettingUnspecified), SetIAMProject(instanceAgg, projectAgg.ID), - AddAPIAppCommand( + c.AddAPIAppCommand( &addAPIApp{ AddApp: AddApp{ Aggregate: *projectAgg, @@ -304,7 +305,7 @@ func (c *Commands) SetUpInstance(ctx context.Context, setup *InstanceSetup) (str nil, ), - AddAPIAppCommand( + c.AddAPIAppCommand( &addAPIApp{ AddApp: AddApp{ Aggregate: *projectAgg, @@ -316,7 +317,7 @@ func (c *Commands) SetUpInstance(ctx context.Context, setup *InstanceSetup) (str nil, ), - AddAPIAppCommand( + c.AddAPIAppCommand( &addAPIApp{ AddApp: AddApp{ Aggregate: *projectAgg, @@ -328,7 +329,7 @@ func (c *Commands) SetUpInstance(ctx context.Context, setup *InstanceSetup) (str nil, ), - AddOIDCAppCommand(console, nil), + c.AddOIDCAppCommand(console, nil), SetIAMConsoleID(instanceAgg, &console.ClientID, &setup.zitadel.consoleAppID), ) addGenerateddDomain, err := c.addGeneratedInstanceDomain(ctx, instanceAgg, setup.InstanceName) diff --git a/internal/command/org.go b/internal/command/org.go index 2d2a0282ca..4248b84cb5 100644 --- a/internal/command/org.go +++ b/internal/command/org.go @@ -11,7 +11,6 @@ import ( "github.com/zitadel/zitadel/internal/errors" caos_errs "github.com/zitadel/zitadel/internal/errors" "github.com/zitadel/zitadel/internal/eventstore" - "github.com/zitadel/zitadel/internal/id" "github.com/zitadel/zitadel/internal/repository/org" user_repo "github.com/zitadel/zitadel/internal/repository/user" ) @@ -23,12 +22,12 @@ type OrgSetup struct { } func (c *Commands) SetUpOrg(ctx context.Context, o *OrgSetup, userIDs ...string) (string, *domain.ObjectDetails, error) { - orgID, err := id.SonyFlakeGenerator.Next() + orgID, err := c.idGenerator.Next() if err != nil { return "", nil, err } - userID, err := id.SonyFlakeGenerator.Next() + userID, err := c.idGenerator.Next() if err != nil { return "", nil, err } diff --git a/internal/command/project_application_api.go b/internal/command/project_application_api.go index 969416d0e0..87a1dabec2 100644 --- a/internal/command/project_application_api.go +++ b/internal/command/project_application_api.go @@ -11,7 +11,6 @@ import ( "github.com/zitadel/zitadel/internal/domain" "github.com/zitadel/zitadel/internal/errors" "github.com/zitadel/zitadel/internal/eventstore" - "github.com/zitadel/zitadel/internal/id" project_repo "github.com/zitadel/zitadel/internal/repository/project" "github.com/zitadel/zitadel/internal/telemetry/tracing" ) @@ -25,7 +24,7 @@ type addAPIApp struct { ClientSecretPlain string } -func AddAPIAppCommand(app *addAPIApp, clientSecretAlg crypto.HashAlgorithm) preparation.Validation { +func (c *Commands) AddAPIAppCommand(app *addAPIApp, clientSecretAlg crypto.HashAlgorithm) preparation.Validation { return func() (preparation.CreateCommands, error) { if app.ID == "" { return nil, errors.ThrowInvalidArgument(nil, "PROJE-XHsKt", "Errors.Invalid.Argument") @@ -39,7 +38,7 @@ func AddAPIAppCommand(app *addAPIApp, clientSecretAlg crypto.HashAlgorithm) prep return nil, errors.ThrowNotFound(err, "PROJE-Sf2gb", "Errors.Project.NotFound") } - app.ClientID, err = domain.NewClientID(id.SonyFlakeGenerator, project.Name) + app.ClientID, err = domain.NewClientID(c.idGenerator, project.Name) if err != nil { return nil, errors.ThrowInternal(err, "V2-f0pgP", "Errors.Internal") } diff --git a/internal/command/project_application_api_test.go b/internal/command/project_application_api_test.go index 8af5e5a753..7db5a30bfc 100644 --- a/internal/command/project_application_api_test.go +++ b/internal/command/project_application_api_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/zitadel/zitadel/internal/command/preparation" "github.com/zitadel/zitadel/internal/crypto" "github.com/zitadel/zitadel/internal/domain" @@ -18,6 +19,9 @@ import ( ) func TestAddAPIConfig(t *testing.T) { + type fields struct { + idGenerator id.Generator + } type args struct { a *project.Aggregate appID string @@ -29,12 +33,14 @@ func TestAddAPIConfig(t *testing.T) { agg := project.NewAggregate("test", "test") tests := []struct { - name string - args args - want Want + name string + fields fields + args args + want Want }{ { - name: "invalid appID", + name: "invalid appID", + fields: fields{}, args: args{ a: agg, appID: "", @@ -45,7 +51,8 @@ func TestAddAPIConfig(t *testing.T) { }, }, { - name: "invalid name", + name: "invalid name", + fields: fields{}, args: args{ a: agg, appID: "appID", @@ -56,7 +63,8 @@ func TestAddAPIConfig(t *testing.T) { }, }, { - name: "project not exists", + name: "project not exists", + fields: fields{}, args: args{ a: agg, appID: "id", @@ -73,6 +81,9 @@ func TestAddAPIConfig(t *testing.T) { }, { name: "correct without client secret", + fields: fields{ + idGenerator: id_mock.NewIDGeneratorExpectIDs(t, "clientID"), + }, args: args{ a: agg, appID: "appID", @@ -103,7 +114,7 @@ func TestAddAPIConfig(t *testing.T) { ), project.NewAPIConfigAddedEvent(ctx, &agg.Aggregate, "appID", - "", + "clientID@project", nil, domain.APIAuthMethodTypePrivateKeyJWT, ), @@ -113,8 +124,11 @@ func TestAddAPIConfig(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + c := &Commands{ + idGenerator: tt.fields.idGenerator, + } AssertValidation(t, - AddAPIAppCommand( + c.AddAPIAppCommand( &addAPIApp{ AddApp: AddApp{ Aggregate: *tt.args.a, diff --git a/internal/command/project_application_oidc.go b/internal/command/project_application_oidc.go index 15ed080903..aabb310448 100644 --- a/internal/command/project_application_oidc.go +++ b/internal/command/project_application_oidc.go @@ -14,7 +14,6 @@ import ( "github.com/zitadel/zitadel/internal/errors" caos_errs "github.com/zitadel/zitadel/internal/errors" "github.com/zitadel/zitadel/internal/eventstore" - "github.com/zitadel/zitadel/internal/id" project_repo "github.com/zitadel/zitadel/internal/repository/project" "github.com/zitadel/zitadel/internal/telemetry/tracing" ) @@ -42,7 +41,7 @@ type addOIDCApp struct { } //AddOIDCAppCommand prepares the commands to add an oidc app. The ClientID will be set during the CreateCommands -func AddOIDCAppCommand(app *addOIDCApp, clientSecretAlg crypto.HashAlgorithm) preparation.Validation { +func (c *Commands) AddOIDCAppCommand(app *addOIDCApp, clientSecretAlg crypto.HashAlgorithm) preparation.Validation { return func() (preparation.CreateCommands, error) { if app.ID == "" { return nil, errors.ThrowInvalidArgument(nil, "PROJE-NnavI", "Errors.Invalid.Argument") @@ -72,7 +71,7 @@ func AddOIDCAppCommand(app *addOIDCApp, clientSecretAlg crypto.HashAlgorithm) pr return nil, errors.ThrowNotFound(err, "PROJE-6swVG", "Errors.Project.NotFound") } - app.ClientID, err = domain.NewClientID(id.SonyFlakeGenerator, project.Name) + app.ClientID, err = domain.NewClientID(c.idGenerator, project.Name) if err != nil { return nil, errors.ThrowInternal(err, "V2-VMSQ1", "Errors.Internal") } diff --git a/internal/command/project_application_oidc_test.go b/internal/command/project_application_oidc_test.go index 9e7d13ab4f..3028e84e88 100644 --- a/internal/command/project_application_oidc_test.go +++ b/internal/command/project_application_oidc_test.go @@ -20,6 +20,9 @@ import ( ) func TestAddOIDCApp(t *testing.T) { + type fields struct { + idGenerator id.Generator + } type args struct { app *addOIDCApp clientSecretAlg crypto.HashAlgorithm @@ -30,12 +33,14 @@ func TestAddOIDCApp(t *testing.T) { agg := project.NewAggregate("test", "test") tests := []struct { - name string - args args - want Want + name string + fields fields + args args + want Want }{ { - name: "invalid appID", + name: "invalid appID", + fields: fields{}, args: args{ app: &addOIDCApp{ AddApp: AddApp{ @@ -56,7 +61,8 @@ func TestAddOIDCApp(t *testing.T) { }, }, { - name: "invalid name", + name: "invalid name", + fields: fields{}, args: args{ app: &addOIDCApp{ AddApp: AddApp{ @@ -77,7 +83,8 @@ func TestAddOIDCApp(t *testing.T) { }, }, { - name: "project not exists", + name: "project not exists", + fields: fields{}, args: args{ app: &addOIDCApp{ AddApp: AddApp{ @@ -104,6 +111,9 @@ func TestAddOIDCApp(t *testing.T) { }, { name: "correct", + fields: fields{ + idGenerator: id_mock.NewIDGeneratorExpectIDs(t, "clientID"), + }, args: args{ app: &addOIDCApp{ AddApp: AddApp{ @@ -144,7 +154,7 @@ func TestAddOIDCApp(t *testing.T) { project.NewOIDCConfigAddedEvent(ctx, &agg.Aggregate, domain.OIDCVersionV1, "id", - "", + "clientID@project", nil, nil, []domain.OIDCResponseType{domain.OIDCResponseTypeCode}, @@ -166,8 +176,11 @@ func TestAddOIDCApp(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + c := Commands{ + idGenerator: tt.fields.idGenerator, + } AssertValidation(t, - AddOIDCAppCommand( + c.AddOIDCAppCommand( tt.args.app, tt.args.clientSecretAlg, ), tt.args.filter, tt.want) diff --git a/internal/eventstore/handler/crdb/handler_stmt_test.go b/internal/eventstore/handler/crdb/handler_stmt_test.go index 18bbb1aa64..a1ed5f01ba 100644 --- a/internal/eventstore/handler/crdb/handler_stmt_test.go +++ b/internal/eventstore/handler/crdb/handler_stmt_test.go @@ -15,6 +15,7 @@ import ( "github.com/zitadel/zitadel/internal/eventstore/handler" "github.com/zitadel/zitadel/internal/eventstore/repository" es_repo_mock "github.com/zitadel/zitadel/internal/eventstore/repository/mock" + "github.com/zitadel/zitadel/internal/id" ) var ( @@ -123,6 +124,7 @@ func TestProjectionHandler_SearchQuery(t *testing.T) { } defer client.Close() + id.Configure(&id.Config{Identification: id.Identification{PrivateIp: id.PrivateIp{Enabled: true}}}) h := NewStatementHandler(context.Background(), StatementHandlerConfig{ ProjectionHandlerConfig: handler.ProjectionHandlerConfig{ ProjectionName: tt.fields.projectionName, diff --git a/internal/eventstore/handler/crdb/lock.go b/internal/eventstore/handler/crdb/lock.go index ccd32f0494..cb8e2408cc 100644 --- a/internal/eventstore/handler/crdb/lock.go +++ b/internal/eventstore/handler/crdb/lock.go @@ -33,7 +33,7 @@ type locker struct { } func NewLocker(client *sql.DB, lockTable, projectionName string) Locker { - workerName, err := id.SonyFlakeGenerator.Next() + workerName, err := id.SonyFlakeGenerator().Next() logging.OnError(err).Panic("unable to generate lockID") return &locker{ client: client, diff --git a/internal/eventstore/v1/spooler/config.go b/internal/eventstore/v1/spooler/config.go index 7a9b4ee25f..eb96d9e619 100644 --- a/internal/eventstore/v1/spooler/config.go +++ b/internal/eventstore/v1/spooler/config.go @@ -18,7 +18,7 @@ type Config struct { } func (c *Config) New() *Spooler { - lockID, err := id.SonyFlakeGenerator.Next() + lockID, err := id.SonyFlakeGenerator().Next() logging.OnError(err).Panic("unable to generate lockID") //shuffle the handlers for better balance when running multiple pods diff --git a/internal/id/config.go b/internal/id/config.go new file mode 100644 index 0000000000..741276b089 --- /dev/null +++ b/internal/id/config.go @@ -0,0 +1,46 @@ +package id + +const ( + DefaultWebhookPath = "http://metadata.google.internal/computeMetadata/v1/instance/id" +) + +type Config struct { + // Configuration for the identification of machines. + Identification Identification +} + +type Identification struct { + // Configuration for using private IP to identify a machine. + PrivateIp PrivateIp + // Configuration for using hostname to identify a machine. + Hostname Hostname + // Configuration for using a webhook to identify a machine. + Webhook Webhook +} + +type PrivateIp struct { + // Try to use private IP when identifying the machine uniquely + Enabled bool +} + +type Hostname struct { + // Try to use hostname when identifying the machine uniquely + Enabled bool +} + +type Webhook struct { + // Try to use webhook when identifying the machine uniquely + Enabled bool + // The URL of the metadata endpoint to query + Url string + // (Optional) A JSONPath expression for the data to extract from the response from the metadata endpoint + JPath *string + // (Optional) Headers to pass in the metadata request + Headers *map[string]string +} + +func Configure(config *Config) { + if config != nil { + GeneratorConfig = config + } +} diff --git a/internal/id/sonyflake.go b/internal/id/sonyflake.go index 6fe768c236..7a5f773804 100644 --- a/internal/id/sonyflake.go +++ b/internal/id/sonyflake.go @@ -1,6 +1,7 @@ package id import ( + "encoding/json" "errors" "fmt" "hash/fnv" @@ -8,10 +9,12 @@ import ( "net" "net/http" "os" - "strconv" + "strings" "time" + "github.com/drone/envsubst" + "github.com/jarcoal/jpath" "github.com/sony/sonyflake" ) @@ -28,14 +31,25 @@ func (s *sonyflakeGenerator) Next() (string, error) { } var ( - SonyFlakeGenerator = Generator(&sonyflakeGenerator{ - sonyflake.NewSonyflake(sonyflake.Settings{ - MachineID: machineID, - StartTime: time.Date(2019, 4, 29, 0, 0, 0, 0, time.UTC), - }), - }) + GeneratorConfig *Config = nil + sonyFlakeGenerator *Generator = nil ) +func SonyFlakeGenerator() Generator { + if sonyFlakeGenerator == nil { + sfg := Generator(&sonyflakeGenerator{ + sonyflake.NewSonyflake(sonyflake.Settings{ + MachineID: machineID, + StartTime: time.Date(2019, 4, 29, 0, 0, 0, 0, time.UTC), + }), + }) + + sonyFlakeGenerator = &sfg + } + + return *sonyFlakeGenerator +} + // the following is a copy of sonyflake (https://github.com/sony/sonyflake/blob/master/sonyflake.go) //with the change of using the "POD-IP" if no private ip is found func privateIPv4() (net.IP, error) { @@ -73,16 +87,41 @@ func isPrivateIPv4(ip net.IP) bool { } func machineID() (uint16, error) { - ip, ipErr := lower16BitPrivateIP() - if ipErr == nil { - return ip, nil + if GeneratorConfig == nil { + return 0, errors.New("Cannot create a unique ID for the machine, generator has not been configured.") } - cid, cidErr := cloudRunContainerID() - if cidErr != nil { - return 0, fmt.Errorf("neighter found a private ip nor a cloud run container instance id: private ip err: %w, cloud run ip err: %s", ipErr, cidErr.Error()) + errors := []string{} + if GeneratorConfig.Identification.PrivateIp.Enabled { + ip, ipErr := lower16BitPrivateIP() + if ipErr == nil { + return ip, nil + } + errors = append(errors, fmt.Sprintf("failed to get Private IP address %s", ipErr)) } - return cid, nil + + if GeneratorConfig.Identification.Hostname.Enabled { + hn, hostErr := hostname() + if hostErr == nil { + return hn, nil + } + errors = append(errors, fmt.Sprintf("failed to get Hostname %s", hostErr)) + } + + if GeneratorConfig.Identification.Webhook.Enabled { + cid, cidErr := metadataWebhookID() + if cidErr == nil { + return cid, nil + } + + errors = append(errors, fmt.Sprintf("failed to query metadata webhook %s", cidErr)) + } + + if len(errors) == 0 { + errors = append(errors, "No machine identification method enabled.") + } + + return 0, fmt.Errorf("none of the enabled methods for identifying the machine succeeded: %s", strings.Join(errors, ". ")) } func lower16BitPrivateIP() (uint16, error) { @@ -94,16 +133,42 @@ func lower16BitPrivateIP() (uint16, error) { return uint16(ip[2])<<8 + uint16(ip[3]), nil } -func cloudRunContainerID() (uint16, error) { +func hostname() (uint16, error) { + host, err := os.Hostname() + if err != nil { + return 0, err + } + + h := fnv.New32() + _, hashErr := h.Write([]byte(host)) + if hashErr != nil { + return 0, hashErr + } + + return uint16(h.Sum32()), nil +} + +func metadataWebhookID() (uint16, error) { + webhook := GeneratorConfig.Identification.Webhook + url, err := envsubst.EvalEnv(webhook.Url) + if err != nil { + url = webhook.Url + } + req, err := http.NewRequest( http.MethodGet, - "http://metadata.google.internal/computeMetadata/v1/instance/id", + url, nil, ) if err != nil { return 0, err } - req.Header.Set("Metadata-Flavor", "Google") + + if webhook.Headers != nil { + for key, value := range *webhook.Headers { + req.Header.Set(key, value) + } + } resp, err := (&http.Client{}).Do(req) if err != nil { @@ -113,16 +178,39 @@ func cloudRunContainerID() (uint16, error) { defer resp.Body.Close() if resp.StatusCode >= 400 && resp.StatusCode < 600 { - return 0, fmt.Errorf("cloud metadata returned an unsuccessful status code %d", resp.StatusCode) + return 0, fmt.Errorf("metadata endpoint returned an unsuccessful status code %d", resp.StatusCode) } body, err := ioutil.ReadAll(resp.Body) if err != nil { return 0, err } + data, err := extractMetadataResponse(webhook.JPath, body) + if err != nil { + return 0, err + } + h := fnv.New32() - if _, err = h.Write(body); err != nil { + if _, err = h.Write(data); err != nil { return 0, err } return uint16(h.Sum32()), nil } + +func extractMetadataResponse(path *string, data []byte) ([]byte, error) { + if path != nil { + jp, err := jpath.NewFromBytes(data) + if err != nil { + return nil, err + } + + results := jp.Query(*path) + if len(results) == 0 { + return nil, fmt.Errorf("metadata endpoint response was successful, but JSONPath provided didn't match anything in the response: %s", string(data[:])) + } + + return json.Marshal(results) + } + + return data, nil +} diff --git a/internal/repository/project/api_config.go b/internal/repository/project/api_config.go index a5be49df20..1eeff48961 100644 --- a/internal/repository/project/api_config.go +++ b/internal/repository/project/api_config.go @@ -67,6 +67,9 @@ func (e *APIConfigAddedEvent) Validate(cmd eventstore.Command) bool { if e.AppID != c.AppID { return false } + if e.ClientID != c.ClientID { + return false + } if e.AuthMethodType != c.AuthMethodType { return false } diff --git a/internal/repository/project/oidc_config.go b/internal/repository/project/oidc_config.go index 4878d241f8..df809daa7c 100644 --- a/internal/repository/project/oidc_config.go +++ b/internal/repository/project/oidc_config.go @@ -109,7 +109,7 @@ func (e *OIDCConfigAddedEvent) Validate(cmd eventstore.Command) bool { if e.AppID != c.AppID { return false } - if e.ClientID != "" && e.ClientID != c.ClientID { + if e.ClientID != c.ClientID { return false } if e.ClientSecret != c.ClientSecret {