ci(e2e): Run Tests in Pipelines (#3903)

* cy10 changes

* test: setup local e2e env

* test(e2e): migrate e2e setup

* add more config

* make e2e setup work

* align variables

* fix config

* skip mfa

* set user register to false

* read ids from database if not provided

* don't read ids withing env file

* fix escaping in id queries

* fix project root

* export projectRoot path

* export projectRoot

* add e2e-setup.sh

* specify GOOS and GOARCH for dockerfile compatible binary

* add org default redirect uri

* correctly initialize org policy

* await ids

* fix awaiting ids

* fix cypress configuration

* fix some tests

* initial compose setup

* fix working directory

* fix references

* make tests less flaky

* run go tests

* compose works until e2e-setup incl

* pass created e2e sa key

* make cypress run

* derive e2e orgs domain from baseurl

* use host from baseurl for setup ctx

* move defaults.yaml back to cmd pkg

* just create org owner

* Don't render element if no roles are passed

* use map instead of switchMap

* fix e2e tests

* added testdata for e3e

* zipped dump

* removed dumpDir

* cypress workflow with compose

* quote name

* cleanup vars

* eliminate need for e2e setup

* compose has no builds anymore

* use compose run and zitadel nw

* test e2e on pr (#4114)

* test e2e on pr

* install goreleaser

* install npm dev dependencies

* run cypress wf

* dynamic release version

* skip flaky user tests

* skip flaky permissions test

* cache docker layers in pipeline

* Update .github/workflows/cypress.yml

Co-authored-by: Florian Forster <florian@caos.ch>

* align goreleaser version

* get rid of install.sh

* remove cypress-terminal-report

* Revert "remove cypress-terminal-report"

This reverts commit 254b5a1f87.

* just one npm e2e:build command

* cache npm dependencies

* install node modules using docker

* dedicated e2e context

* fix syntax

* don't copy node modules from goreleaser

* add npm-copy target

* add tsconfig.json

* remove docker caching

* deleted unneeded shellscript

* naming and cleanup

Co-authored-by: Florian Forster <florian@caos.ch>
Co-authored-by: Christian Jakob <christian@caos.ch>

* cleanup

Co-authored-by: Elio Bischof <eliobischof@gmail.com>
Co-authored-by: Christian Jakob <christian@caos.ch>
Co-authored-by: Florian Forster <florian@caos.ch>

e2e/cypress/support/api/apiauth.ts

@@ -0,0 +1,15 @@
+import { login, User } from 'support/login/users'
+
+export interface apiCallProperties {
+    authHeader: string
+    mgntBaseURL: string
+}
+
+export function apiAuth(): Cypress.Chainable<apiCallProperties> {
+    return login(User.IAMAdminUser, 'Password1!', false, true).then(token => {
+        return <apiCallProperties>{
+            authHeader: `Bearer ${token}`,
+            mgntBaseURL: `/management/v1/`,
+        }
+    })
+}

e2e/cypress/support/api/ensure.ts

@@ -0,0 +1,86 @@
+import { apiCallProperties } from "./apiauth"
+
+export function ensureSomethingExists(api: apiCallProperties, searchPath: string, find: (entity: any) => boolean, createPath: string, body: any): Cypress.Chainable<number> {
+
+    return searchSomething(api, searchPath, find).then(sRes => {
+        if (sRes.entity) {
+            return cy.wrap({
+                id: sRes.entity.id,
+                initialSequence: 0
+            })
+        }
+        return cy.request({
+            method: 'POST',
+            url: `${api.mgntBaseURL}${createPath}`,
+            headers: {
+                Authorization: api.authHeader
+            },
+            body: body,
+            failOnStatusCode: false,
+            followRedirect: false,
+        }).then(cRes => {
+            expect(cRes.status).to.equal(200)
+            return {
+                id: cRes.body.id,
+                initialSequence: sRes.sequence
+            }
+        })
+    }).then((data) => {
+        awaitDesired(30, (entity) => !!entity, data.initialSequence, api, searchPath, find)
+        return cy.wrap<number>(data.id)
+    })
+}
+
+export function ensureSomethingDoesntExist(api: apiCallProperties, searchPath: string, find: (entity: any) => boolean, deletePath: (entity: any) => string): Cypress.Chainable<null> {
+
+    return searchSomething(api, searchPath, find).then(sRes => {
+        if (!sRes.entity) {
+            return cy.wrap(0)
+        }
+        return cy.request({
+            method: 'DELETE',
+            url: `${api.mgntBaseURL}${deletePath(sRes.entity)}`,
+            headers: {
+                Authorization: api.authHeader
+            },
+            failOnStatusCode: false
+        }).then((dRes) => {
+            expect(dRes.status).to.equal(200)
+            return sRes.sequence
+        })
+    }).then((initialSequence) => {
+        awaitDesired(30, (entity) => !entity , initialSequence, api, searchPath, find)
+        return null
+    })
+}
+
+type SearchResult = {
+    entity: any
+    sequence: number
+}
+
+function searchSomething(api: apiCallProperties, searchPath: string, find: (entity: any) => boolean): Cypress.Chainable<SearchResult> {
+
+    return cy.request({
+        method: 'POST',
+        url: `${api.mgntBaseURL}${searchPath}`,
+        headers: {
+            Authorization: api.authHeader
+        },
+    }).then(res => {
+        return {
+            entity: res.body.result?.find(find) || null,
+            sequence: res.body.details.processedSequence
+        }
+    })
+}
+
+function awaitDesired(trials: number, expectEntity: (entity: any) => boolean, initialSequence: number, api: apiCallProperties, searchPath: string, find: (entity: any) => boolean) {
+    searchSomething(api, searchPath, find).then(resp => {
+        if (!expectEntity(resp.entity) || resp.sequence <= initialSequence) {
+            expect(trials, `trying ${trials} more times`).to.be.greaterThan(0);
+            cy.wait(1000)
+            awaitDesired(trials - 1, expectEntity, initialSequence, api, searchPath, find)
+        }            
+    })
+}

e2e/cypress/support/api/policies.ts

@@ -0,0 +1,19 @@
+import { apiCallProperties } from "./apiauth"
+
+
+export enum Policy {
+    Label = "label"
+}
+
+export function resetPolicy(api: apiCallProperties, policy: Policy) {
+    cy.request({
+        method: 'DELETE',
+        url: `${api.mgntBaseURL}/policies/${policy}`,
+        headers: {
+            Authorization: api.authHeader
+        },
+    }).then(res => {
+        expect(res.status).to.equal(200)
+        return null
+    })    
+}

e2e/cypress/support/api/projects.ts

@@ -0,0 +1,80 @@
+import { apiCallProperties } from "./apiauth"
+import { ensureSomethingDoesntExist, ensureSomethingExists } from "./ensure"
+
+export function ensureProjectExists(api: apiCallProperties, projectName: string): Cypress.Chainable<number> {
+    
+    return ensureSomethingExists(
+        api,
+        `projects/_search`,
+        (project: any) => project.name === projectName,
+        'projects',
+        { name: projectName },
+    )
+}
+
+export function ensureProjectDoesntExist(api: apiCallProperties, projectName: string): Cypress.Chainable<null> {
+    
+    return ensureSomethingDoesntExist(
+        api,
+        `projects/_search`,
+        (project: any) => project.name === projectName,
+        (project) => `projects/${project.id}`,
+    )
+}
+
+class ResourceType {
+    constructor(
+        public resourcePath: string,
+        public compareProperty: string,
+        public identifierProperty: string,
+    ){}
+}
+
+export const Apps = new ResourceType('apps', 'name', 'id')
+export const Roles = new ResourceType('roles', 'key', 'key')
+//export const Grants = new ResourceType('apps', 'name')
+
+
+export function ensureProjectResourceDoesntExist(api: apiCallProperties, projectId: number, resourceType: ResourceType, resourceName: string): Cypress.Chainable<null> {
+    return ensureSomethingDoesntExist(
+        api,
+        `projects/${projectId}/${resourceType.resourcePath}/_search`,
+        (resource: any) => {
+            return resource[resourceType.compareProperty] === resourceName
+        },
+        (resource) => {
+            return `projects/${projectId}/${resourceType.resourcePath}/${resource[resourceType.identifierProperty]}`
+        }
+    )
+}
+
+export function ensureApplicationExists(api: apiCallProperties, projectId: number, appName: string): Cypress.Chainable<number> {
+
+    return ensureSomethingExists(
+        api,
+        `projects/${projectId}/${Apps.resourcePath}/_search`,
+        (resource: any) => resource.name === appName,
+        `projects/${projectId}/${Apps.resourcePath}/oidc`,
+        {
+            name: appName,
+            redirectUris: [
+                'https://e2eredirecturl.org'
+            ],
+            responseTypes: [
+                "OIDC_RESPONSE_TYPE_CODE"
+            ],
+            grantTypes: [
+                "OIDC_GRANT_TYPE_AUTHORIZATION_CODE"
+            ],
+            authMethodType: "OIDC_AUTH_METHOD_TYPE_NONE",
+            postLogoutRedirectUris: [
+                'https://e2elogoutredirecturl.org'
+            ],
+/*            "clientId": "129383004379407963@e2eprojectpermission",
+            "clockSkew": "0s",
+            "allowedOrigins": [
+                "https://testurl.org"
+            ]*/
+        },
+    )
+}

e2e/cypress/support/api/users.ts

@@ -0,0 +1,49 @@
+import { apiCallProperties } from "./apiauth"
+import { ensureSomethingDoesntExist, ensureSomethingExists } from "./ensure"
+
+export function ensureHumanUserExists(api: apiCallProperties, username: string): Cypress.Chainable<number> {
+
+    return ensureSomethingExists(
+        api,
+        'users/_search',
+        (user: any) => user.userName === username,
+        'users/human',
+        {
+            user_name: username,
+            profile: {
+                first_name: 'e2efirstName',
+                last_name: 'e2elastName',
+            },
+            email: { 
+                email: 'e2e@email.ch',
+            },
+            phone: {
+                phone: '+41 123456789',
+        },
+    })
+}
+
+export function ensureMachineUserExists(api: apiCallProperties, username: string): Cypress.Chainable<number> {
+    
+    return ensureSomethingExists(
+        api,
+        'users/_search',
+        (user: any) => user.userName === username,
+        'users/machine',
+        {
+            user_name: username,
+            name: 'e2emachinename',
+            description: 'e2emachinedescription',
+        },
+    )
+}
+
+export function ensureUserDoesntExist(api: apiCallProperties, username: string): Cypress.Chainable<null> {
+
+    return ensureSomethingDoesntExist(
+        api,
+        'users/_search',
+        (user: any) => user.userName === username,
+        (user) => `users/${user.id}`
+    )
+}

e2e/cypress/support/commands.ts

@@ -0,0 +1,26 @@
+/*
+namespace Cypress {
+    interface Chainable {
+*/
+        /**
+         * Custom command that authenticates a user.
+         *
+         * @example cy.consolelogin('hodor', 'hodor1234')
+         */
+/*        consolelogin(username: string, password: string): void
+    }
+}
+
+Cypress.Commands.add('consolelogin', { prevSubject: false }, (username: string, password: string) => {
+
+    window.sessionStorage.removeItem("zitadel:access_token")
+    cy.visit(Cypress.config('baseUrl')/ui/console).then(() => {
+        // fill the fields and push button
+        cy.get('#loginName').type(username, { log: false })
+        cy.get('#submit-button').click()
+        cy.get('#password').type(password, { log: false })
+        cy.get('#submit-button').click()
+        cy.location('pathname', {timeout: 5 * 1000}).should('eq', '/');
+    })
+})
+*/

e2e/cypress/support/e2e.ts

@@ -0,0 +1,20 @@
+// ***********************************************************
+// This example support/e2e.ts is processed and
+// loaded automatically before your test files.
+//
+// This is a great place to put global configuration and
+// behavior that modifies Cypress.
+//
+// You can change the location of this file or turn off
+// automatically serving support files with the
+// 'supportFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/configuration
+// ***********************************************************
+
+// Import commands.js using ES2015 syntax:
+import './commands'
+
+// Alternatively you can use CommonJS syntax:
+// require('./commands')

e2e/cypress/support/index.ts

@@ -0,0 +1,12 @@
+require('cypress-terminal-report/src/installLogsCollector')();  
+
+/**
+ * @type {Cypress.PluginConfig}
+ */
+module.exports = (on, config) => {
+
+  // `on` is used to hook into various events Cypress emits
+  // `config` is the resolved Cypress config
+}
+//import './commands'
+

e2e/cypress/support/login/users.ts

@@ -0,0 +1,219 @@
+import { debug } from "console";
+
+export enum User {
+  OrgOwner = 'org_owner',
+  OrgOwnerViewer = 'org_owner_viewer',
+  OrgProjectCreator = 'org_project_creator',
+  LoginPolicyUser = 'login_policy_user',
+  PasswordComplexityUser = 'password_complexity_user',
+  IAMAdminUser = 'zitadel-admin',
+}
+
+export function login(
+  user: User,
+  pw?: string,
+  force?: boolean,
+  skipMFAChangePW?: boolean,
+  onUsernameScreen?: () => void,
+  onPasswordScreen?: () => void,
+  onAuthenticated?: () => void,
+): Cypress.Chainable<string> {
+  let creds = credentials(user, pw);
+
+  const loginUrl: string = '/ui/login';
+  const issuerUrl: string = '/oauth/v2';
+  const otherZitadelIdpInstance: boolean = Cypress.env('otherZitadelIdpInstance');
+
+  return cy.session(
+    creds.username,
+    () => {
+      const cookies = new Map<string, string>();
+
+      cy.intercept(
+        {
+          method: 'GET',
+          url: `${loginUrl}*`,
+          times: 1,
+        },
+        (req) => {
+          req.headers['cookie'] = requestCookies(cookies);
+          req.continue((res) => {
+            updateCookies(res.headers['set-cookie'] as string[], cookies);
+          });
+        },
+      ).as('login');
+
+      cy.intercept(
+        {
+          method: 'POST',
+          url: `${loginUrl}/loginname*`,
+          times: 1,
+        },
+        (req) => {
+          req.headers['cookie'] = requestCookies(cookies);
+          req.continue((res) => {
+            updateCookies(res.headers['set-cookie'] as string[], cookies);
+          });
+        },
+      ).as('loginName');
+
+      cy.intercept(
+        {
+          method: 'POST',
+          url: `${loginUrl}/password*`,
+          times: 1,
+        },
+        (req) => {
+          req.headers['cookie'] = requestCookies(cookies);
+          req.continue((res) => {
+            updateCookies(res.headers['set-cookie'] as string[], cookies);
+          });
+        },
+      ).as('password');
+
+      cy.intercept(
+        {
+          method: 'GET',
+          url: `${loginUrl}/success*`,
+          times: 1,
+        },
+        (req) => {
+          req.headers['cookie'] = requestCookies(cookies);
+          req.continue((res) => {
+            updateCookies(res.headers['set-cookie'] as string[], cookies);
+          });
+        },
+      ).as('success');
+
+      cy.intercept(
+        {
+          method: 'GET',
+          url: `${issuerUrl}/authorize/callback*`,
+          times: 1,
+        },
+        (req) => {
+          req.headers['cookie'] = requestCookies(cookies);
+          req.continue((res) => {
+            updateCookies(res.headers['set-cookie'] as string[], cookies);
+          });
+        },
+      ).as('callback');
+
+      cy.intercept(
+        {
+          method: 'GET',
+          url: `${issuerUrl}/authorize*`,
+          times: 1,
+        },
+        (req) => {
+          req.continue((res) => {
+            updateCookies(res.headers['set-cookie'] as string[], cookies);
+          });
+        },
+      );
+
+      let userToken: string
+      cy.intercept({
+        method: 'POST',
+        url: `${issuerUrl}/token`,
+      }, req => {
+        req.continue(res => {
+          userToken = res.body["access_token"]}
+        )
+      }).as('token')
+
+      cy.visit(loginUrl, { retryOnNetworkFailure: true });
+
+      otherZitadelIdpInstance && cy.wait('@login');
+      onUsernameScreen ? onUsernameScreen() : null;
+      cy.get('#loginName').type(creds.username);
+      cy.get('#submit-button').click();
+
+      otherZitadelIdpInstance && cy.wait('@loginName');
+      onPasswordScreen ? onPasswordScreen() : null;
+      cy.get('#password').type(creds.password);
+      cy.get('#submit-button').click();
+
+      cy.wait('@password').then((interception) => {
+        if (interception.response.body.indexOf('Multifactor Setup') === -1){
+          return
+        }
+
+        cy.contains('button', 'skip').click()
+        cy.get('#change-old-password').type(creds.password)
+        cy.get('#change-new-password').type(creds.password)
+        cy.get('#change-password-confirmation').type(creds.password)
+        cy.contains('button', 'next').click()
+        cy.contains('button', 'next').click()
+      })
+
+      cy.wait('@token').then(() => {
+        cy.task('safetoken', {key: creds.username, token: userToken})
+      })
+
+      onAuthenticated ? onAuthenticated() : null;
+
+      otherZitadelIdpInstance && cy.wait('@callback');
+
+      cy.location('pathname', { timeout: 5 * 1000 }).should('eq', '/ui/console/');
+    },
+    {
+      validate: () => {
+        if (force) {
+          throw new Error('clear session');
+        }
+      },
+    },
+  ).then(() => {
+    return cy.task('loadtoken', {key: creds.username})
+  });
+}
+
+export function loginname(withoutDomain: string, org?: string): string {
+  return `${withoutDomain}@${org}.${host(Cypress.config('baseUrl'))}`;
+}
+
+function credentials(user: User, pw?: string) {
+
+  // TODO: ugly
+  const woDomain = user == User.IAMAdminUser ? User.IAMAdminUser : `${user}_user_name`
+  const org = Cypress.env('ORGANIZATION') ? Cypress.env('ORGANIZATION') : 'zitadel'
+
+  return {
+    username: loginname(woDomain, org),
+    password: pw ? pw : Cypress.env(`${user}_password`),
+  };
+}
+
+function updateCookies(newCookies: string[] | undefined, currentCookies: Map<string, string>) {
+  if (newCookies === undefined) {
+    return;
+  }
+  newCookies.forEach((cs) => {
+    cs.split('; ').forEach((cookie) => {
+      const idx = cookie.indexOf('=');
+      currentCookies.set(cookie.substring(0, idx), cookie.substring(idx + 1));
+    });
+  });
+}
+
+function requestCookies(currentCookies: Map<string, string>): string[] {
+  let list: Array<string> = [];
+  currentCookies.forEach((val, key) => {
+    list.push(key + '=' + val);
+  });
+  return list;
+}
+
+export function host(url: string): string {
+  return stripPort(stripProtocol(url));
+}
+
+function stripPort(s: string): string {
+  const idx = s.indexOf(':');
+  return idx === -1 ? s : s.substring(0, idx);
+}
+
+function stripProtocol(url: string): string {
+  return url.replace('http://', '').replace('https://', '');
+}