feat: call webhooks at least once (#5454)

* feat: call webhooks at least once

* self review

* feat: improve notification observability

* feat: add notification tracing

* test(e2e): test at-least-once webhook delivery

* fix webhook notifications

* dedicated quota notifications handler

* fix linting

* fix e2e test

* wait less in e2e test

* fix: don't ignore failed events in handlers

* fix: don't ignore failed events in handlers

* faster requeues

* question

* fix retries

* fix retries

* retry

* don't instance ids query

* revert handler_projection

* statements can be nil

* cleanup

* make unit tests pass

* add comments

* add comments

* lint

* spool only active instances

* feat(config): handle inactive instances

* customizable HandleInactiveInstances

* call inactive instances quota webhooks

* test: handling with and w/o inactive instances

* omit retrying noop statements

* docs: describe projection options

* enable global handling of inactive instances

* self review

* requeue quota notifications every 5m

* remove caos_errors reference

* fix comment styles

* make handlers package flat

* fix linting

* fix repeating quota notifications

* test with more usage

* debug log channel init failures

e2e/config/host.docker.internal/zitadel.yaml

@@ -1,3 +1,6 @@
+Log:
+  Level: debug
+
 ExternalDomain: host.docker.internal
 ExternalSecure: false
 
@@ -32,6 +35,11 @@ Quotas:
     ExhaustedCookieKey: "zitadel.quota.limiting"
     ExhaustedCookieMaxAge: "60s"
 
+Projections:
+  Customizations:
+    NotificationsQuotas:
+      RequeueEvery: 1s
+
 DefaultInstance:
   LoginPolicy:
     MfaInitSkipLifetime: "0"

e2e/config/localhost/docker-compose.yaml

@@ -27,4 +27,5 @@ services:
       retries: 5
       start_period: '20s'
     ports:
-      - "26257:26257"
+      - "26257:26257"
+      - "9090:9090"

e2e/config/localhost/zitadel.yaml

@@ -1,3 +1,6 @@
+Log:
+  Level: debug
+
 ExternalDomain: localhost
 ExternalSecure: false
 
@@ -32,6 +35,11 @@ Quotas:
     ExhaustedCookieKey: "zitadel.quota.limiting"
     ExhaustedCookieMaxAge: "60s"
 
+Projections:
+  Customizations:
+    NotificationsQuotas:
+      RequeueEvery: 1s
+
 DefaultInstance:
   LoginPolicy:
     MfaInitSkipLifetime: "0"

e2e/cypress.config.ts

@@ -34,8 +34,8 @@ YkTaa1AFLstnf348ZjuvBN3USUYZo3X3mxnS+uluVuRSGwIKsN0a
 -----END RSA PRIVATE KEY-----`
 
 let tokensCache = new Map<string,string>()
-
 let webhookEvents = new Array<ZITADELWebhookEvent>()
+let failWebhookEventsCount = 0
 
 export default defineConfig({
   reporter: 'mochawesome',
@@ -98,10 +98,15 @@ export default defineConfig({
         },
         resetWebhookEvents() {
           webhookEvents = []
+          failWebhookEventsCount = 0
           return null
         },
         handledWebhookEvents(){
           return webhookEvents
+        },
+        failWebhookEvents(count: number){
+          failWebhookEventsCount = count
+          return null
         }
       })
     },
@@ -127,11 +132,17 @@ function startWebhookEventHandler() {
     req.on("data", (chunk) => {
       chunks.push(chunk);
     });
+    const sendStatus = failWebhookEventsCount ? 500 : 200
     req.on("end", () => {
-      webhookEvents.push(JSON.parse(Buffer.concat(chunks).toString()));
+      webhookEvents.push({
+        sentStatus: sendStatus,
+        payload: JSON.parse(Buffer.concat(chunks).toString())
+      });
     });
-
-    res.writeHead(200);
+    if (failWebhookEventsCount > 0){
+      failWebhookEventsCount--
+    }
+    res.writeHead(sendStatus);
     res.end()
   });
 

e2e/cypress/e2e/quotas/quotas.cy.ts

@@ -2,6 +2,7 @@ import { addQuota, ensureQuotaIsAdded, ensureQuotaIsRemoved, removeQuota, Unit }
 import { createHumanUser, ensureUserDoesntExist } from 'support/api/users';
 import { Context } from 'support/commands';
 import { ZITADELWebhookEvent } from 'support/types';
+import { textChangeRangeIsUnchanged } from 'typescript';
 
 beforeEach(() => {
   cy.context().as('ctx');
@@ -144,7 +145,7 @@ describe('quotas', () => {
 
       const amount = 100;
       const percent = 10;
-      const usage = 25;
+      const usage = 35;
 
       describe('without repetition', () => {
         beforeEach(() => {
@@ -160,7 +161,7 @@ describe('quotas', () => {
           });
         });
 
-        it('fires once with the expected payload', () => {
+        it('fires at least once with the expected payload', () => {
           cy.get<Array<string>>('@authenticatedUrls').then((urls) => {
             cy.get<Context>('@ctx').then((ctx) => {
               for (let i = 0; i < usage; i++) {
@@ -175,19 +176,71 @@ describe('quotas', () => {
             });
             cy.waitUntil(() =>
               cy.task<Array<ZITADELWebhookEvent>>('handledWebhookEvents').then((events) => {
-                if (events.length != 1) {
+                if (events.length < 1) {
                   return false;
                 }
                 return Cypress._.matches(<ZITADELWebhookEvent>{
-                  callURL: callURL,
-                  threshold: percent,
-                  unit: 1,
-                  usage: percent,
+                  sentStatus: 200,
+                  payload: {
+                    callURL: callURL,
+                    threshold: percent,
+                    unit: 1,
+                    usage: percent,
+                  },
                 })(events[0]);
               }),
             );
           });
         });
+
+        it('fires until the webhook returns a successful message', () => {
+          cy.task('failWebhookEvents', 8);
+          cy.get<Array<string>>('@authenticatedUrls').then((urls) => {
+            cy.get<Context>('@ctx').then((ctx) => {
+              for (let i = 0; i < usage; i++) {
+                cy.request({
+                  url: urls[0],
+                  method: 'GET',
+                  auth: {
+                    bearer: ctx.api.token,
+                  },
+                });
+              }
+            });
+            cy.waitUntil(
+              () =>
+                cy.task<Array<ZITADELWebhookEvent>>('handledWebhookEvents').then((events) => {
+                  if (events.length != 9) {
+                    return false;
+                  }
+                  return events.reduce<boolean>((a, b, i) => {
+                    return !a
+                      ? a
+                      : i < 8
+                      ? Cypress._.matches(<ZITADELWebhookEvent>{
+                          sentStatus: 500,
+                          payload: {
+                            callURL: callURL,
+                            threshold: percent,
+                            unit: 1,
+                            usage: percent,
+                          },
+                        })(b)
+                      : Cypress._.matches(<ZITADELWebhookEvent>{
+                          sentStatus: 200,
+                          payload: {
+                            callURL: callURL,
+                            threshold: percent,
+                            unit: 1,
+                            usage: percent,
+                          },
+                        })(b);
+                  }, true);
+                }),
+              { timeout: 60_000 },
+            );
+          });
+        });
       });
 
       describe('with repetition', () => {
@@ -222,23 +275,25 @@ describe('quotas', () => {
           });
           cy.waitUntil(() =>
             cy.task<Array<ZITADELWebhookEvent>>('handledWebhookEvents').then((events) => {
-              if (events.length != 1) {
-                return false;
-              }
+              let foundExpected = 0;
               for (let i = 0; i < events.length; i++) {
-                const threshold = percent * (i + 1);
-                if (
-                  !Cypress._.matches(<ZITADELWebhookEvent>{
-                    callURL: callURL,
-                    threshold: threshold,
-                    unit: 1,
-                    usage: threshold,
-                  })(events[i])
-                ) {
-                  return false;
+                for (let expect = 10; expect <= 30; expect += 10) {
+                  if (
+                    Cypress._.matches(<ZITADELWebhookEvent>{
+                      sentStatus: 200,
+                      payload: {
+                        callURL: callURL,
+                        threshold: expect,
+                        unit: 1,
+                        usage: expect,
+                      },
+                    })(events[i])
+                  ) {
+                    foundExpected++;
+                  }
                 }
               }
-              return true;
+              return foundExpected >= 3;
             }),
           );
         });

e2e/cypress/support/types.ts

@@ -1,10 +1,13 @@
 let webhookEventSchema = {
-  unit: 0,
-  id: '',
-  callURL: '',
-  periodStart: new Date(),
-  threshold: 0,
-  usage: 0,
+  sentStatus: 0,
+  payload: {
+    unit: 0,
+    id: '',
+    callURL: '',
+    periodStart: new Date(),
+    threshold: 0,
+    usage: 0,
+  },
 };
 
 export type ZITADELWebhookEvent = typeof webhookEventSchema;