feat: call webhooks at least once (#5454)

* feat: call webhooks at least once

* self review

* feat: improve notification observability

* feat: add notification tracing

* test(e2e): test at-least-once webhook delivery

* fix webhook notifications

* dedicated quota notifications handler

* fix linting

* fix e2e test

* wait less in e2e test

* fix: don't ignore failed events in handlers

* fix: don't ignore failed events in handlers

* faster requeues

* question

* fix retries

* fix retries

* retry

* don't instance ids query

* revert handler_projection

* statements can be nil

* cleanup

* make unit tests pass

* add comments

* add comments

* lint

* spool only active instances

* feat(config): handle inactive instances

* customizable HandleInactiveInstances

* call inactive instances quota webhooks

* test: handling with and w/o inactive instances

* omit retrying noop statements

* docs: describe projection options

* enable global handling of inactive instances

* self review

* requeue quota notifications every 5m

* remove caos_errors reference

* fix comment styles

* make handlers package flat

* fix linting

* fix repeating quota notifications

* test with more usage

* debug log channel init failures

e2e/cypress.config.ts

@@ -34,8 +34,8 @@ YkTaa1AFLstnf348ZjuvBN3USUYZo3X3mxnS+uluVuRSGwIKsN0a
 -----END RSA PRIVATE KEY-----`
 
 let tokensCache = new Map<string,string>()
-
 let webhookEvents = new Array<ZITADELWebhookEvent>()
+let failWebhookEventsCount = 0
 
 export default defineConfig({
   reporter: 'mochawesome',
@@ -98,10 +98,15 @@ export default defineConfig({
         },
         resetWebhookEvents() {
           webhookEvents = []
+          failWebhookEventsCount = 0
           return null
         },
         handledWebhookEvents(){
           return webhookEvents
+        },
+        failWebhookEvents(count: number){
+          failWebhookEventsCount = count
+          return null
         }
       })
     },
@@ -127,11 +132,17 @@ function startWebhookEventHandler() {
     req.on("data", (chunk) => {
       chunks.push(chunk);
     });
+    const sendStatus = failWebhookEventsCount ? 500 : 200
     req.on("end", () => {
-      webhookEvents.push(JSON.parse(Buffer.concat(chunks).toString()));
+      webhookEvents.push({
+        sentStatus: sendStatus,
+        payload: JSON.parse(Buffer.concat(chunks).toString())
+      });
     });
-
-    res.writeHead(200);
+    if (failWebhookEventsCount > 0){
+      failWebhookEventsCount--
+    }
+    res.writeHead(sendStatus);
     res.end()
   });