perf: project quotas and usages (#6441)

* project quota added

* project quota removed

* add periods table

* make log record generic

* accumulate usage

* query usage

* count action run seconds

* fix filter in ReportQuotaUsage

* fix existing tests

* fix logstore tests

* fix typo

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* move notifications into debouncer and improve limit querying

* cleanup

* comment

* fix: add quota unit tests command side

* fix remaining quota usage query

* implement InmemLogStorage

* cleanup and linting

* improve test

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* fix: add quota unit tests command side

* action notifications and fixes for notifications query

* revert console prefix

* fix: add quota unit tests command side

* fix: add quota integration tests

* improve accountable requests

* improve accountable requests

* fix: add quota integration tests

* fix: add quota integration tests

* fix: add quota integration tests

* comment

* remove ability to store logs in db and other changes requested from review

* changes requested from review

* changes requested from review

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* tests: fix quotas integration tests

* improve incrementUsageStatement

* linting

* fix: delete e2e tests as intergation tests cover functionality

* Update internal/api/http/middleware/access_interceptor.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* backup

* fix conflict

* create rc

* create prerelease

* remove issue release labeling

* fix tracing

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Stefan Benz <stefan@caos.ch>
Co-authored-by: adlerhurst <silvan.reusser@gmail.com>
This commit is contained in:
Elio Bischof
2023-09-15 16:58:45 +02:00
committed by GitHub
parent b4d0d2c9a7
commit 1a49b7d298
66 changed files with 3423 additions and 1413 deletions

View File

@@ -20,11 +20,13 @@ var (
func TestMain(m *testing.M) {
os.Exit(func() int {
ctx, _, cancel := integration.Contexts(5 * time.Minute)
CTX = ctx
defer cancel()
CTX = ctx
Tester = integration.NewTester(ctx)
SystemCTX = Tester.WithAuthorization(ctx, integration.SystemUser)
defer Tester.Done()
SystemCTX = Tester.WithAuthorization(ctx, integration.SystemUser)
return m.Run()
}())
}

View File

@@ -5,11 +5,6 @@ package handlers_test
import (
"bytes"
"encoding/json"
"io"
"net"
"net/http"
"net/http/httptest"
"reflect"
"testing"
"time"
@@ -18,49 +13,27 @@ import (
)
func TestServer_TelemetryPushMilestones(t *testing.T) {
bodies := make(chan []byte, 0)
mockServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
if err != nil {
t.Error(err)
}
if r.Header.Get("single-value") != "single-value" {
t.Error("single-value header not set")
}
if reflect.DeepEqual(r.Header.Get("multi-value"), "multi-value-1,multi-value-2") {
t.Error("single-value header not set")
}
bodies <- body
w.WriteHeader(http.StatusOK)
}))
listener, err := net.Listen("tcp", "localhost:8081")
if err != nil {
t.Fatal(err)
}
mockServer.Listener = listener
mockServer.Start()
t.Cleanup(mockServer.Close)
primaryDomain, instanceID, iamOwnerCtx := Tester.UseIsolatedInstance(CTX, SystemCTX)
t.Log("testing against instance with primary domain", primaryDomain)
awaitMilestone(t, bodies, primaryDomain, "InstanceCreated")
awaitMilestone(t, Tester.MilestoneChan, primaryDomain, "InstanceCreated")
project, err := Tester.Client.Mgmt.AddProject(iamOwnerCtx, &management.AddProjectRequest{Name: "integration"})
if err != nil {
t.Fatal(err)
}
awaitMilestone(t, bodies, primaryDomain, "ProjectCreated")
awaitMilestone(t, Tester.MilestoneChan, primaryDomain, "ProjectCreated")
if _, err = Tester.Client.Mgmt.AddOIDCApp(iamOwnerCtx, &management.AddOIDCAppRequest{
ProjectId: project.GetId(),
Name: "integration",
}); err != nil {
t.Fatal(err)
}
awaitMilestone(t, bodies, primaryDomain, "ApplicationCreated")
awaitMilestone(t, Tester.MilestoneChan, primaryDomain, "ApplicationCreated")
// TODO: trigger and await milestone AuthenticationSucceededOnInstance
// TODO: trigger and await milestone AuthenticationSucceededOnApplication
if _, err = Tester.Client.System.RemoveInstance(SystemCTX, &system.RemoveInstanceRequest{InstanceId: instanceID}); err != nil {
t.Fatal(err)
}
awaitMilestone(t, bodies, primaryDomain, "InstanceDeleted")
awaitMilestone(t, Tester.MilestoneChan, primaryDomain, "InstanceDeleted")
}
func awaitMilestone(t *testing.T, bodies chan []byte, primaryDomain, expectMilestoneType string) {