Files
zitadel/internal/migration/trigger_test.go

532 lines
13 KiB
Go
Raw Normal View History

feat(projections): resource counters (#9979) # Which Problems Are Solved Add the ability to keep track of the current counts of projection resources. We want to prevent calling `SELECT COUNT(*)` on tables, as that forces a full scan and sudden spikes of DB resource uses. # How the Problems Are Solved - A resource_counts table is added - Triggers that increment and decrement the counted values on inserts and deletes - Triggers that delete all counts of a table when the source table is TRUNCATEd. This is not in the business logic, but prevents wrong counts in case someone want to force a re-projection. - Triggers that delete all counts if the parent resource is deleted - Script to pre-populate the resource_counts table when a new source table is added. The triggers are reusable for any type of resource, in case we choose to add more in the future. Counts are aggregated by a given parent. Currently only `instance` and `organization` are defined as possible parent. This can later be extended to other types, such as `project`, should the need arise. I deliberately chose to use `parent_id` to distinguish from the de-factor `resource_owner` which is usually an organization ID. For example: - For users the parent is an organization and the `parent_id` matches `resource_owner`. - For organizations the parent is an instance, but the `resource_owner` is the `org_id`. In this case the `parent_id` is the `instance_id`. - Applications would have a similar problem, where the parent is a project, but the `resource_owner` is the `org_id` # Additional Context Closes https://github.com/zitadel/zitadel/issues/9957
2025-06-03 17:15:30 +03:00
package migration
import (
"context"
"regexp"
"testing"
"github.com/DATA-DOG/go-sqlmock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zitadel/zitadel/internal/database"
)
const (
feat(service ping): add additional resource counts (#10621) # Which Problems Are Solved Using the service ping, we want to have some additional insights to how zitadel is configured. The current resource count report contains already some amount of configured policies, such as the login_policy. But we do not know if for example MFA is enforced. # How the Problems Are Solved - Added the following counts to the report: - service users per organization - MFA enforcements (though login policy) - Notification policies with password change option enabled - SCIM provisioned users (using user metadata) - Since all of the above are conditional based on at least a column inside a projection, a new `migration.CountTriggerConditional` has been added, where a condition (column values) and an option to track updates on that column should be considered for the count. - For this to be possible, the following changes had to be made to the existing sql resources: - the `resource_name` has been added to unique constraint on the `projection.resource_counts` table - triggers have been added / changed to individually track `INSERT`, `UPDATE`(s) and `DELETE` and be able to handle conditions - an optional argument has been added to the `projections.count_resource()` function to allow providing the information to `UP` or `DOWN` count the resource on an update. # Additional Changes None # Additional Context - partially solves #10244 (reporting audit log retention limit will be handled in #10245 directly) - backport to v4.x (cherry picked from commit 2dbe21fb300907ac430ad3badea1cda140b792a5)
2025-09-08 18:30:03 +02:00
expCountTriggerQuery = `-- In case the old trigger exists, drop it to prevent duplicated counts.
DROP TRIGGER IF EXISTS count_resource ON table;
CREATE OR REPLACE TRIGGER count_resource_insert
AFTER INSERT
feat(projections): resource counters (#9979) # Which Problems Are Solved Add the ability to keep track of the current counts of projection resources. We want to prevent calling `SELECT COUNT(*)` on tables, as that forces a full scan and sudden spikes of DB resource uses. # How the Problems Are Solved - A resource_counts table is added - Triggers that increment and decrement the counted values on inserts and deletes - Triggers that delete all counts of a table when the source table is TRUNCATEd. This is not in the business logic, but prevents wrong counts in case someone want to force a re-projection. - Triggers that delete all counts if the parent resource is deleted - Script to pre-populate the resource_counts table when a new source table is added. The triggers are reusable for any type of resource, in case we choose to add more in the future. Counts are aggregated by a given parent. Currently only `instance` and `organization` are defined as possible parent. This can later be extended to other types, such as `project`, should the need arise. I deliberately chose to use `parent_id` to distinguish from the de-factor `resource_owner` which is usually an organization ID. For example: - For users the parent is an organization and the `parent_id` matches `resource_owner`. - For organizations the parent is an instance, but the `resource_owner` is the `org_id`. In this case the `parent_id` is the `instance_id`. - Applications would have a similar problem, where the parent is a project, but the `resource_owner` is the `org_id` # Additional Context Closes https://github.com/zitadel/zitadel/issues/9957
2025-06-03 17:15:30 +03:00
ON table
FOR EACH ROW
feat(service ping): add additional resource counts (#10621) # Which Problems Are Solved Using the service ping, we want to have some additional insights to how zitadel is configured. The current resource count report contains already some amount of configured policies, such as the login_policy. But we do not know if for example MFA is enforced. # How the Problems Are Solved - Added the following counts to the report: - service users per organization - MFA enforcements (though login policy) - Notification policies with password change option enabled - SCIM provisioned users (using user metadata) - Since all of the above are conditional based on at least a column inside a projection, a new `migration.CountTriggerConditional` has been added, where a condition (column values) and an option to track updates on that column should be considered for the count. - For this to be possible, the following changes had to be made to the existing sql resources: - the `resource_name` has been added to unique constraint on the `projection.resource_counts` table - triggers have been added / changed to individually track `INSERT`, `UPDATE`(s) and `DELETE` and be able to handle conditions - an optional argument has been added to the `projections.count_resource()` function to allow providing the information to `UP` or `DOWN` count the resource on an update. # Additional Changes None # Additional Context - partially solves #10244 (reporting audit log retention limit will be handled in #10245 directly) - backport to v4.x (cherry picked from commit 2dbe21fb300907ac430ad3badea1cda140b792a5)
2025-09-08 18:30:03 +02:00
-- Only count if the conditions are met in the newly added row.
EXECUTE FUNCTION projections.count_resource(
'instance',
'instance_id',
'parent_id',
'resource'
);
CREATE OR REPLACE TRIGGER count_resource_delete
AFTER DELETE
ON table
FOR EACH ROW
-- Only count down if the conditions were met in the old / deleted row.
feat(projections): resource counters (#9979) # Which Problems Are Solved Add the ability to keep track of the current counts of projection resources. We want to prevent calling `SELECT COUNT(*)` on tables, as that forces a full scan and sudden spikes of DB resource uses. # How the Problems Are Solved - A resource_counts table is added - Triggers that increment and decrement the counted values on inserts and deletes - Triggers that delete all counts of a table when the source table is TRUNCATEd. This is not in the business logic, but prevents wrong counts in case someone want to force a re-projection. - Triggers that delete all counts if the parent resource is deleted - Script to pre-populate the resource_counts table when a new source table is added. The triggers are reusable for any type of resource, in case we choose to add more in the future. Counts are aggregated by a given parent. Currently only `instance` and `organization` are defined as possible parent. This can later be extended to other types, such as `project`, should the need arise. I deliberately chose to use `parent_id` to distinguish from the de-factor `resource_owner` which is usually an organization ID. For example: - For users the parent is an organization and the `parent_id` matches `resource_owner`. - For organizations the parent is an instance, but the `resource_owner` is the `org_id`. In this case the `parent_id` is the `instance_id`. - Applications would have a similar problem, where the parent is a project, but the `resource_owner` is the `org_id` # Additional Context Closes https://github.com/zitadel/zitadel/issues/9957
2025-06-03 17:15:30 +03:00
EXECUTE FUNCTION projections.count_resource(
'instance',
'instance_id',
'parent_id',
'resource'
);
CREATE OR REPLACE TRIGGER truncate_resource_counts
AFTER TRUNCATE
ON table
FOR EACH STATEMENT
EXECUTE FUNCTION projections.delete_table_counts();
-- Prevent inserts and deletes while we populate the counts.
LOCK TABLE table IN SHARE MODE;
-- Populate the resource counts for the existing data in the table.
INSERT INTO projections.resource_counts(
instance_id,
table_name,
parent_type,
parent_id,
resource_name,
amount
)
SELECT
instance_id,
'table',
'instance',
parent_id,
'resource',
COUNT(*) AS amount
FROM table
GROUP BY (instance_id, parent_id)
feat(service ping): add additional resource counts (#10621) # Which Problems Are Solved Using the service ping, we want to have some additional insights to how zitadel is configured. The current resource count report contains already some amount of configured policies, such as the login_policy. But we do not know if for example MFA is enforced. # How the Problems Are Solved - Added the following counts to the report: - service users per organization - MFA enforcements (though login policy) - Notification policies with password change option enabled - SCIM provisioned users (using user metadata) - Since all of the above are conditional based on at least a column inside a projection, a new `migration.CountTriggerConditional` has been added, where a condition (column values) and an option to track updates on that column should be considered for the count. - For this to be possible, the following changes had to be made to the existing sql resources: - the `resource_name` has been added to unique constraint on the `projection.resource_counts` table - triggers have been added / changed to individually track `INSERT`, `UPDATE`(s) and `DELETE` and be able to handle conditions - an optional argument has been added to the `projections.count_resource()` function to allow providing the information to `UP` or `DOWN` count the resource on an update. # Additional Changes None # Additional Context - partially solves #10244 (reporting audit log retention limit will be handled in #10245 directly) - backport to v4.x (cherry picked from commit 2dbe21fb300907ac430ad3badea1cda140b792a5)
2025-09-08 18:30:03 +02:00
ON CONFLICT (instance_id, table_name, parent_type, parent_id, resource_name) DO
UPDATE SET updated_at = now(), amount = EXCLUDED.amount;`
expCountTriggerConditionalQuery = `-- In case the old trigger exists, drop it to prevent duplicated counts.
DROP TRIGGER IF EXISTS count_resource ON table;
CREATE OR REPLACE TRIGGER count_resource_insert
AFTER INSERT
ON table
FOR EACH ROW
-- Only count if the conditions are met in the newly added row.
WHEN ((NEW.col1 = 'value1' OR NEW.col2 = 'value1'))
EXECUTE FUNCTION projections.count_resource(
'instance',
'instance_id',
'parent_id',
'resource'
);
CREATE OR REPLACE TRIGGER count_resource_delete
AFTER DELETE
ON table
FOR EACH ROW
-- Only count down if the conditions were met in the old / deleted row.
WHEN ((OLD.col1 = 'value1' OR OLD.col2 = 'value1'))
EXECUTE FUNCTION projections.count_resource(
'instance',
'instance_id',
'parent_id',
'resource'
);
CREATE OR REPLACE TRIGGER count_resource_update_up
AFTER UPDATE
ON table
FOR EACH ROW
-- Only count up if the conditions are met in the new state, but were not in the old.
WHEN ((NEW.col1 = 'value1' OR NEW.col2 = 'value1') AND (OLD.col1 <> 'value1' AND OLD.col2 <> 'value1'))
EXECUTE FUNCTION projections.count_resource(
'instance',
'instance_id',
'parent_id',
'resource',
'UP'
);
CREATE OR REPLACE TRIGGER count_resource_update_down
AFTER UPDATE
ON table
FOR EACH ROW
-- Only count down if the conditions are not met in the new state, but were in the old.
WHEN ((NEW.col1 <> 'value1' AND NEW.col2 <> 'value1') AND (OLD.col1 = 'value1' OR OLD.col2 = 'value1'))
EXECUTE FUNCTION projections.count_resource(
'instance',
'instance_id',
'parent_id',
'resource',
'DOWN'
);
CREATE OR REPLACE TRIGGER truncate_resource_counts
AFTER TRUNCATE
ON table
FOR EACH STATEMENT
EXECUTE FUNCTION projections.delete_table_counts();
-- Prevent inserts and deletes while we populate the counts.
LOCK TABLE table IN SHARE MODE;
-- Populate the resource counts for the existing data in the table.
INSERT INTO projections.resource_counts(
instance_id,
table_name,
parent_type,
parent_id,
resource_name,
amount
)
SELECT
instance_id,
'table',
'instance',
parent_id,
'resource',
COUNT(*) AS amount
FROM table
WHERE (table.col1 = 'value1' OR table.col2 = 'value1')
GROUP BY (instance_id, parent_id)
ON CONFLICT (instance_id, table_name, parent_type, parent_id, resource_name) DO
feat(projections): resource counters (#9979) # Which Problems Are Solved Add the ability to keep track of the current counts of projection resources. We want to prevent calling `SELECT COUNT(*)` on tables, as that forces a full scan and sudden spikes of DB resource uses. # How the Problems Are Solved - A resource_counts table is added - Triggers that increment and decrement the counted values on inserts and deletes - Triggers that delete all counts of a table when the source table is TRUNCATEd. This is not in the business logic, but prevents wrong counts in case someone want to force a re-projection. - Triggers that delete all counts if the parent resource is deleted - Script to pre-populate the resource_counts table when a new source table is added. The triggers are reusable for any type of resource, in case we choose to add more in the future. Counts are aggregated by a given parent. Currently only `instance` and `organization` are defined as possible parent. This can later be extended to other types, such as `project`, should the need arise. I deliberately chose to use `parent_id` to distinguish from the de-factor `resource_owner` which is usually an organization ID. For example: - For users the parent is an organization and the `parent_id` matches `resource_owner`. - For organizations the parent is an instance, but the `resource_owner` is the `org_id`. In this case the `parent_id` is the `instance_id`. - Applications would have a similar problem, where the parent is a project, but the `resource_owner` is the `org_id` # Additional Context Closes https://github.com/zitadel/zitadel/issues/9957
2025-06-03 17:15:30 +03:00
UPDATE SET updated_at = now(), amount = EXCLUDED.amount;`
expDeleteParentCountsQuery = `CREATE OR REPLACE TRIGGER delete_parent_counts_trigger
AFTER DELETE
ON table
FOR EACH ROW
EXECUTE FUNCTION projections.delete_parent_counts(
'instance',
'instance_id',
'parent_id'
);`
)
func Test_triggerMigration_Execute(t *testing.T) {
type fields struct {
triggerConfig triggerConfig
templateName string
}
tests := []struct {
name string
fields fields
expects func(sqlmock.Sqlmock)
wantErr bool
}{
{
name: "template error",
fields: fields{
triggerConfig: triggerConfig{
Table: "table",
ParentType: "instance",
InstanceIDColumn: "instance_id",
ParentIDColumn: "parent_id",
Resource: "resource",
},
templateName: "foo",
},
expects: func(_ sqlmock.Sqlmock) {},
wantErr: true,
},
{
name: "db error",
fields: fields{
triggerConfig: triggerConfig{
Table: "table",
ParentType: "instance",
InstanceIDColumn: "instance_id",
ParentIDColumn: "parent_id",
Resource: "resource",
},
templateName: countTriggerTmpl,
},
expects: func(mock sqlmock.Sqlmock) {
mock.ExpectExec(regexp.QuoteMeta(expCountTriggerQuery)).
WillReturnError(assert.AnError)
},
wantErr: true,
},
{
name: "count trigger",
fields: fields{
triggerConfig: triggerConfig{
Table: "table",
ParentType: "instance",
InstanceIDColumn: "instance_id",
ParentIDColumn: "parent_id",
Resource: "resource",
},
templateName: countTriggerTmpl,
},
expects: func(mock sqlmock.Sqlmock) {
mock.ExpectExec(regexp.QuoteMeta(expCountTriggerQuery)).
WithoutArgs().
WillReturnResult(
sqlmock.NewResult(1, 1),
)
},
},
feat(service ping): add additional resource counts (#10621) # Which Problems Are Solved Using the service ping, we want to have some additional insights to how zitadel is configured. The current resource count report contains already some amount of configured policies, such as the login_policy. But we do not know if for example MFA is enforced. # How the Problems Are Solved - Added the following counts to the report: - service users per organization - MFA enforcements (though login policy) - Notification policies with password change option enabled - SCIM provisioned users (using user metadata) - Since all of the above are conditional based on at least a column inside a projection, a new `migration.CountTriggerConditional` has been added, where a condition (column values) and an option to track updates on that column should be considered for the count. - For this to be possible, the following changes had to be made to the existing sql resources: - the `resource_name` has been added to unique constraint on the `projection.resource_counts` table - triggers have been added / changed to individually track `INSERT`, `UPDATE`(s) and `DELETE` and be able to handle conditions - an optional argument has been added to the `projections.count_resource()` function to allow providing the information to `UP` or `DOWN` count the resource on an update. # Additional Changes None # Additional Context - partially solves #10244 (reporting audit log retention limit will be handled in #10245 directly) - backport to v4.x (cherry picked from commit 2dbe21fb300907ac430ad3badea1cda140b792a5)
2025-09-08 18:30:03 +02:00
{
name: "count trigger conditionally",
fields: fields{
triggerConfig: triggerConfig{
Table: "table",
ParentType: "instance",
InstanceIDColumn: "instance_id",
ParentIDColumn: "parent_id",
Resource: "resource",
Conditions: OrCondition{
Conditions: []TriggerCondition{
{Column: "col1", Value: "value1"},
{Column: "col2", Value: "value1"},
},
},
TrackChange: true,
},
templateName: countTriggerTmpl,
},
expects: func(mock sqlmock.Sqlmock) {
mock.ExpectExec(regexp.QuoteMeta(expCountTriggerConditionalQuery)).
WithoutArgs().
WillReturnResult(
sqlmock.NewResult(1, 1),
)
},
},
feat(projections): resource counters (#9979) # Which Problems Are Solved Add the ability to keep track of the current counts of projection resources. We want to prevent calling `SELECT COUNT(*)` on tables, as that forces a full scan and sudden spikes of DB resource uses. # How the Problems Are Solved - A resource_counts table is added - Triggers that increment and decrement the counted values on inserts and deletes - Triggers that delete all counts of a table when the source table is TRUNCATEd. This is not in the business logic, but prevents wrong counts in case someone want to force a re-projection. - Triggers that delete all counts if the parent resource is deleted - Script to pre-populate the resource_counts table when a new source table is added. The triggers are reusable for any type of resource, in case we choose to add more in the future. Counts are aggregated by a given parent. Currently only `instance` and `organization` are defined as possible parent. This can later be extended to other types, such as `project`, should the need arise. I deliberately chose to use `parent_id` to distinguish from the de-factor `resource_owner` which is usually an organization ID. For example: - For users the parent is an organization and the `parent_id` matches `resource_owner`. - For organizations the parent is an instance, but the `resource_owner` is the `org_id`. In this case the `parent_id` is the `instance_id`. - Applications would have a similar problem, where the parent is a project, but the `resource_owner` is the `org_id` # Additional Context Closes https://github.com/zitadel/zitadel/issues/9957
2025-06-03 17:15:30 +03:00
{
name: "count trigger",
fields: fields{
triggerConfig: triggerConfig{
Table: "table",
ParentType: "instance",
InstanceIDColumn: "instance_id",
ParentIDColumn: "parent_id",
Resource: "resource",
},
templateName: deleteParentCountsTmpl,
},
expects: func(mock sqlmock.Sqlmock) {
mock.ExpectExec(regexp.QuoteMeta(expDeleteParentCountsQuery)).
WithoutArgs().
WillReturnResult(
sqlmock.NewResult(1, 1),
)
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
defer func() {
err := mock.ExpectationsWereMet()
require.NoError(t, err)
}()
defer db.Close()
tt.expects(mock)
mock.ExpectClose()
m := &triggerMigration{
db: &database.DB{
DB: db,
},
triggerConfig: tt.fields.triggerConfig,
templateName: tt.fields.templateName,
}
err = m.Execute(context.Background(), nil)
if tt.wantErr {
assert.Error(t, err)
return
}
require.NoError(t, err)
})
}
}
func Test_triggerConfig_Check(t *testing.T) {
type fields struct {
Table string
ParentType string
InstanceIDColumn string
ParentIDColumn string
Resource string
feat(service ping): add additional resource counts (#10621) # Which Problems Are Solved Using the service ping, we want to have some additional insights to how zitadel is configured. The current resource count report contains already some amount of configured policies, such as the login_policy. But we do not know if for example MFA is enforced. # How the Problems Are Solved - Added the following counts to the report: - service users per organization - MFA enforcements (though login policy) - Notification policies with password change option enabled - SCIM provisioned users (using user metadata) - Since all of the above are conditional based on at least a column inside a projection, a new `migration.CountTriggerConditional` has been added, where a condition (column values) and an option to track updates on that column should be considered for the count. - For this to be possible, the following changes had to be made to the existing sql resources: - the `resource_name` has been added to unique constraint on the `projection.resource_counts` table - triggers have been added / changed to individually track `INSERT`, `UPDATE`(s) and `DELETE` and be able to handle conditions - an optional argument has been added to the `projections.count_resource()` function to allow providing the information to `UP` or `DOWN` count the resource on an update. # Additional Changes None # Additional Context - partially solves #10244 (reporting audit log retention limit will be handled in #10245 directly) - backport to v4.x (cherry picked from commit 2dbe21fb300907ac430ad3badea1cda140b792a5)
2025-09-08 18:30:03 +02:00
TrackChange bool
Conditions TriggerConditions
feat(projections): resource counters (#9979) # Which Problems Are Solved Add the ability to keep track of the current counts of projection resources. We want to prevent calling `SELECT COUNT(*)` on tables, as that forces a full scan and sudden spikes of DB resource uses. # How the Problems Are Solved - A resource_counts table is added - Triggers that increment and decrement the counted values on inserts and deletes - Triggers that delete all counts of a table when the source table is TRUNCATEd. This is not in the business logic, but prevents wrong counts in case someone want to force a re-projection. - Triggers that delete all counts if the parent resource is deleted - Script to pre-populate the resource_counts table when a new source table is added. The triggers are reusable for any type of resource, in case we choose to add more in the future. Counts are aggregated by a given parent. Currently only `instance` and `organization` are defined as possible parent. This can later be extended to other types, such as `project`, should the need arise. I deliberately chose to use `parent_id` to distinguish from the de-factor `resource_owner` which is usually an organization ID. For example: - For users the parent is an organization and the `parent_id` matches `resource_owner`. - For organizations the parent is an instance, but the `resource_owner` is the `org_id`. In this case the `parent_id` is the `instance_id`. - Applications would have a similar problem, where the parent is a project, but the `resource_owner` is the `org_id` # Additional Context Closes https://github.com/zitadel/zitadel/issues/9957
2025-06-03 17:15:30 +03:00
}
type args struct {
lastRun map[string]any
}
tests := []struct {
name string
fields fields
args args
want bool
}{
{
name: "should",
fields: fields{
Table: "users2",
ParentType: "instance",
InstanceIDColumn: "instance_id",
ParentIDColumn: "parent_id",
Resource: "user",
feat(service ping): add additional resource counts (#10621) # Which Problems Are Solved Using the service ping, we want to have some additional insights to how zitadel is configured. The current resource count report contains already some amount of configured policies, such as the login_policy. But we do not know if for example MFA is enforced. # How the Problems Are Solved - Added the following counts to the report: - service users per organization - MFA enforcements (though login policy) - Notification policies with password change option enabled - SCIM provisioned users (using user metadata) - Since all of the above are conditional based on at least a column inside a projection, a new `migration.CountTriggerConditional` has been added, where a condition (column values) and an option to track updates on that column should be considered for the count. - For this to be possible, the following changes had to be made to the existing sql resources: - the `resource_name` has been added to unique constraint on the `projection.resource_counts` table - triggers have been added / changed to individually track `INSERT`, `UPDATE`(s) and `DELETE` and be able to handle conditions - an optional argument has been added to the `projections.count_resource()` function to allow providing the information to `UP` or `DOWN` count the resource on an update. # Additional Changes None # Additional Context - partially solves #10244 (reporting audit log retention limit will be handled in #10245 directly) - backport to v4.x (cherry picked from commit 2dbe21fb300907ac430ad3badea1cda140b792a5)
2025-09-08 18:30:03 +02:00
TrackChange: true,
Conditions: &OrCondition{
Conditions: []TriggerCondition{
{Column: "col1", Value: "value1"},
{Column: "col2", Value: "value1"},
},
},
feat(projections): resource counters (#9979) # Which Problems Are Solved Add the ability to keep track of the current counts of projection resources. We want to prevent calling `SELECT COUNT(*)` on tables, as that forces a full scan and sudden spikes of DB resource uses. # How the Problems Are Solved - A resource_counts table is added - Triggers that increment and decrement the counted values on inserts and deletes - Triggers that delete all counts of a table when the source table is TRUNCATEd. This is not in the business logic, but prevents wrong counts in case someone want to force a re-projection. - Triggers that delete all counts if the parent resource is deleted - Script to pre-populate the resource_counts table when a new source table is added. The triggers are reusable for any type of resource, in case we choose to add more in the future. Counts are aggregated by a given parent. Currently only `instance` and `organization` are defined as possible parent. This can later be extended to other types, such as `project`, should the need arise. I deliberately chose to use `parent_id` to distinguish from the de-factor `resource_owner` which is usually an organization ID. For example: - For users the parent is an organization and the `parent_id` matches `resource_owner`. - For organizations the parent is an instance, but the `resource_owner` is the `org_id`. In this case the `parent_id` is the `instance_id`. - Applications would have a similar problem, where the parent is a project, but the `resource_owner` is the `org_id` # Additional Context Closes https://github.com/zitadel/zitadel/issues/9957
2025-06-03 17:15:30 +03:00
},
args: args{
lastRun: map[string]any{
"table": "users1",
"parent_type": "instance",
"instance_id_column": "instance_id",
"parent_id_column": "parent_id",
"resource": "user",
feat(service ping): add additional resource counts (#10621) # Which Problems Are Solved Using the service ping, we want to have some additional insights to how zitadel is configured. The current resource count report contains already some amount of configured policies, such as the login_policy. But we do not know if for example MFA is enforced. # How the Problems Are Solved - Added the following counts to the report: - service users per organization - MFA enforcements (though login policy) - Notification policies with password change option enabled - SCIM provisioned users (using user metadata) - Since all of the above are conditional based on at least a column inside a projection, a new `migration.CountTriggerConditional` has been added, where a condition (column values) and an option to track updates on that column should be considered for the count. - For this to be possible, the following changes had to be made to the existing sql resources: - the `resource_name` has been added to unique constraint on the `projection.resource_counts` table - triggers have been added / changed to individually track `INSERT`, `UPDATE`(s) and `DELETE` and be able to handle conditions - an optional argument has been added to the `projections.count_resource()` function to allow providing the information to `UP` or `DOWN` count the resource on an update. # Additional Changes None # Additional Context - partially solves #10244 (reporting audit log retention limit will be handled in #10245 directly) - backport to v4.x (cherry picked from commit 2dbe21fb300907ac430ad3badea1cda140b792a5)
2025-09-08 18:30:03 +02:00
"track_change": true,
"conditions": map[string]any{
"orConditions": []any{
map[string]any{"column": "col1", "value": "value1"},
map[string]any{"column": "col2", "value": "value1"},
},
},
feat(projections): resource counters (#9979) # Which Problems Are Solved Add the ability to keep track of the current counts of projection resources. We want to prevent calling `SELECT COUNT(*)` on tables, as that forces a full scan and sudden spikes of DB resource uses. # How the Problems Are Solved - A resource_counts table is added - Triggers that increment and decrement the counted values on inserts and deletes - Triggers that delete all counts of a table when the source table is TRUNCATEd. This is not in the business logic, but prevents wrong counts in case someone want to force a re-projection. - Triggers that delete all counts if the parent resource is deleted - Script to pre-populate the resource_counts table when a new source table is added. The triggers are reusable for any type of resource, in case we choose to add more in the future. Counts are aggregated by a given parent. Currently only `instance` and `organization` are defined as possible parent. This can later be extended to other types, such as `project`, should the need arise. I deliberately chose to use `parent_id` to distinguish from the de-factor `resource_owner` which is usually an organization ID. For example: - For users the parent is an organization and the `parent_id` matches `resource_owner`. - For organizations the parent is an instance, but the `resource_owner` is the `org_id`. In this case the `parent_id` is the `instance_id`. - Applications would have a similar problem, where the parent is a project, but the `resource_owner` is the `org_id` # Additional Context Closes https://github.com/zitadel/zitadel/issues/9957
2025-06-03 17:15:30 +03:00
},
},
want: true,
},
{
name: "should not",
fields: fields{
Table: "users1",
ParentType: "instance",
InstanceIDColumn: "instance_id",
ParentIDColumn: "parent_id",
Resource: "user",
feat(service ping): add additional resource counts (#10621) # Which Problems Are Solved Using the service ping, we want to have some additional insights to how zitadel is configured. The current resource count report contains already some amount of configured policies, such as the login_policy. But we do not know if for example MFA is enforced. # How the Problems Are Solved - Added the following counts to the report: - service users per organization - MFA enforcements (though login policy) - Notification policies with password change option enabled - SCIM provisioned users (using user metadata) - Since all of the above are conditional based on at least a column inside a projection, a new `migration.CountTriggerConditional` has been added, where a condition (column values) and an option to track updates on that column should be considered for the count. - For this to be possible, the following changes had to be made to the existing sql resources: - the `resource_name` has been added to unique constraint on the `projection.resource_counts` table - triggers have been added / changed to individually track `INSERT`, `UPDATE`(s) and `DELETE` and be able to handle conditions - an optional argument has been added to the `projections.count_resource()` function to allow providing the information to `UP` or `DOWN` count the resource on an update. # Additional Changes None # Additional Context - partially solves #10244 (reporting audit log retention limit will be handled in #10245 directly) - backport to v4.x (cherry picked from commit 2dbe21fb300907ac430ad3badea1cda140b792a5)
2025-09-08 18:30:03 +02:00
TrackChange: true,
Conditions: &AndCondition{
Conditions: []TriggerCondition{
{Column: "col1", Value: "value1"},
{Column: "col2", Value: "value1"},
},
},
feat(projections): resource counters (#9979) # Which Problems Are Solved Add the ability to keep track of the current counts of projection resources. We want to prevent calling `SELECT COUNT(*)` on tables, as that forces a full scan and sudden spikes of DB resource uses. # How the Problems Are Solved - A resource_counts table is added - Triggers that increment and decrement the counted values on inserts and deletes - Triggers that delete all counts of a table when the source table is TRUNCATEd. This is not in the business logic, but prevents wrong counts in case someone want to force a re-projection. - Triggers that delete all counts if the parent resource is deleted - Script to pre-populate the resource_counts table when a new source table is added. The triggers are reusable for any type of resource, in case we choose to add more in the future. Counts are aggregated by a given parent. Currently only `instance` and `organization` are defined as possible parent. This can later be extended to other types, such as `project`, should the need arise. I deliberately chose to use `parent_id` to distinguish from the de-factor `resource_owner` which is usually an organization ID. For example: - For users the parent is an organization and the `parent_id` matches `resource_owner`. - For organizations the parent is an instance, but the `resource_owner` is the `org_id`. In this case the `parent_id` is the `instance_id`. - Applications would have a similar problem, where the parent is a project, but the `resource_owner` is the `org_id` # Additional Context Closes https://github.com/zitadel/zitadel/issues/9957
2025-06-03 17:15:30 +03:00
},
args: args{
lastRun: map[string]any{
"table": "users1",
"parent_type": "instance",
"instance_id_column": "instance_id",
"parent_id_column": "parent_id",
"resource": "user",
feat(service ping): add additional resource counts (#10621) # Which Problems Are Solved Using the service ping, we want to have some additional insights to how zitadel is configured. The current resource count report contains already some amount of configured policies, such as the login_policy. But we do not know if for example MFA is enforced. # How the Problems Are Solved - Added the following counts to the report: - service users per organization - MFA enforcements (though login policy) - Notification policies with password change option enabled - SCIM provisioned users (using user metadata) - Since all of the above are conditional based on at least a column inside a projection, a new `migration.CountTriggerConditional` has been added, where a condition (column values) and an option to track updates on that column should be considered for the count. - For this to be possible, the following changes had to be made to the existing sql resources: - the `resource_name` has been added to unique constraint on the `projection.resource_counts` table - triggers have been added / changed to individually track `INSERT`, `UPDATE`(s) and `DELETE` and be able to handle conditions - an optional argument has been added to the `projections.count_resource()` function to allow providing the information to `UP` or `DOWN` count the resource on an update. # Additional Changes None # Additional Context - partially solves #10244 (reporting audit log retention limit will be handled in #10245 directly) - backport to v4.x (cherry picked from commit 2dbe21fb300907ac430ad3badea1cda140b792a5)
2025-09-08 18:30:03 +02:00
"track_change": true,
"conditions": map[string]any{
"andConditions": []any{
map[string]any{"column": "col1", "value": "value1"},
map[string]any{"column": "col2", "value": "value1"},
},
},
feat(projections): resource counters (#9979) # Which Problems Are Solved Add the ability to keep track of the current counts of projection resources. We want to prevent calling `SELECT COUNT(*)` on tables, as that forces a full scan and sudden spikes of DB resource uses. # How the Problems Are Solved - A resource_counts table is added - Triggers that increment and decrement the counted values on inserts and deletes - Triggers that delete all counts of a table when the source table is TRUNCATEd. This is not in the business logic, but prevents wrong counts in case someone want to force a re-projection. - Triggers that delete all counts if the parent resource is deleted - Script to pre-populate the resource_counts table when a new source table is added. The triggers are reusable for any type of resource, in case we choose to add more in the future. Counts are aggregated by a given parent. Currently only `instance` and `organization` are defined as possible parent. This can later be extended to other types, such as `project`, should the need arise. I deliberately chose to use `parent_id` to distinguish from the de-factor `resource_owner` which is usually an organization ID. For example: - For users the parent is an organization and the `parent_id` matches `resource_owner`. - For organizations the parent is an instance, but the `resource_owner` is the `org_id`. In this case the `parent_id` is the `instance_id`. - Applications would have a similar problem, where the parent is a project, but the `resource_owner` is the `org_id` # Additional Context Closes https://github.com/zitadel/zitadel/issues/9957
2025-06-03 17:15:30 +03:00
},
},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := &triggerConfig{
Table: tt.fields.Table,
ParentType: tt.fields.ParentType,
InstanceIDColumn: tt.fields.InstanceIDColumn,
ParentIDColumn: tt.fields.ParentIDColumn,
Resource: tt.fields.Resource,
feat(service ping): add additional resource counts (#10621) # Which Problems Are Solved Using the service ping, we want to have some additional insights to how zitadel is configured. The current resource count report contains already some amount of configured policies, such as the login_policy. But we do not know if for example MFA is enforced. # How the Problems Are Solved - Added the following counts to the report: - service users per organization - MFA enforcements (though login policy) - Notification policies with password change option enabled - SCIM provisioned users (using user metadata) - Since all of the above are conditional based on at least a column inside a projection, a new `migration.CountTriggerConditional` has been added, where a condition (column values) and an option to track updates on that column should be considered for the count. - For this to be possible, the following changes had to be made to the existing sql resources: - the `resource_name` has been added to unique constraint on the `projection.resource_counts` table - triggers have been added / changed to individually track `INSERT`, `UPDATE`(s) and `DELETE` and be able to handle conditions - an optional argument has been added to the `projections.count_resource()` function to allow providing the information to `UP` or `DOWN` count the resource on an update. # Additional Changes None # Additional Context - partially solves #10244 (reporting audit log retention limit will be handled in #10245 directly) - backport to v4.x (cherry picked from commit 2dbe21fb300907ac430ad3badea1cda140b792a5)
2025-09-08 18:30:03 +02:00
TrackChange: tt.fields.TrackChange,
Conditions: tt.fields.Conditions,
feat(projections): resource counters (#9979) # Which Problems Are Solved Add the ability to keep track of the current counts of projection resources. We want to prevent calling `SELECT COUNT(*)` on tables, as that forces a full scan and sudden spikes of DB resource uses. # How the Problems Are Solved - A resource_counts table is added - Triggers that increment and decrement the counted values on inserts and deletes - Triggers that delete all counts of a table when the source table is TRUNCATEd. This is not in the business logic, but prevents wrong counts in case someone want to force a re-projection. - Triggers that delete all counts if the parent resource is deleted - Script to pre-populate the resource_counts table when a new source table is added. The triggers are reusable for any type of resource, in case we choose to add more in the future. Counts are aggregated by a given parent. Currently only `instance` and `organization` are defined as possible parent. This can later be extended to other types, such as `project`, should the need arise. I deliberately chose to use `parent_id` to distinguish from the de-factor `resource_owner` which is usually an organization ID. For example: - For users the parent is an organization and the `parent_id` matches `resource_owner`. - For organizations the parent is an instance, but the `resource_owner` is the `org_id`. In this case the `parent_id` is the `instance_id`. - Applications would have a similar problem, where the parent is a project, but the `resource_owner` is the `org_id` # Additional Context Closes https://github.com/zitadel/zitadel/issues/9957
2025-06-03 17:15:30 +03:00
}
got := c.Check(tt.args.lastRun)
assert.Equal(t, tt.want, got)
})
}
}
feat(service ping): add additional resource counts (#10621) # Which Problems Are Solved Using the service ping, we want to have some additional insights to how zitadel is configured. The current resource count report contains already some amount of configured policies, such as the login_policy. But we do not know if for example MFA is enforced. # How the Problems Are Solved - Added the following counts to the report: - service users per organization - MFA enforcements (though login policy) - Notification policies with password change option enabled - SCIM provisioned users (using user metadata) - Since all of the above are conditional based on at least a column inside a projection, a new `migration.CountTriggerConditional` has been added, where a condition (column values) and an option to track updates on that column should be considered for the count. - For this to be possible, the following changes had to be made to the existing sql resources: - the `resource_name` has been added to unique constraint on the `projection.resource_counts` table - triggers have been added / changed to individually track `INSERT`, `UPDATE`(s) and `DELETE` and be able to handle conditions - an optional argument has been added to the `projections.count_resource()` function to allow providing the information to `UP` or `DOWN` count the resource on an update. # Additional Changes None # Additional Context - partially solves #10244 (reporting audit log retention limit will be handled in #10245 directly) - backport to v4.x (cherry picked from commit 2dbe21fb300907ac430ad3badea1cda140b792a5)
2025-09-08 18:30:03 +02:00
func Test_TriggerConditions_ToSQL(t *testing.T) {
type fields struct {
conditions TriggerConditions
}
type args struct {
table string
conditionsMet bool
}
tests := []struct {
name string
fields fields
args args
want string
}{
{
name: "single condition",
fields: fields{
conditions: TriggerCondition{
Column: "col1",
Value: "value1",
},
},
args: args{
table: "table",
conditionsMet: true,
},
want: "table.col1 = 'value1'",
},
{
name: "single condition not met",
fields: fields{
conditions: TriggerCondition{
Column: "col1",
Value: "value1",
},
},
args: args{
table: "table",
conditionsMet: false,
},
want: "table.col1 <> 'value1'",
},
{
name: "or condition",
fields: fields{
conditions: OrCondition{
Conditions: []TriggerCondition{
{Column: "col1", Value: "value1"},
{Column: "col2", Value: "value1"},
},
},
},
args: args{
table: "table",
conditionsMet: true,
},
want: "(table.col1 = 'value1' OR table.col2 = 'value1')",
},
{
name: "or condition not met",
fields: fields{
conditions: OrCondition{
Conditions: []TriggerCondition{
{Column: "col1", Value: "value1"},
{Column: "col2", Value: "value1"},
},
},
},
args: args{
table: "table",
conditionsMet: false,
},
want: "(table.col1 <> 'value1' AND table.col2 <> 'value1')",
},
{
name: "and condition",
fields: fields{
conditions: AndCondition{
Conditions: []TriggerCondition{
{Column: "col1", Value: "value1"},
{Column: "col2", Value: "value1"},
},
},
},
args: args{
table: "table",
conditionsMet: true,
},
want: "(table.col1 = 'value1' AND table.col2 = 'value1')",
},
{
name: "and condition not met",
fields: fields{
conditions: AndCondition{
Conditions: []TriggerCondition{
{Column: "col1", Value: "value1"},
{Column: "col2", Value: "value1"},
},
},
},
args: args{
table: "table",
conditionsMet: false,
},
want: "(table.col1 <> 'value1' OR table.col2 <> 'value1')",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.fields.conditions.ToSQL(tt.args.table, tt.args.conditionsMet)
assert.Equal(t, tt.want, got)
})
}
}