[MM-62553]+[MM-62554] Property Architecture: cursor based pagination (#30119)

* refactor: Replace pagination with cursor-based pagination for custom profile attributes

* remove pagination loop on property value retrieval for CPA

* add migrations to optimize pagination on property fields and values

* adapt test to remove pagination check

* update migrations list

* postgres: drop index concurrently

* concurrent index manipulation must be done outside of a Tx

* fix: Correct SQL index drop syntax from "OM" to "ON" in migration files

* test: Add CountForGroup test cases for property field store

* refactor: Add CountForGroup method to PropertyFieldStore interface and implementations

* Fix style and i18n

* feat: Add optional deleted property field filtering to CountForGroup method

* refactor: Update CountForGroup to support optional deleted property fields

* test: Add comprehensive tests for CountForGroup with includeDeleted parameter

* adapt test + gen layers

* rename property service method and set the includeDelete to false

* refactor: Remove redundant constant and use CustomProfileAttributesFieldLimit directly

* fix tests

---------

Co-authored-by: Mattermost Build <build@mattermost.com>

server/channels/db/migrations/migrations.list

@@ -257,6 +257,10 @@ channels/db/migrations/mysql/000129_add_property_system_architecture.down.sql
 channels/db/migrations/mysql/000129_add_property_system_architecture.up.sql
 channels/db/migrations/mysql/000130_system_console_stats.down.sql
 channels/db/migrations/mysql/000130_system_console_stats.up.sql
+channels/db/migrations/mysql/000131_create_index_pagination_on_property_values.down.sql
+channels/db/migrations/mysql/000131_create_index_pagination_on_property_values.up.sql
+channels/db/migrations/mysql/000132_create_index_pagination_on_property_fields.down.sql
+channels/db/migrations/mysql/000132_create_index_pagination_on_property_fields.up.sql
 channels/db/migrations/postgres/000001_create_teams.down.sql
 channels/db/migrations/postgres/000001_create_teams.up.sql
 channels/db/migrations/postgres/000002_create_team_members.down.sql
@@ -515,3 +519,7 @@ channels/db/migrations/postgres/000129_add_property_system_architecture.down.sql
 channels/db/migrations/postgres/000129_add_property_system_architecture.up.sql
 channels/db/migrations/postgres/000130_system_console_stats.down.sql
 channels/db/migrations/postgres/000130_system_console_stats.up.sql
+channels/db/migrations/postgres/000131_create_index_pagination_on_property_values.down.sql
+channels/db/migrations/postgres/000131_create_index_pagination_on_property_values.up.sql
+channels/db/migrations/postgres/000132_create_index_pagination_on_property_fields.down.sql
+channels/db/migrations/postgres/000132_create_index_pagination_on_property_fields.up.sql

server/channels/db/migrations/mysql/000131_create_index_pagination_on_property_values.down.sql

@@ -0,0 +1,14 @@
+SET @preparedStatement = (SELECT IF(
+    (
+        SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
+        WHERE table_name = 'PropertyValues'
+        AND table_schema = DATABASE()
+        AND index_name = 'idx_propertyvalues_create_at_id'
+    ) > 0,
+    'DROP INDEX idx_propertyvalues_create_at_id ON PropertyValues;',
+    'SELECT 1'
+));
+
+PREPARE removeIndexIfExists FROM @preparedStatement;
+EXECUTE removeIndexIfExists;
+DEALLOCATE PREPARE removeIndexIfExists;

server/channels/db/migrations/mysql/000131_create_index_pagination_on_property_values.up.sql

@@ -0,0 +1,14 @@
+SET @preparedStatement = (SELECT IF(
+    (
+        SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
+        WHERE table_name = 'PropertyValues'
+        AND table_schema = DATABASE()
+        AND index_name = 'idx_propertyvalues_create_at_id'
+    ) > 0,
+    'SELECT 1',
+    'CREATE INDEX idx_propertyvalues_create_at_id ON PropertyValues(CreateAt, ID);'
+));
+
+PREPARE createIndexIfNotExists FROM @preparedStatement;
+EXECUTE createIndexIfNotExists;
+DEALLOCATE PREPARE createIndexIfNotExists;

server/channels/db/migrations/mysql/000132_create_index_pagination_on_property_fields.down.sql

@@ -0,0 +1,14 @@
+SET @preparedStatement = (SELECT IF(
+    (
+        SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
+        WHERE table_name = 'PropertyFields'
+        AND table_schema = DATABASE()
+        AND index_name = 'idx_propertyfields_create_at_id'
+    ) > 0,
+    'DROP INDEX idx_propertyfields_create_at_id ON PropertyFields;',
+    'SELECT 1'
+));
+
+PREPARE removeIndexIfExists FROM @preparedStatement;
+EXECUTE removeIndexIfExists;
+DEALLOCATE PREPARE removeIndexIfExists;

server/channels/db/migrations/mysql/000132_create_index_pagination_on_property_fields.up.sql

@@ -0,0 +1,14 @@
+SET @preparedStatement = (SELECT IF(
+    (
+        SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
+        WHERE table_name = 'PropertyFields'
+        AND table_schema = DATABASE()
+        AND index_name = 'idx_propertyfields_create_at_id'
+    ) > 0,
+    'SELECT 1',
+    'CREATE INDEX idx_propertyfields_create_at_id ON PropertyFields(CreateAt, ID);'
+));
+
+PREPARE createIndexIfNotExists FROM @preparedStatement;
+EXECUTE createIndexIfNotExists;
+DEALLOCATE PREPARE createIndexIfNotExists;

server/channels/db/migrations/postgres/000131_create_index_pagination_on_property_values.down.sql

@@ -0,0 +1,2 @@
+-- morph:nontransactional
+DROP INDEX CONCURRENTLY IF EXISTS idx_propertyvalues_create_at_id;

server/channels/db/migrations/postgres/000131_create_index_pagination_on_property_values.up.sql

@@ -0,0 +1,2 @@
+-- morph:nontransactional
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_propertyvalues_create_at_id ON PropertyValues(CreateAt, ID)

server/channels/db/migrations/postgres/000132_create_index_pagination_on_property_fields.down.sql

@@ -0,0 +1,2 @@
+-- morph:nontransactional
+DROP INDEX CONCURRENTLY IF EXISTS idx_propertyfields_create_at_id;

server/channels/db/migrations/postgres/000132_create_index_pagination_on_property_fields.up.sql

@@ -0,0 +1,2 @@
+-- morph:nontransactional
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_propertyfields_create_at_id ON PropertyFields(CreateAt, ID)