[MM-62553]+[MM-62554] Property Architecture: cursor based pagination (#30119)

* refactor: Replace pagination with cursor-based pagination for custom profile attributes

* remove pagination loop on property value retrieval for CPA

* add migrations to optimize pagination on property fields and values

* adapt test to remove pagination check

* update migrations list

* postgres: drop index concurrently

* concurrent index manipulation must be done outside of a Tx

* fix: Correct SQL index drop syntax from "OM" to "ON" in migration files

* test: Add CountForGroup test cases for property field store

* refactor: Add CountForGroup method to PropertyFieldStore interface and implementations

* Fix style and i18n

* feat: Add optional deleted property field filtering to CountForGroup method

* refactor: Update CountForGroup to support optional deleted property fields

* test: Add comprehensive tests for CountForGroup with includeDeleted parameter

* adapt test + gen layers

* rename property service method and set the includeDelete to false

* refactor: Remove redundant constant and use CustomProfileAttributesFieldLimit directly

* fix tests

---------

Co-authored-by: Mattermost Build <build@mattermost.com>

server/channels/store/retrylayer/retrylayer.go

@@ -8949,6 +8949,27 @@ func (s *RetryLayerProductNoticesStore) View(userID string, notices []string) er
 
 }
 
+func (s *RetryLayerPropertyFieldStore) CountForGroup(groupID string, includeDeleted bool) (int64, error) {
+
+	tries := 0
+	for {
+		result, err := s.PropertyFieldStore.CountForGroup(groupID, includeDeleted)
+		if err == nil {
+			return result, nil
+		}
+		if !isRepeatableError(err) {
+			return result, err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return result, err
+		}
+		timepkg.Sleep(100 * timepkg.Millisecond)
+	}
+
+}
+
 func (s *RetryLayerPropertyFieldStore) Create(field *model.PropertyField) (*model.PropertyField, error) {
 
 	tries := 0

server/channels/store/sqlstore/property_field_store.go

@@ -78,9 +78,26 @@ func (s *SqlPropertyFieldStore) GetMany(ids []string) ([]*model.PropertyField, e
 	return fields, nil
 }
 
+func (s *SqlPropertyFieldStore) CountForGroup(groupID string, includeDeleted bool) (int64, error) {
+	var count int64
+	builder := s.getQueryBuilder().
+		Select("COUNT(id)").
+		From("PropertyFields").
+		Where(sq.Eq{"GroupID": groupID})
+
+	if !includeDeleted {
+		builder = builder.Where(sq.Eq{"DeleteAt": 0})
+	}
+
+	if err := s.GetReplica().GetBuilder(&count, builder); err != nil {
+		return int64(0), errors.Wrap(err, "failed to count Sessions")
+	}
+	return count, nil
+}
+
 func (s *SqlPropertyFieldStore) SearchPropertyFields(opts model.PropertyFieldSearchOpts) ([]*model.PropertyField, error) {
-	if opts.Page < 0 {
-		return nil, errors.New("page must be positive integer")
+	if err := opts.Cursor.IsValid(); err != nil {
+		return nil, fmt.Errorf("cursor is invalid: %w", err)
 	}
 
 	if opts.PerPage < 1 {
@@ -88,10 +105,19 @@ func (s *SqlPropertyFieldStore) SearchPropertyFields(opts model.PropertyFieldSea
 	}
 
 	builder := s.tableSelectQuery.
-		OrderBy("CreateAt ASC").
-		Offset(uint64(opts.Page * opts.PerPage)).
+		OrderBy("CreateAt ASC, Id ASC").
 		Limit(uint64(opts.PerPage))
 
+	if !opts.Cursor.IsEmpty() {
+		builder = builder.Where(sq.Or{
+			sq.Gt{"CreateAt": opts.Cursor.CreateAt},
+			sq.And{
+				sq.Eq{"CreateAt": opts.Cursor.CreateAt},
+				sq.Gt{"Id": opts.Cursor.PropertyFieldID},
+			},
+		})
+	}
+
 	if !opts.IncludeDeleted {
 		builder = builder.Where(sq.Eq{"DeleteAt": 0})
 	}

server/channels/store/sqlstore/property_value_store.go

@@ -83,8 +83,8 @@ func (s *SqlPropertyValueStore) GetMany(ids []string) ([]*model.PropertyValue, e
 }
 
 func (s *SqlPropertyValueStore) SearchPropertyValues(opts model.PropertyValueSearchOpts) ([]*model.PropertyValue, error) {
-	if opts.Page < 0 {
-		return nil, errors.New("page must be positive integer")
+	if err := opts.Cursor.IsValid(); err != nil {
+		return nil, fmt.Errorf("cursor is invalid: %w", err)
 	}
 
 	if opts.PerPage < 1 {
@@ -92,10 +92,19 @@ func (s *SqlPropertyValueStore) SearchPropertyValues(opts model.PropertyValueSea
 	}
 
 	builder := s.tableSelectQuery.
-		OrderBy("CreateAt ASC").
-		Offset(uint64(opts.Page * opts.PerPage)).
+		OrderBy("CreateAt ASC, Id ASC").
 		Limit(uint64(opts.PerPage))
 
+	if !opts.Cursor.IsEmpty() {
+		builder = builder.Where(sq.Or{
+			sq.Gt{"CreateAt": opts.Cursor.CreateAt},
+			sq.And{
+				sq.Eq{"CreateAt": opts.Cursor.CreateAt},
+				sq.Gt{"Id": opts.Cursor.PropertyValueID},
+			},
+		})
+	}
+
 	if !opts.IncludeDeleted {
 		builder = builder.Where(sq.Eq{"DeleteAt": 0})
 	}

server/channels/store/store.go

@@ -1089,6 +1089,7 @@ type PropertyFieldStore interface {
 	Create(field *model.PropertyField) (*model.PropertyField, error)
 	Get(id string) (*model.PropertyField, error)
 	GetMany(ids []string) ([]*model.PropertyField, error)
+	CountForGroup(groupID string, includeDeleted bool) (int64, error)
 	SearchPropertyFields(opts model.PropertyFieldSearchOpts) ([]*model.PropertyField, error)
 	Update(fields []*model.PropertyField) ([]*model.PropertyField, error)
 	Delete(id string) error

server/channels/store/storetest/mocks/PropertyFieldStore.go

@@ -14,6 +14,34 @@ type PropertyFieldStore struct {
 	mock.Mock
 }
 
+// CountForGroup provides a mock function with given fields: groupID, includeDeleted
+func (_m *PropertyFieldStore) CountForGroup(groupID string, includeDeleted bool) (int64, error) {
+	ret := _m.Called(groupID, includeDeleted)
+
+	if len(ret) == 0 {
+		panic("no return value specified for CountForGroup")
+	}
+
+	var r0 int64
+	var r1 error
+	if rf, ok := ret.Get(0).(func(string, bool) (int64, error)); ok {
+		return rf(groupID, includeDeleted)
+	}
+	if rf, ok := ret.Get(0).(func(string, bool) int64); ok {
+		r0 = rf(groupID, includeDeleted)
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	if rf, ok := ret.Get(1).(func(string, bool) error); ok {
+		r1 = rf(groupID, includeDeleted)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // Create provides a mock function with given fields: field
 func (_m *PropertyFieldStore) Create(field *model.PropertyField) (*model.PropertyField, error) {
 	ret := _m.Called(field)

server/channels/store/storetest/property_field_store.go

@@ -5,6 +5,7 @@ package storetest
 
 import (
 	"database/sql"
+	"fmt"
 	"testing"
 	"time"
 
@@ -21,6 +22,7 @@ func TestPropertyFieldStore(t *testing.T, rctx request.CTX, ss store.Store, s Sq
 	t.Run("UpdatePropertyField", func(t *testing.T) { testUpdatePropertyField(t, rctx, ss) })
 	t.Run("DeletePropertyField", func(t *testing.T) { testDeletePropertyField(t, rctx, ss) })
 	t.Run("SearchPropertyFields", func(t *testing.T) { testSearchPropertyFields(t, rctx, ss) })
+	t.Run("CountForGroup", func(t *testing.T) { testCountForGroup(t, rctx, ss) })
 }
 
 func testCreatePropertyField(t *testing.T, _ request.CTX, ss store.Store) {
@@ -356,6 +358,97 @@ func testDeletePropertyField(t *testing.T, _ request.CTX, ss store.Store) {
 	})
 }
 
+func testCountForGroup(t *testing.T, _ request.CTX, ss store.Store) {
+	t.Run("should return 0 for group with no properties", func(t *testing.T) {
+		count, err := ss.PropertyField().CountForGroup(model.NewId(), false)
+		require.NoError(t, err)
+		require.Equal(t, int64(0), count)
+	})
+
+	t.Run("should return correct count for group with properties", func(t *testing.T) {
+		groupID := model.NewId()
+
+		// Create 5 property fields
+		for i := 0; i < 5; i++ {
+			field := &model.PropertyField{
+				GroupID: groupID,
+				Name:    fmt.Sprintf("Field %d", i),
+				Type:    model.PropertyFieldTypeText,
+			}
+			_, err := ss.PropertyField().Create(field)
+			require.NoError(t, err)
+		}
+
+		count, err := ss.PropertyField().CountForGroup(groupID, false)
+		require.NoError(t, err)
+		require.Equal(t, int64(5), count)
+	})
+
+	t.Run("should not count deleted properties when includeDeleted is false", func(t *testing.T) {
+		groupID := model.NewId()
+
+		// Create 5 property fields
+		for i := 0; i < 5; i++ {
+			field := &model.PropertyField{
+				GroupID: groupID,
+				Name:    fmt.Sprintf("Field %d", i),
+				Type:    model.PropertyFieldTypeText,
+			}
+			_, err := ss.PropertyField().Create(field)
+			require.NoError(t, err)
+		}
+
+		// Create one more and delete it
+		deletedField := &model.PropertyField{
+			GroupID: groupID,
+			Name:    "To be deleted",
+			Type:    model.PropertyFieldTypeText,
+		}
+		_, err := ss.PropertyField().Create(deletedField)
+		require.NoError(t, err)
+
+		err = ss.PropertyField().Delete(deletedField.ID)
+		require.NoError(t, err)
+
+		// Count should be 5 since the deleted field shouldn't be counted
+		count, err := ss.PropertyField().CountForGroup(groupID, false)
+		require.NoError(t, err)
+		require.Equal(t, int64(5), count)
+	})
+
+	t.Run("should count deleted properties when includeDeleted is true", func(t *testing.T) {
+		groupID := model.NewId()
+
+		// Create 5 property fields
+		for i := 0; i < 5; i++ {
+			field := &model.PropertyField{
+				GroupID: groupID,
+				Name:    fmt.Sprintf("Field %d", i),
+				Type:    model.PropertyFieldTypeText,
+			}
+			_, err := ss.PropertyField().Create(field)
+			require.NoError(t, err)
+		}
+
+		// Create one more and delete it
+		deletedField := &model.PropertyField{
+			GroupID: groupID,
+			Name:    "To be deleted",
+			Type:    model.PropertyFieldTypeText,
+		}
+		_, err := ss.PropertyField().Create(deletedField)
+		require.NoError(t, err)
+
+		err = ss.PropertyField().Delete(deletedField.ID)
+		require.NoError(t, err)
+
+		// Count should be 6 since we're including deleted fields
+		count, err := ss.PropertyField().CountForGroup(groupID, true)
+		require.NoError(t, err)
+		require.Equal(t, int64(6), count)
+	})
+}
+
 func testSearchPropertyFields(t *testing.T, _ request.CTX, ss store.Store) {
 	groupID := model.NewId()
 	targetID := model.NewId()
@@ -406,18 +499,9 @@ func testSearchPropertyFields(t *testing.T, _ request.CTX, ss store.Store) {
 		expectedError bool
 		expectedIDs   []string
 	}{
-		{
-			name: "negative page",
-			opts: model.PropertyFieldSearchOpts{
-				Page:    -1,
-				PerPage: 10,
-			},
-			expectedError: true,
-		},
 		{
 			name: "negative per_page",
 			opts: model.PropertyFieldSearchOpts{
-				Page:    0,
 				PerPage: -1,
 			},
 			expectedError: true,
@@ -426,7 +510,6 @@ func testSearchPropertyFields(t *testing.T, _ request.CTX, ss store.Store) {
 			name: "filter by group_id",
 			opts: model.PropertyFieldSearchOpts{
 				GroupID: groupID,
-				Page:    0,
 				PerPage: 10,
 			},
 			expectedIDs: []string{field1.ID, field2.ID},
@@ -435,7 +518,6 @@ func testSearchPropertyFields(t *testing.T, _ request.CTX, ss store.Store) {
 			name: "filter by group_id including deleted",
 			opts: model.PropertyFieldSearchOpts{
 				GroupID:        groupID,
-				Page:           0,
 				PerPage:        10,
 				IncludeDeleted: true,
 			},
@@ -445,7 +527,6 @@ func testSearchPropertyFields(t *testing.T, _ request.CTX, ss store.Store) {
 			name: "filter by target_type",
 			opts: model.PropertyFieldSearchOpts{
 				TargetType: "test_type",
-				Page:       0,
 				PerPage:    10,
 			},
 			expectedIDs: []string{field1.ID, field3.ID},
@@ -454,7 +535,6 @@ func testSearchPropertyFields(t *testing.T, _ request.CTX, ss store.Store) {
 			name: "filter by target_id",
 			opts: model.PropertyFieldSearchOpts{
 				TargetID: targetID,
-				Page:     0,
 				PerPage:  10,
 			},
 			expectedIDs: []string{field1.ID, field2.ID},
@@ -463,7 +543,6 @@ func testSearchPropertyFields(t *testing.T, _ request.CTX, ss store.Store) {
 			name: "pagination page 0",
 			opts: model.PropertyFieldSearchOpts{
 				GroupID:        groupID,
-				Page:           0,
 				PerPage:        2,
 				IncludeDeleted: true,
 			},
@@ -472,8 +551,11 @@ func testSearchPropertyFields(t *testing.T, _ request.CTX, ss store.Store) {
 		{
 			name: "pagination page 1",
 			opts: model.PropertyFieldSearchOpts{
-				GroupID:        groupID,
-				Page:           1,
+				GroupID: groupID,
+				Cursor: model.PropertyFieldSearchCursor{
+					CreateAt:        field2.CreateAt,
+					PropertyFieldID: field2.ID,
+				},
 				PerPage:        2,
 				IncludeDeleted: true,
 			},
@@ -490,7 +572,7 @@ func testSearchPropertyFields(t *testing.T, _ request.CTX, ss store.Store) {
 			}
 
 			require.NoError(t, err)
-			var ids = make([]string, len(results))
+			ids := make([]string, len(results))
 			for i, field := range results {
 				ids[i] = field.ID
 			}

server/channels/store/storetest/property_value_store.go

@@ -567,18 +567,9 @@ func testSearchPropertyValues(t *testing.T, _ request.CTX, ss store.Store) {
 		expectedError bool
 		expectedIDs   []string
 	}{
-		{
-			name: "negative page",
-			opts: model.PropertyValueSearchOpts{
-				Page:    -1,
-				PerPage: 10,
-			},
-			expectedError: true,
-		},
 		{
 			name: "negative per_page",
 			opts: model.PropertyValueSearchOpts{
-				Page:    0,
 				PerPage: -1,
 			},
 			expectedError: true,
@@ -587,7 +578,6 @@ func testSearchPropertyValues(t *testing.T, _ request.CTX, ss store.Store) {
 			name: "filter by group_id",
 			opts: model.PropertyValueSearchOpts{
 				GroupID: groupID,
-				Page:    0,
 				PerPage: 10,
 			},
 			expectedIDs: []string{value1.ID, value2.ID},
@@ -597,7 +587,6 @@ func testSearchPropertyValues(t *testing.T, _ request.CTX, ss store.Store) {
 			opts: model.PropertyValueSearchOpts{
 				GroupID:    groupID,
 				TargetType: "test_type",
-				Page:       0,
 				PerPage:    10,
 			},
 			expectedIDs: []string{value1.ID},
@@ -608,7 +597,6 @@ func testSearchPropertyValues(t *testing.T, _ request.CTX, ss store.Store) {
 				GroupID:        groupID,
 				TargetType:     "test_type",
 				IncludeDeleted: true,
-				Page:           0,
 				PerPage:        10,
 			},
 			expectedIDs: []string{value1.ID, value4.ID},
@@ -617,7 +605,6 @@ func testSearchPropertyValues(t *testing.T, _ request.CTX, ss store.Store) {
 			name: "filter by target_id",
 			opts: model.PropertyValueSearchOpts{
 				TargetID: targetID,
-				Page:     0,
 				PerPage:  10,
 			},
 			expectedIDs: []string{value1.ID, value2.ID},
@@ -627,7 +614,6 @@ func testSearchPropertyValues(t *testing.T, _ request.CTX, ss store.Store) {
 			opts: model.PropertyValueSearchOpts{
 				GroupID:  groupID,
 				TargetID: targetID,
-				Page:     0,
 				PerPage:  10,
 			},
 			expectedIDs: []string{value1.ID, value2.ID},
@@ -636,7 +622,6 @@ func testSearchPropertyValues(t *testing.T, _ request.CTX, ss store.Store) {
 			name: "filter by field_id",
 			opts: model.PropertyValueSearchOpts{
 				FieldID: fieldID,
-				Page:    0,
 				PerPage: 10,
 			},
 			expectedIDs: []string{value1.ID},
@@ -646,7 +631,6 @@ func testSearchPropertyValues(t *testing.T, _ request.CTX, ss store.Store) {
 			opts: model.PropertyValueSearchOpts{
 				FieldID:        fieldID,
 				IncludeDeleted: true,
-				Page:           0,
 				PerPage:        10,
 			},
 			expectedIDs: []string{value1.ID, value4.ID},
@@ -655,7 +639,6 @@ func testSearchPropertyValues(t *testing.T, _ request.CTX, ss store.Store) {
 			name: "pagination page 0",
 			opts: model.PropertyValueSearchOpts{
 				GroupID: groupID,
-				Page:    0,
 				PerPage: 1,
 			},
 			expectedIDs: []string{value1.ID},
@@ -664,7 +647,10 @@ func testSearchPropertyValues(t *testing.T, _ request.CTX, ss store.Store) {
 			name: "pagination page 1",
 			opts: model.PropertyValueSearchOpts{
 				GroupID: groupID,
-				Page:    1,
+				Cursor: model.PropertyValueSearchCursor{
+					CreateAt:        value1.CreateAt,
+					PropertyValueID: value1.ID,
+				},
 				PerPage: 1,
 			},
 			expectedIDs: []string{value2.ID},
@@ -680,7 +666,7 @@ func testSearchPropertyValues(t *testing.T, _ request.CTX, ss store.Store) {
 			}
 
 			require.NoError(t, err)
-			var ids = make([]string, len(results))
+			ids := make([]string, len(results))
 			for i, value := range results {
 				ids[i] = value.ID
 			}

server/channels/store/timerlayer/timerlayer.go

@@ -7105,6 +7105,22 @@ func (s *TimerLayerProductNoticesStore) View(userID string, notices []string) er
 	return err
 }
 
+func (s *TimerLayerPropertyFieldStore) CountForGroup(groupID string, includeDeleted bool) (int64, error) {
+	start := time.Now()
+
+	result, err := s.PropertyFieldStore.CountForGroup(groupID, includeDeleted)
+
+	elapsed := float64(time.Since(start)) / float64(time.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("PropertyFieldStore.CountForGroup", success, elapsed)
+	}
+	return result, err
+}
+
 func (s *TimerLayerPropertyFieldStore) Create(field *model.PropertyField) (*model.PropertyField, error) {
 	start := time.Now()