[MM-30831] granular data retention wireup (#17417)

* pre-checkout commit

* add API endpoints for retention policies

* allow deleting multiple teams/channels from a policy in a single request

* pre-checkout commit

* add auditing in API functions

* add permission checks

* update the store layers

* update storetest

* add check constraint on PostDuration column

* pre-checkout commit

* add query to delete posts under the scope of a granular retention policy

* add suggestions from sbishel

* allow clients to specify channels/teams when creating a new policy

* remove foreign keys referencing Channels and Teams tables

* add checks for whether teams and channels exist

* pre-checkout commit

* remove data referencing the Posts table

* pre-checkout commit

* write data store tests

* sort results of buildGetPoliciesQuery

* add missing test cases for teams

* pre-checkout commit

* add Client4 methods for data retention policy endpoints

* add uint and uint64 to app/layer_generators

* make granular policies override global policies

* fix lint errors

* pre-checkout commit

* add license to top of files

* add tests for data store layer

* add missing test cases for store layer

* run make i18n-extract

* add query to delete ChannelMemberHistory

* work in progress

* add test for old reply to old post

* fix lint error

* use COALESCE on each Posts column

* begin implementing orphaned rows worker

* split PR

* pre-checkout commit

* use RetentionPolicyWithTeamAndChannelCounts

* update app and api layers

* run make i18n-extract

* add RetentionPolicy to retrylayer_test.go

* Revert "split PR"

This reverts commit b316f03dd307a30deae931944ca7e4a1cc904605.

* fix errors caused by revert

* add suggestions from sbishel

* fix copy-paste error

* fix lint errors

* pre-checkout commit

* add function to delete orphaned rows

* use -1 for infinite retention

* remove check constraint

* copy i18n entries from master

* re-run tests with newer enterprise branch

* add team data to channel list

* add search for channels and teams in a policy

* add store tests for channel and team search

* add suggestions from mkraft

* run make einterfaces-mocks

* fix lint errors

* add suggestions from mkraft

* move removeOrphanedRows method to wireup branch

* Revert "move removeOrphanedRows method to wireup branch"

This reverts commit 94605c9b4a5378ffa44a3dec4d3f8e3306b9d33e.

* use DeleteOrphanedRows where possible

* run make i18n-extract

* use COMPLIANCE permissions

* run make migrations-bindadta

* clean up teams before test

* fix tests for TestRetentionPolicyStore

* add API endpoints for mobile

* fix lint error

* fix some of the lint errors

* move user/data_retention endpoints to data_retention.go

* Revert "fix some of the lint errors"

This reverts commit b5b2dc27566c427187db942c5c0afe319e7679c4.

* add exclude_policy_constrained parameter for /channels and /teams

* fix lint errors

* add policy_id field to GET endpoints for channels and teams

* use PolicyWithTeamID in RetentionPolicy layer

* fix lint errors

* run make i18n-extract

* update mock call in telemetry_test.go

* return status:OK in JSON instead of 204

* pre-checkout commit

* add policy_id field on channels/teams

* fix lint errors

* use sq.Eq instead of '?'

* use new subsection permissions

* update channels and teams endpoints to use new subsection permissions

* add extra search opts for channels in a policy

* fix lint errors

* allow negative post duration in patch

* remove DELETE FROM query in retention policy tests

* use *int64 for PostDuration

* re-run CI tests

* use 3-step deletion strategy for each table

* fix lint errors

* run make store-layers

* re-run CI tests

* add test with channel, team and global policies

* use common function for SQL queries

* add pagination test

* use struct for args to common SQL function

* fix lint errors

* run make i18n-extract

* check if Channels.TeamId is "" or nil

* use three OR clauses

* write separate genericRetentionPoliciesDeletion function

* add config setting for BatchSize

* add telemetry for BatchSize

* use feature flag

* add old i18n messages back in

* re-run CI tests

* update call signature in storetest

* MM-30831: Adds constant for retention default batch size.

* MM-30831: Removes comment re: optimization.

* MM-30831: Converts days to milliseconds.

* MM-30831: Reverts change to test.

* Revert "MM-30831: Reverts change to test."

This reverts commit 6d14275a1ceae682bb9e17ec69b39252b44e0c0c.

* Revert "MM-30831: Converts days to milliseconds."

This reverts commit a0cb6ec09d854a05194c1daee1c5333f260231c3.

* MM-30831: Fixes tests.

* MM-30381: Fix for change to method sig.

Co-authored-by: Max Erenberg <max.erenberg@mattermost.com>
Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Co-authored-by: Martin Kraft <martin@upspin.org>

store/retrylayer/retrylayer.go

@@ -2488,6 +2488,26 @@ func (s *RetryLayerChannelStore) UserBelongsToChannels(userID string, channelIds
 
 }
 
+func (s *RetryLayerChannelMemberHistoryStore) DeleteOrphanedRows(limit int) (int64, error) {
+
+	tries := 0
+	for {
+		result, err := s.ChannelMemberHistoryStore.DeleteOrphanedRows(limit)
+		if err == nil {
+			return result, nil
+		}
+		if !isRepeatableError(err) {
+			return result, err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return result, err
+		}
+	}
+
+}
+
 func (s *RetryLayerChannelMemberHistoryStore) GetUsersInChannelDuring(startTime int64, endTime int64, channelID string) ([]*model.ChannelMemberHistoryResult, error) {
 
 	tries := 0
@@ -2568,6 +2588,26 @@ func (s *RetryLayerChannelMemberHistoryStore) PermanentDeleteBatch(endTime int64
 
 }
 
+func (s *RetryLayerChannelMemberHistoryStore) PermanentDeleteBatchForRetentionPolicies(now int64, globalPolicyEndTime int64, limit int64, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error) {
+
+	tries := 0
+	for {
+		result, resultVar1, err := s.ChannelMemberHistoryStore.PermanentDeleteBatchForRetentionPolicies(now, globalPolicyEndTime, limit, cursor)
+		if err == nil {
+			return result, resultVar1, nil
+		}
+		if !isRepeatableError(err) {
+			return result, resultVar1, err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return result, resultVar1, err
+		}
+	}
+
+}
+
 func (s *RetryLayerClusterDiscoveryStore) Cleanup() error {
 
 	tries := 0
@@ -5472,6 +5512,26 @@ func (s *RetryLayerPostStore) Delete(postID string, time int64, deleteByID strin
 
 }
 
+func (s *RetryLayerPostStore) DeleteOrphanedRows(limit int) (int64, error) {
+
+	tries := 0
+	for {
+		result, err := s.PostStore.DeleteOrphanedRows(limit)
+		if err == nil {
+			return result, nil
+		}
+		if !isRepeatableError(err) {
+			return result, err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return result, err
+		}
+	}
+
+}
+
 func (s *RetryLayerPostStore) Get(ctx context.Context, id string, skipFetchThreads bool, collapsedThreads bool, collapsedThreadsExtended bool, userID string) (*model.PostList, error) {
 
 	tries := 0
@@ -5990,6 +6050,26 @@ func (s *RetryLayerPostStore) PermanentDeleteBatch(endTime int64, limit int64) (
 
 }
 
+func (s *RetryLayerPostStore) PermanentDeleteBatchForRetentionPolicies(now int64, globalPolicyEndTime int64, limit int64, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error) {
+
+	tries := 0
+	for {
+		result, resultVar1, err := s.PostStore.PermanentDeleteBatchForRetentionPolicies(now, globalPolicyEndTime, limit, cursor)
+		if err == nil {
+			return result, resultVar1, nil
+		}
+		if !isRepeatableError(err) {
+			return result, resultVar1, err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return result, resultVar1, err
+		}
+	}
+
+}
+
 func (s *RetryLayerPostStore) PermanentDeleteByChannel(channelID string) error {
 
 	tries := 0
@@ -6210,6 +6290,26 @@ func (s *RetryLayerPreferenceStore) DeleteCategoryAndName(category string, name
 
 }
 
+func (s *RetryLayerPreferenceStore) DeleteOrphanedRows(limit int) (int64, error) {
+
+	tries := 0
+	for {
+		result, err := s.PreferenceStore.DeleteOrphanedRows(limit)
+		if err == nil {
+			return result, nil
+		}
+		if !isRepeatableError(err) {
+			return result, err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return result, err
+		}
+	}
+
+}
+
 func (s *RetryLayerPreferenceStore) Get(userID string, category string, name string) (*model.Preference, error) {
 
 	tries := 0
@@ -6450,6 +6550,26 @@ func (s *RetryLayerReactionStore) DeleteAllWithEmojiName(emojiName string) error
 
 }
 
+func (s *RetryLayerReactionStore) DeleteOrphanedRows(limit int) (int64, error) {
+
+	tries := 0
+	for {
+		result, err := s.ReactionStore.DeleteOrphanedRows(limit)
+		if err == nil {
+			return result, nil
+		}
+		if !isRepeatableError(err) {
+			return result, err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return result, err
+		}
+	}
+
+}
+
 func (s *RetryLayerReactionStore) GetForPost(postID string, allowFromCache bool) ([]*model.Reaction, error) {
 
 	tries := 0
@@ -6730,6 +6850,26 @@ func (s *RetryLayerRetentionPolicyStore) Delete(id string) error {
 
 }
 
+func (s *RetryLayerRetentionPolicyStore) DeleteOrphanedRows(limit int) (int64, error) {
+
+	tries := 0
+	for {
+		result, err := s.RetentionPolicyStore.DeleteOrphanedRows(limit)
+		if err == nil {
+			return result, nil
+		}
+		if !isRepeatableError(err) {
+			return result, err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return result, err
+		}
+	}
+
+}
+
 func (s *RetryLayerRetentionPolicyStore) Get(id string) (*model.RetentionPolicyWithTeamAndChannelCounts, error) {
 
 	tries := 0
@@ -9588,6 +9728,26 @@ func (s *RetryLayerThreadStore) DeleteMembershipForUser(userId string, postID st
 
 }
 
+func (s *RetryLayerThreadStore) DeleteOrphanedRows(limit int) (int64, error) {
+
+	tries := 0
+	for {
+		result, err := s.ThreadStore.DeleteOrphanedRows(limit)
+		if err == nil {
+			return result, nil
+		}
+		if !isRepeatableError(err) {
+			return result, err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return result, err
+		}
+	}
+
+}
+
 func (s *RetryLayerThreadStore) Get(id string) (*model.Thread, error) {
 
 	tries := 0
@@ -9808,6 +9968,46 @@ func (s *RetryLayerThreadStore) MarkAsRead(userID string, threadID string, times
 
 }
 
+func (s *RetryLayerThreadStore) PermanentDeleteBatchForRetentionPolicies(now int64, globalPolicyEndTime int64, limit int64, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error) {
+
+	tries := 0
+	for {
+		result, resultVar1, err := s.ThreadStore.PermanentDeleteBatchForRetentionPolicies(now, globalPolicyEndTime, limit, cursor)
+		if err == nil {
+			return result, resultVar1, nil
+		}
+		if !isRepeatableError(err) {
+			return result, resultVar1, err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return result, resultVar1, err
+		}
+	}
+
+}
+
+func (s *RetryLayerThreadStore) PermanentDeleteBatchThreadMembershipsForRetentionPolicies(now int64, globalPolicyEndTime int64, limit int64, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error) {
+
+	tries := 0
+	for {
+		result, resultVar1, err := s.ThreadStore.PermanentDeleteBatchThreadMembershipsForRetentionPolicies(now, globalPolicyEndTime, limit, cursor)
+		if err == nil {
+			return result, resultVar1, nil
+		}
+		if !isRepeatableError(err) {
+			return result, resultVar1, err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return result, resultVar1, err
+		}
+	}
+
+}
+
 func (s *RetryLayerThreadStore) Save(thread *model.Thread) (*model.Thread, error) {
 
 	tries := 0