[MM-30831] granular data retention wireup (#17417)

* pre-checkout commit

* add API endpoints for retention policies

* allow deleting multiple teams/channels from a policy in a single request

* pre-checkout commit

* add auditing in API functions

* add permission checks

* update the store layers

* update storetest

* add check constraint on PostDuration column

* pre-checkout commit

* add query to delete posts under the scope of a granular retention policy

* add suggestions from sbishel

* allow clients to specify channels/teams when creating a new policy

* remove foreign keys referencing Channels and Teams tables

* add checks for whether teams and channels exist

* pre-checkout commit

* remove data referencing the Posts table

* pre-checkout commit

* write data store tests

* sort results of buildGetPoliciesQuery

* add missing test cases for teams

* pre-checkout commit

* add Client4 methods for data retention policy endpoints

* add uint and uint64 to app/layer_generators

* make granular policies override global policies

* fix lint errors

* pre-checkout commit

* add license to top of files

* add tests for data store layer

* add missing test cases for store layer

* run make i18n-extract

* add query to delete ChannelMemberHistory

* work in progress

* add test for old reply to old post

* fix lint error

* use COALESCE on each Posts column

* begin implementing orphaned rows worker

* split PR

* pre-checkout commit

* use RetentionPolicyWithTeamAndChannelCounts

* update app and api layers

* run make i18n-extract

* add RetentionPolicy to retrylayer_test.go

* Revert "split PR"

This reverts commit b316f03dd307a30deae931944ca7e4a1cc904605.

* fix errors caused by revert

* add suggestions from sbishel

* fix copy-paste error

* fix lint errors

* pre-checkout commit

* add function to delete orphaned rows

* use -1 for infinite retention

* remove check constraint

* copy i18n entries from master

* re-run tests with newer enterprise branch

* add team data to channel list

* add search for channels and teams in a policy

* add store tests for channel and team search

* add suggestions from mkraft

* run make einterfaces-mocks

* fix lint errors

* add suggestions from mkraft

* move removeOrphanedRows method to wireup branch

* Revert "move removeOrphanedRows method to wireup branch"

This reverts commit 94605c9b4a5378ffa44a3dec4d3f8e3306b9d33e.

* use DeleteOrphanedRows where possible

* run make i18n-extract

* use COMPLIANCE permissions

* run make migrations-bindadta

* clean up teams before test

* fix tests for TestRetentionPolicyStore

* add API endpoints for mobile

* fix lint error

* fix some of the lint errors

* move user/data_retention endpoints to data_retention.go

* Revert "fix some of the lint errors"

This reverts commit b5b2dc27566c427187db942c5c0afe319e7679c4.

* add exclude_policy_constrained parameter for /channels and /teams

* fix lint errors

* add policy_id field to GET endpoints for channels and teams

* use PolicyWithTeamID in RetentionPolicy layer

* fix lint errors

* run make i18n-extract

* update mock call in telemetry_test.go

* return status:OK in JSON instead of 204

* pre-checkout commit

* add policy_id field on channels/teams

* fix lint errors

* use sq.Eq instead of '?'

* use new subsection permissions

* update channels and teams endpoints to use new subsection permissions

* add extra search opts for channels in a policy

* fix lint errors

* allow negative post duration in patch

* remove DELETE FROM query in retention policy tests

* use *int64 for PostDuration

* re-run CI tests

* use 3-step deletion strategy for each table

* fix lint errors

* run make store-layers

* re-run CI tests

* add test with channel, team and global policies

* use common function for SQL queries

* add pagination test

* use struct for args to common SQL function

* fix lint errors

* run make i18n-extract

* check if Channels.TeamId is "" or nil

* use three OR clauses

* write separate genericRetentionPoliciesDeletion function

* add config setting for BatchSize

* add telemetry for BatchSize

* use feature flag

* add old i18n messages back in

* re-run CI tests

* update call signature in storetest

* MM-30831: Adds constant for retention default batch size.

* MM-30831: Removes comment re: optimization.

* MM-30831: Converts days to milliseconds.

* MM-30831: Reverts change to test.

* Revert "MM-30831: Reverts change to test."

This reverts commit 6d14275a1ceae682bb9e17ec69b39252b44e0c0c.

* Revert "MM-30831: Converts days to milliseconds."

This reverts commit a0cb6ec09d854a05194c1daee1c5333f260231c3.

* MM-30831: Fixes tests.

* MM-30381: Fix for change to method sig.

Co-authored-by: Max Erenberg <max.erenberg@mattermost.com>
Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Co-authored-by: Martin Kraft <martin@upspin.org>

store/sqlstore/thread_store.go

@@ -132,12 +132,15 @@ func (s *SqlThreadStore) GetThreadsForUser(userId, teamId string, opts model.Get
 
 	unreadRepliesQuery := "SELECT COUNT(Posts.Id) From Posts Where Posts.RootId=ThreadMemberships.PostId AND Posts.CreateAt >= ThreadMemberships.LastViewed"
 	fetchConditions := sq.And{
-		sq.Or{sq.Eq{"Channels.TeamId": teamId}, sq.Eq{"Channels.TeamId": ""}},
+		sq.Or{sq.Eq{"Channels.TeamId": teamId}, sq.Eq{"Channels.TeamId": ""}, sq.Eq{"Channels.TeamId": nil}},
 		sq.Eq{"ThreadMemberships.UserId": userId},
 		sq.Eq{"ThreadMemberships.Following": true},
 	}
 	if !opts.Deleted {
-		fetchConditions = sq.And{fetchConditions, sq.Eq{"Posts.DeleteAt": 0}}
+		fetchConditions = sq.And{
+			fetchConditions,
+			sq.Eq{"COALESCE(Posts.DeleteAt, 0)": 0},
+		}
 	}
 
 	pageSize := uint64(30)
@@ -217,7 +220,10 @@ func (s *SqlThreadStore) GetThreadsForUser(userId, teamId string, opts model.Get
 		}
 		var threads []*JoinedThread
 		query, args, _ := s.getQueryBuilder().
-			Select("Threads.*, Posts.*, ThreadMemberships.LastViewed as LastViewedAt, ThreadMemberships.UnreadMentions as UnreadMentions").
+			Select(`Threads.*,
+				` + postSliceCoalesceQuery() + `,
+				ThreadMemberships.LastViewed as LastViewedAt,
+				ThreadMemberships.UnreadMentions as UnreadMentions`).
 			From("Threads").
 			Column(sq.Alias(sq.Expr(unreadRepliesQuery), "UnreadReplies")).
 			LeftJoin("Posts ON Posts.Id = Threads.PostId").
@@ -309,7 +315,7 @@ func (s *SqlThreadStore) GetThreadsForUser(userId, teamId string, opts model.Get
 			UnreadReplies:  thread.UnreadReplies,
 			UnreadMentions: thread.UnreadMentions,
 			Participants:   participants,
-			Post:           &thread.Post,
+			Post:           thread.Post.ToNilIfInvalid(),
 		})
 	}
 
@@ -401,7 +407,7 @@ func (s *SqlThreadStore) GetThreadForUser(teamId string, threadMembership *model
 		UnreadReplies:  thread.UnreadReplies,
 		UnreadMentions: thread.UnreadMentions,
 		Participants:   users,
-		Post:           &thread.Post,
+		Post:           thread.Post.ToNilIfInvalid(),
 	}
 
 	return result, nil
@@ -691,3 +697,86 @@ func (s *SqlThreadStore) GetPosts(threadId string, since int64) ([]*model.Post,
 	}
 	return result, nil
 }
+
+// PermanentDeleteBatchForRetentionPolicies deletes a batch of records which are affected by
+// the global or a granular retention policy.
+// See `genericPermanentDeleteBatchForRetentionPolicies` for details.
+func (s *SqlThreadStore) PermanentDeleteBatchForRetentionPolicies(now, globalPolicyEndTime, limit int64, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error) {
+	builder := s.getQueryBuilder().
+		Select("Threads.PostId").
+		From("Threads")
+	return genericPermanentDeleteBatchForRetentionPolicies(RetentionPolicyBatchDeletionInfo{
+		BaseBuilder:         builder,
+		Table:               "Threads",
+		TimeColumn:          "LastReplyAt",
+		PrimaryKeys:         []string{"PostId"},
+		ChannelIDTable:      "Threads",
+		NowMillis:           now,
+		GlobalPolicyEndTime: globalPolicyEndTime,
+		Limit:               limit,
+	}, s.SqlStore, cursor)
+}
+
+// PermanentDeleteBatchThreadMembershipsForRetentionPolicies deletes a batch of records
+// which are affected by the global or a granular retention policy.
+// See `genericPermanentDeleteBatchForRetentionPolicies` for details.
+func (s *SqlThreadStore) PermanentDeleteBatchThreadMembershipsForRetentionPolicies(now, globalPolicyEndTime, limit int64, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error) {
+	builder := s.getQueryBuilder().
+		Select("ThreadMemberships.PostId").
+		From("ThreadMemberships").
+		InnerJoin("Threads ON ThreadMemberships.PostId = Threads.PostId")
+	return genericPermanentDeleteBatchForRetentionPolicies(RetentionPolicyBatchDeletionInfo{
+		BaseBuilder:         builder,
+		Table:               "ThreadMemberships",
+		TimeColumn:          "LastUpdated",
+		PrimaryKeys:         []string{"PostId"},
+		ChannelIDTable:      "Threads",
+		NowMillis:           now,
+		GlobalPolicyEndTime: globalPolicyEndTime,
+		Limit:               limit,
+	}, s.SqlStore, cursor)
+}
+
+// DeleteOrphanedRows removes orphaned rows from Threads and ThreadMemberships
+func (s *SqlThreadStore) DeleteOrphanedRows(limit int) (deleted int64, err error) {
+	// We need the extra level of nesting to deal with MySQL's locking
+	const threadsQuery = `
+	DELETE FROM Threads WHERE PostId IN (
+		SELECT * FROM (
+			SELECT Threads.PostId FROM Threads
+			LEFT JOIN Channels ON Threads.ChannelId = Channels.Id
+			WHERE Channels.Id IS NULL
+			LIMIT :Limit
+		) AS A
+	)`
+	// We only delete a thread membership if the entire thread no longer exists,
+	// not if the root post has been deleted
+	const threadMembershipsQuery = `
+	DELETE FROM ThreadMemberships WHERE PostId IN (
+		SELECT * FROM (
+			SELECT ThreadMemberships.PostId FROM ThreadMemberships
+			LEFT JOIN Threads ON ThreadMemberships.PostId = Threads.PostId
+			WHERE Threads.PostId IS NULL
+			LIMIT :Limit
+		) AS A
+	)`
+	props := map[string]interface{}{"Limit": limit}
+	result, err := s.GetMaster().Exec(threadsQuery, props)
+	if err != nil {
+		return
+	}
+	rpcDeleted, err := result.RowsAffected()
+	if err != nil {
+		return
+	}
+	result, err = s.GetMaster().Exec(threadMembershipsQuery, props)
+	if err != nil {
+		return
+	}
+	rptDeleted, err := result.RowsAffected()
+	if err != nil {
+		return
+	}
+	deleted = rpcDeleted + rptDeleted
+	return
+}