[MM-30831] granular data retention wireup (#17417)

* pre-checkout commit

* add API endpoints for retention policies

* allow deleting multiple teams/channels from a policy in a single request

* pre-checkout commit

* add auditing in API functions

* add permission checks

* update the store layers

* update storetest

* add check constraint on PostDuration column

* pre-checkout commit

* add query to delete posts under the scope of a granular retention policy

* add suggestions from sbishel

* allow clients to specify channels/teams when creating a new policy

* remove foreign keys referencing Channels and Teams tables

* add checks for whether teams and channels exist

* pre-checkout commit

* remove data referencing the Posts table

* pre-checkout commit

* write data store tests

* sort results of buildGetPoliciesQuery

* add missing test cases for teams

* pre-checkout commit

* add Client4 methods for data retention policy endpoints

* add uint and uint64 to app/layer_generators

* make granular policies override global policies

* fix lint errors

* pre-checkout commit

* add license to top of files

* add tests for data store layer

* add missing test cases for store layer

* run make i18n-extract

* add query to delete ChannelMemberHistory

* work in progress

* add test for old reply to old post

* fix lint error

* use COALESCE on each Posts column

* begin implementing orphaned rows worker

* split PR

* pre-checkout commit

* use RetentionPolicyWithTeamAndChannelCounts

* update app and api layers

* run make i18n-extract

* add RetentionPolicy to retrylayer_test.go

* Revert "split PR"

This reverts commit b316f03dd307a30deae931944ca7e4a1cc904605.

* fix errors caused by revert

* add suggestions from sbishel

* fix copy-paste error

* fix lint errors

* pre-checkout commit

* add function to delete orphaned rows

* use -1 for infinite retention

* remove check constraint

* copy i18n entries from master

* re-run tests with newer enterprise branch

* add team data to channel list

* add search for channels and teams in a policy

* add store tests for channel and team search

* add suggestions from mkraft

* run make einterfaces-mocks

* fix lint errors

* add suggestions from mkraft

* move removeOrphanedRows method to wireup branch

* Revert "move removeOrphanedRows method to wireup branch"

This reverts commit 94605c9b4a5378ffa44a3dec4d3f8e3306b9d33e.

* use DeleteOrphanedRows where possible

* run make i18n-extract

* use COMPLIANCE permissions

* run make migrations-bindadta

* clean up teams before test

* fix tests for TestRetentionPolicyStore

* add API endpoints for mobile

* fix lint error

* fix some of the lint errors

* move user/data_retention endpoints to data_retention.go

* Revert "fix some of the lint errors"

This reverts commit b5b2dc27566c427187db942c5c0afe319e7679c4.

* add exclude_policy_constrained parameter for /channels and /teams

* fix lint errors

* add policy_id field to GET endpoints for channels and teams

* use PolicyWithTeamID in RetentionPolicy layer

* fix lint errors

* run make i18n-extract

* update mock call in telemetry_test.go

* return status:OK in JSON instead of 204

* pre-checkout commit

* add policy_id field on channels/teams

* fix lint errors

* use sq.Eq instead of '?'

* use new subsection permissions

* update channels and teams endpoints to use new subsection permissions

* add extra search opts for channels in a policy

* fix lint errors

* allow negative post duration in patch

* remove DELETE FROM query in retention policy tests

* use *int64 for PostDuration

* re-run CI tests

* use 3-step deletion strategy for each table

* fix lint errors

* run make store-layers

* re-run CI tests

* add test with channel, team and global policies

* use common function for SQL queries

* add pagination test

* use struct for args to common SQL function

* fix lint errors

* run make i18n-extract

* check if Channels.TeamId is "" or nil

* use three OR clauses

* write separate genericRetentionPoliciesDeletion function

* add config setting for BatchSize

* add telemetry for BatchSize

* use feature flag

* add old i18n messages back in

* re-run CI tests

* update call signature in storetest

* MM-30831: Adds constant for retention default batch size.

* MM-30831: Removes comment re: optimization.

* MM-30831: Converts days to milliseconds.

* MM-30831: Reverts change to test.

* Revert "MM-30831: Reverts change to test."

This reverts commit 6d14275a1ceae682bb9e17ec69b39252b44e0c0c.

* Revert "MM-30831: Converts days to milliseconds."

This reverts commit a0cb6ec09d854a05194c1daee1c5333f260231c3.

* MM-30831: Fixes tests.

* MM-30381: Fix for change to method sig.

Co-authored-by: Max Erenberg <max.erenberg@mattermost.com>
Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Co-authored-by: Martin Kraft <martin@upspin.org>

model/config.go

@@ -206,6 +206,7 @@ const (
 	DATA_RETENTION_SETTINGS_DEFAULT_MESSAGE_RETENTION_DAYS  = 365
 	DATA_RETENTION_SETTINGS_DEFAULT_FILE_RETENTION_DAYS     = 365
 	DATA_RETENTION_SETTINGS_DEFAULT_DELETION_JOB_START_TIME = "02:00"
+	DATA_RETENTION_SETTINGS_DEFAULT_BATCH_SIZE              = 3000
 
 	PLUGIN_SETTINGS_DEFAULT_DIRECTORY          = "./plugins"
 	PLUGIN_SETTINGS_DEFAULT_CLIENT_DIRECTORY   = "./client/plugins"
@@ -2700,6 +2701,7 @@ type DataRetentionSettings struct {
 	MessageRetentionDays  *int    `access:"compliance_data_retention_policy"`
 	FileRetentionDays     *int    `access:"compliance_data_retention_policy"`
 	DeletionJobStartTime  *string `access:"compliance_data_retention_policy"`
+	BatchSize             *int    `access:"compliance_data_retention_policy"`
 }
 
 func (s *DataRetentionSettings) SetDefaults() {
@@ -2722,6 +2724,10 @@ func (s *DataRetentionSettings) SetDefaults() {
 	if s.DeletionJobStartTime == nil {
 		s.DeletionJobStartTime = NewString(DATA_RETENTION_SETTINGS_DEFAULT_DELETION_JOB_START_TIME)
 	}
+
+	if s.BatchSize == nil {
+		s.BatchSize = NewInt(DATA_RETENTION_SETTINGS_DEFAULT_BATCH_SIZE)
+	}
 }
 
 type JobSettings struct {

model/data_retention_policy.go

@@ -68,6 +68,12 @@ type RetentionPolicyForChannelList struct {
 	TotalCount int64                        `json:"total_count"`
 }
 
+type RetentionPolicyCursor struct {
+	ChannelPoliciesDone bool
+	TeamPoliciesDone    bool
+	GlobalPoliciesDone  bool
+}
+
 func (rp *GlobalRetentionPolicy) ToJson() []byte {
 	b, _ := json.Marshal(rp)
 	return b

model/post.go

@@ -371,6 +371,9 @@ func (o *Post) IsValid(maxPostSize int) *AppError {
 }
 
 func (o *Post) SanitizeProps() {
+	if o == nil {
+		return
+	}
 	membersToSanitize := []string{
 		PROPS_ADD_CHANNEL_MEMBER,
 	}
@@ -728,3 +731,10 @@ func (o *Post) IsFromOAuthBot() bool {
 	props := o.GetProps()
 	return props["from_webhook"] == "true" && props["override_username"] != ""
 }
+
+func (o *Post) ToNilIfInvalid() *Post {
+	if o.Id == "" {
+		return nil
+	}
+	return o
+}