Boards data retention (#19262)

* add Boards to DataRetention, add hook for data retention

* remove replaces

* update hook to remove parameter

* add boards data retention to telemetry

* fix unit test

* update test, update hooks

* update RunDataRetention server version

* put behind a feature flag

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

app/plugin_hooks_test.go

@@ -1215,3 +1215,47 @@ func TestHookReactionHasBeenRemoved(t *testing.T) {
 
 	require.Nil(t, err)
 }
+
+func TestHookRunDataRetention(t *testing.T) {
+	th := Setup(t).InitBasic()
+	defer th.TearDown()
+
+	tearDown, pluginIDs, _ := SetAppEnvironmentWithPlugins(t,
+		[]string{
+			`
+		package main
+
+		import (
+			"github.com/mattermost/mattermost-server/v6/plugin"
+		)
+
+		type MyPlugin struct {
+			plugin.MattermostPlugin
+		}
+
+		func (p *MyPlugin) RunDataRetention(nowMillis, batchSize int64) (int64, error){
+			return 100, nil
+		}
+
+		func main() {
+			plugin.ClientMain(&MyPlugin{})
+		}
+	`}, th.App, th.NewPluginAPI)
+	defer tearDown()
+
+	require.Len(t, pluginIDs, 1)
+	pluginID := pluginIDs[0]
+
+	require.True(t, th.App.GetPluginsEnvironment().IsActive(pluginID))
+
+	hookCalled := false
+	th.App.GetPluginsEnvironment().RunMultiPluginHook(func(hooks plugin.Hooks) bool {
+		n, _ := hooks.RunDataRetention(0, 0)
+		// Ensure return it correct
+		assert.Equal(t, int64(100), n)
+		hookCalled = true
+		return hookCalled
+	}, plugin.RunDataRetentionID)
+
+	require.True(t, hookCalled)
+}

config/client.go

@@ -126,6 +126,8 @@ func GenerateClientConfig(c *model.Config, telemetryID string, license *model.Li
 	props["DataRetentionMessageRetentionDays"] = "0"
 	props["DataRetentionEnableFileDeletion"] = "false"
 	props["DataRetentionFileRetentionDays"] = "0"
+	props["DataRetentionEnableBoardsDeletion"] = "false"
+	props["DataRetentionBoardsRetentionDays"] = "0"
 	props["CWSURL"] = ""
 
 	props["CustomUrlSchemes"] = strings.Join(c.DisplaySettings.CustomURLSchemes, ",")
@@ -190,6 +192,8 @@ func GenerateClientConfig(c *model.Config, telemetryID string, license *model.Li
 			props["DataRetentionMessageRetentionDays"] = strconv.FormatInt(int64(*c.DataRetentionSettings.MessageRetentionDays), 10)
 			props["DataRetentionEnableFileDeletion"] = strconv.FormatBool(*c.DataRetentionSettings.EnableFileDeletion)
 			props["DataRetentionFileRetentionDays"] = strconv.FormatInt(int64(*c.DataRetentionSettings.FileRetentionDays), 10)
+			props["DataRetentionEnableBoardsDeletion"] = strconv.FormatBool(*c.DataRetentionSettings.EnableBoardsDeletion)
+			props["DataRetentionBoardsRetentionDays"] = strconv.FormatInt(int64(*c.DataRetentionSettings.BoardsRetentionDays), 10)
 		}
 
 		if *license.Features.Cloud {

jobs/workers.go

@@ -221,9 +221,9 @@ func (workers *Workers) handleConfigChange(oldConfig *model.Config, newConfig *m
 	mlog.Debug("Workers received config change.")
 
 	if workers.DataRetention != nil {
-		if (!*oldConfig.DataRetentionSettings.EnableMessageDeletion && !*oldConfig.DataRetentionSettings.EnableFileDeletion) && (*newConfig.DataRetentionSettings.EnableMessageDeletion || *newConfig.DataRetentionSettings.EnableFileDeletion) {
+		if (!*oldConfig.DataRetentionSettings.EnableMessageDeletion && !*oldConfig.DataRetentionSettings.EnableFileDeletion && !*oldConfig.DataRetentionSettings.EnableBoardsDeletion) && (*newConfig.DataRetentionSettings.EnableMessageDeletion || *newConfig.DataRetentionSettings.EnableFileDeletion || *newConfig.DataRetentionSettings.EnableBoardsDeletion) {
 			go workers.DataRetention.Run()
-		} else if (*oldConfig.DataRetentionSettings.EnableMessageDeletion || *oldConfig.DataRetentionSettings.EnableFileDeletion) && (!*newConfig.DataRetentionSettings.EnableMessageDeletion && !*newConfig.DataRetentionSettings.EnableFileDeletion) {
+		} else if (*oldConfig.DataRetentionSettings.EnableMessageDeletion || *oldConfig.DataRetentionSettings.EnableFileDeletion || *oldConfig.DataRetentionSettings.EnableBoardsDeletion) && (!*newConfig.DataRetentionSettings.EnableMessageDeletion && !*newConfig.DataRetentionSettings.EnableFileDeletion && !*newConfig.DataRetentionSettings.EnableBoardsDeletion) {
 			workers.DataRetention.Stop()
 		}
 	}

model/config.go

@@ -202,6 +202,7 @@ const (
 
 	DataRetentionSettingsDefaultMessageRetentionDays = 365
 	DataRetentionSettingsDefaultFileRetentionDays    = 365
+	DataRetentionSettingsDefaultBoardsRetentionDays  = 365
 	DataRetentionSettingsDefaultDeletionJobStartTime = "02:00"
 	DataRetentionSettingsDefaultBatchSize            = 3000
 
@@ -2586,8 +2587,10 @@ func (bs *BleveSettings) SetDefaults() {
 type DataRetentionSettings struct {
 	EnableMessageDeletion *bool   `access:"compliance_data_retention_policy"`
 	EnableFileDeletion    *bool   `access:"compliance_data_retention_policy"`
+	EnableBoardsDeletion  *bool   `access:"compliance_data_retention_policy"`
 	MessageRetentionDays  *int    `access:"compliance_data_retention_policy"`
 	FileRetentionDays     *int    `access:"compliance_data_retention_policy"`
+	BoardsRetentionDays   *int    `access:"compliance_data_retention_policy"`
 	DeletionJobStartTime  *string `access:"compliance_data_retention_policy"`
 	BatchSize             *int    `access:"compliance_data_retention_policy"`
 }
@@ -2601,6 +2604,10 @@ func (s *DataRetentionSettings) SetDefaults() {
 		s.EnableFileDeletion = NewBool(false)
 	}
 
+	if s.EnableBoardsDeletion == nil {
+		s.EnableBoardsDeletion = NewBool(false)
+	}
+
 	if s.MessageRetentionDays == nil {
 		s.MessageRetentionDays = NewInt(DataRetentionSettingsDefaultMessageRetentionDays)
 	}
@@ -2609,6 +2616,10 @@ func (s *DataRetentionSettings) SetDefaults() {
 		s.FileRetentionDays = NewInt(DataRetentionSettingsDefaultFileRetentionDays)
 	}
 
+	if s.BoardsRetentionDays == nil {
+		s.BoardsRetentionDays = NewInt(DataRetentionSettingsDefaultBoardsRetentionDays)
+	}
+
 	if s.DeletionJobStartTime == nil {
 		s.DeletionJobStartTime = NewString(DataRetentionSettingsDefaultDeletionJobStartTime)
 	}

model/data_retention_policy.go

@@ -6,8 +6,10 @@ package model
 type GlobalRetentionPolicy struct {
 	MessageDeletionEnabled bool  `json:"message_deletion_enabled"`
 	FileDeletionEnabled    bool  `json:"file_deletion_enabled"`
+	BoardsDeletionEnabled  bool  `json:"boards_deletion_enabled"`
 	MessageRetentionCutoff int64 `json:"message_retention_cutoff"`
 	FileRetentionCutoff    int64 `json:"file_retention_cutoff"`
+	BoardsRetentionCutoff  int64 `json:"boards_retention_cutoff"`
 }
 
 type RetentionPolicy struct {

model/feature_flags.go

@@ -71,6 +71,9 @@ type FeatureFlags struct {
 	// Enable inline post editing
 	InlinePostEditing bool
 
+	// Enable DataRetention for Boards
+	BoardsDataRetention bool
+
 	NormalizeLdapDNs bool
 }
 
@@ -96,6 +99,7 @@ func (f *FeatureFlags) SetDefaults() {
 	f.ResendInviteEmailInterval = ""
 	f.InviteToTeam = "none"
 	f.InlinePostEditing = false
+	f.BoardsDataRetention = false
 	f.NormalizeLdapDNs = false
 }
 

plugin/client_rpc_generated.go

@@ -669,6 +669,43 @@ func (s *hooksRPCServer) WebSocketMessageHasBeenPosted(args *Z_WebSocketMessageH
 	return nil
 }
 
+func init() {
+	hookNameToId["RunDataRetention"] = RunDataRetentionID
+}
+
+type Z_RunDataRetentionArgs struct {
+	A int64
+	B int64
+}
+
+type Z_RunDataRetentionReturns struct {
+	A int64
+	B error
+}
+
+func (g *hooksRPCClient) RunDataRetention(nowTime, batchSize int64) (int64, error) {
+	_args := &Z_RunDataRetentionArgs{nowTime, batchSize}
+	_returns := &Z_RunDataRetentionReturns{}
+	if g.implemented[RunDataRetentionID] {
+		if err := g.client.Call("Plugin.RunDataRetention", _args, _returns); err != nil {
+			g.log.Error("RPC call RunDataRetention to plugin failed.", mlog.Err(err))
+		}
+	}
+	return _returns.A, _returns.B
+}
+
+func (s *hooksRPCServer) RunDataRetention(args *Z_RunDataRetentionArgs, returns *Z_RunDataRetentionReturns) error {
+	if hook, ok := s.impl.(interface {
+		RunDataRetention(nowTime, batchSize int64) (int64, error)
+	}); ok {
+		returns.A, returns.B = hook.RunDataRetention(args.A, args.B)
+		returns.B = encodableError(returns.B)
+	} else {
+		return encodableError(fmt.Errorf("Hook RunDataRetention called but not implemented."))
+	}
+	return nil
+}
+
 type Z_RegisterCommandArgs struct {
 	A *model.Command
 }

plugin/hooks.go

@@ -39,6 +39,7 @@ const (
 	OnWebSocketConnectID            = 21
 	OnWebSocketDisconnectID         = 22
 	WebSocketMessageHasBeenPostedID = 23
+	RunDataRetentionID              = 24
 	TotalHooksID                    = iota
 )
 
@@ -243,4 +244,9 @@ type Hooks interface {
 	//
 	// Minimum server version: 6.0
 	WebSocketMessageHasBeenPosted(webConnID, userID string, req *model.WebSocketRequest)
+
+	// RunDataRetention is invoked during a DataRetentionJob
+	//
+	// Minimum server version: 6.4
+	RunDataRetention(nowTime, batchSize int64) (int64, error)
 }

plugin/hooks_timer_layer_generated.go

@@ -186,3 +186,10 @@ func (hooks *hooksTimerLayer) WebSocketMessageHasBeenPosted(webConnID, userID st
 	hooks.hooksImpl.WebSocketMessageHasBeenPosted(webConnID, userID, req)
 	hooks.recordTime(startTime, "WebSocketMessageHasBeenPosted", true)
 }
+
+func (hooks *hooksTimerLayer) RunDataRetention(nowTime, batchSize int64) (int64, error) {
+	startTime := timePkg.Now()
+	_returnsA, _returnsB := hooks.hooksImpl.RunDataRetention(nowTime, batchSize)
+	hooks.recordTime(startTime, "RunDataRetention", _returnsB == nil)
+	return _returnsA, _returnsB
+}

plugin/plugintest/hooks.go

@@ -219,6 +219,27 @@ func (_m *Hooks) ReactionHasBeenRemoved(c *plugin.Context, reaction *model.React
 	_m.Called(c, reaction)
 }
 
+// RunDataRetention provides a mock function with given fields: nowTime, batchSize
+func (_m *Hooks) RunDataRetention(nowTime int64, batchSize int64) (int64, error) {
+	ret := _m.Called(nowTime, batchSize)
+
+	var r0 int64
+	if rf, ok := ret.Get(0).(func(int64, int64) int64); ok {
+		r0 = rf(nowTime, batchSize)
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(int64, int64) error); ok {
+		r1 = rf(nowTime, batchSize)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // ServeHTTP provides a mock function with given fields: c, w, r
 func (_m *Hooks) ServeHTTP(c *plugin.Context, w http.ResponseWriter, r *http.Request) {
 	_m.Called(c, w, r)

services/telemetry/telemetry.go

@@ -758,8 +758,10 @@ func (ts *TelemetryService) trackConfig() {
 	ts.SendTelemetry(TrackConfigDataRetention, map[string]interface{}{
 		"enable_message_deletion":     *cfg.DataRetentionSettings.EnableMessageDeletion,
 		"enable_file_deletion":        *cfg.DataRetentionSettings.EnableFileDeletion,
+		"enable_boards_deletion":      *cfg.DataRetentionSettings.EnableBoardsDeletion,
 		"message_retention_days":      *cfg.DataRetentionSettings.MessageRetentionDays,
 		"file_retention_days":         *cfg.DataRetentionSettings.FileRetentionDays,
+		"boards_retention_days":       *cfg.DataRetentionSettings.BoardsRetentionDays,
 		"deletion_job_start_time":     *cfg.DataRetentionSettings.DeletionJobStartTime,
 		"batch_size":                  *cfg.DataRetentionSettings.BatchSize,
 		"cleanup_jobs_threshold_days": *cfg.JobSettings.CleanupJobsThresholdDays,