PLT-6558: Basic data retention job scheduler/worker implementation. (#7449)

* PLT-7639: Batch delete methods for data retention.

* PLT-6558: Basic data retention job worker/scheduler implementation.

config/default.json

@@ -316,7 +316,11 @@
         "PostsAggregatorJobStartTime": "03:00"
     },
     "DataRetentionSettings": {
-        "Enable": false
+        "EnableMessageDeletion": false,
+        "EnableFileDeletion": false,
+        "MessageRetentionDays": 365,
+        "FileRetentionDays": 365,
+        "DeletionJobStartTime": "02:00"
     },
     "JobSettings": {
         "RunJobs": true,

i18n/en.json

@@ -4355,6 +4355,18 @@
     "id": "model.config.is_valid.elastic_search.aggregate_posts_after_days.app_error",
     "translation": "Elasticsearch AggregatePostsAfterDays setting must be a number greater than or equal to 1"
   },
+  {
+    "id": "model.config.is_valid.data_retention.message_retention_days_too_low.app_error",
+    "translation": "DataRetention MessageRetentionDays setting must be at least 1"
+  },
+  {
+    "id": "model.config.is_valid.data_retention.file_retention_days_too_low.app_error",
+    "translation": "DataRetention FileRetentionDays setting must be at least 1"
+  },
+  {
+    "id": "model.config.is_valid.data_retention.deletion_job_start_time.app_error",
+    "translation": "DataRetention DeletionJobStartTime setting must be set to a time in the format \"hh:mm\""
+  },
   {
     "id": "model.config.is_valid.elastic_search.connection_url.app_error",
     "translation": "Elastic Search ConnectionUrl setting must be provided when Elastic Search indexing is enabled."

jobs/schedulers.go

@@ -40,7 +40,7 @@ func (schedulers *Schedulers) Start() *Schedulers {
 	l4g.Info("Starting schedulers")
 
 	schedulers.startOnce.Do(func() {
-		if schedulers.DataRetention != nil && *utils.Cfg.DataRetentionSettings.Enable {
+		if schedulers.DataRetention != nil && (*utils.Cfg.DataRetentionSettings.EnableMessageDeletion || *utils.Cfg.DataRetentionSettings.EnableFileDeletion) {
 			go schedulers.DataRetention.Run()
 		}
 
@@ -56,9 +56,9 @@ func (schedulers *Schedulers) Start() *Schedulers {
 
 func (schedulers *Schedulers) handleConfigChange(oldConfig *model.Config, newConfig *model.Config) {
 	if schedulers.DataRetention != nil {
-		if !*oldConfig.DataRetentionSettings.Enable && *newConfig.DataRetentionSettings.Enable {
+		if (!*oldConfig.DataRetentionSettings.EnableMessageDeletion && !*oldConfig.DataRetentionSettings.EnableFileDeletion) && (*newConfig.DataRetentionSettings.EnableMessageDeletion || *newConfig.DataRetentionSettings.EnableFileDeletion) {
 			go schedulers.DataRetention.Run()
-		} else if *oldConfig.DataRetentionSettings.Enable && !*newConfig.DataRetentionSettings.Enable {
+		} else if (*oldConfig.DataRetentionSettings.EnableMessageDeletion || *oldConfig.DataRetentionSettings.EnableFileDeletion) && (!*newConfig.DataRetentionSettings.EnableMessageDeletion && !*newConfig.DataRetentionSettings.EnableFileDeletion) {
 			schedulers.DataRetention.Stop()
 		}
 	}
@@ -75,7 +75,7 @@ func (schedulers *Schedulers) handleConfigChange(oldConfig *model.Config, newCon
 func (schedulers *Schedulers) Stop() *Schedulers {
 	utils.RemoveConfigListener(schedulers.listenerId)
 
-	if schedulers.DataRetention != nil && *utils.Cfg.DataRetentionSettings.Enable {
+	if schedulers.DataRetention != nil && (*utils.Cfg.DataRetentionSettings.EnableMessageDeletion || *utils.Cfg.DataRetentionSettings.EnableFileDeletion) {
 		schedulers.DataRetention.Stop()
 	}
 

jobs/workers.go

@@ -46,7 +46,7 @@ func (workers *Workers) Start() *Workers {
 	l4g.Info("Starting workers")
 
 	workers.startOnce.Do(func() {
-		if workers.DataRetention != nil && *utils.Cfg.DataRetentionSettings.Enable {
+		if workers.DataRetention != nil && (*utils.Cfg.DataRetentionSettings.EnableMessageDeletion || *utils.Cfg.DataRetentionSettings.EnableFileDeletion) {
 			go workers.DataRetention.Run()
 		}
 
@@ -68,9 +68,9 @@ func (workers *Workers) Start() *Workers {
 
 func (workers *Workers) handleConfigChange(oldConfig *model.Config, newConfig *model.Config) {
 	if workers.DataRetention != nil {
-		if !*oldConfig.DataRetentionSettings.Enable && *newConfig.DataRetentionSettings.Enable {
+		if (!*oldConfig.DataRetentionSettings.EnableMessageDeletion && !*oldConfig.DataRetentionSettings.EnableFileDeletion) && (*newConfig.DataRetentionSettings.EnableMessageDeletion || *newConfig.DataRetentionSettings.EnableFileDeletion) {
 			go workers.DataRetention.Run()
-		} else if *oldConfig.DataRetentionSettings.Enable && !*newConfig.DataRetentionSettings.Enable {
+		} else if (*oldConfig.DataRetentionSettings.EnableMessageDeletion || *oldConfig.DataRetentionSettings.EnableFileDeletion) && (!*newConfig.DataRetentionSettings.EnableMessageDeletion && !*newConfig.DataRetentionSettings.EnableFileDeletion) {
 			workers.DataRetention.Stop()
 		}
 	}
@@ -97,7 +97,7 @@ func (workers *Workers) Stop() *Workers {
 
 	workers.watcher.Stop()
 
-	if workers.DataRetention != nil && *utils.Cfg.DataRetentionSettings.Enable {
+	if workers.DataRetention != nil && (*utils.Cfg.DataRetentionSettings.EnableMessageDeletion || *utils.Cfg.DataRetentionSettings.EnableFileDeletion) {
 		workers.DataRetention.Stop()
 	}
 

model/config.go

@@ -138,6 +138,10 @@ const (
 	ELASTICSEARCH_SETTINGS_DEFAULT_POST_INDEX_SHARDS               = 1
 	ELASTICSEARCH_SETTINGS_DEFAULT_AGGREGATE_POSTS_AFTER_DAYS      = 365
 	ELASTICSEARCH_SETTINGS_DEFAULT_POSTS_AGGREGATOR_JOB_START_TIME = "03:00"
+
+	DATA_RETENTION_SETTINGS_DEFAULT_MESSAGE_RETENTION_DAYS  = 365
+	DATA_RETENTION_SETTINGS_DEFAULT_FILE_RETENTION_DAYS     = 365
+	DATA_RETENTION_SETTINGS_DEFAULT_DELETION_JOB_START_TIME = "02:00"
 )
 
 type ServiceSettings struct {
@@ -480,7 +484,11 @@ type ElasticsearchSettings struct {
 }
 
 type DataRetentionSettings struct {
-	Enable *bool
+	EnableMessageDeletion *bool
+	EnableFileDeletion    *bool
+	MessageRetentionDays  *int
+	FileRetentionDays     *int
+	DeletionJobStartTime  *string
 }
 
 type JobSettings struct {
@@ -1555,9 +1563,29 @@ func (o *Config) SetDefaults() {
 		*o.ElasticsearchSettings.PostsAggregatorJobStartTime = ELASTICSEARCH_SETTINGS_DEFAULT_POSTS_AGGREGATOR_JOB_START_TIME
 	}
 
-	if o.DataRetentionSettings.Enable == nil {
+	if o.DataRetentionSettings.EnableMessageDeletion == nil {
-		o.DataRetentionSettings.Enable = new(bool)
+		o.DataRetentionSettings.EnableMessageDeletion = new(bool)
-		*o.DataRetentionSettings.Enable = false
+		*o.DataRetentionSettings.EnableMessageDeletion = false
+	}
+
+	if o.DataRetentionSettings.EnableFileDeletion == nil {
+		o.DataRetentionSettings.EnableFileDeletion = new(bool)
+		*o.DataRetentionSettings.EnableMessageDeletion = false
+	}
+
+	if o.DataRetentionSettings.MessageRetentionDays == nil {
+		o.DataRetentionSettings.MessageRetentionDays = new(int)
+		*o.DataRetentionSettings.MessageRetentionDays = DATA_RETENTION_SETTINGS_DEFAULT_MESSAGE_RETENTION_DAYS
+	}
+
+	if o.DataRetentionSettings.FileRetentionDays == nil {
+		o.DataRetentionSettings.FileRetentionDays = new(int)
+		*o.DataRetentionSettings.FileRetentionDays = DATA_RETENTION_SETTINGS_DEFAULT_FILE_RETENTION_DAYS
+	}
+
+	if o.DataRetentionSettings.DeletionJobStartTime == nil {
+		o.DataRetentionSettings.DeletionJobStartTime = new(string)
+		*o.DataRetentionSettings.DeletionJobStartTime = DATA_RETENTION_SETTINGS_DEFAULT_DELETION_JOB_START_TIME
 	}
 
 	if o.JobSettings.RunJobs == nil {
@@ -1816,6 +1844,18 @@ func (o *Config) IsValid() *AppError {
 		return NewAppError("Config.IsValid", "model.config.is_valid.elastic_search.posts_aggregator_job_start_time.app_error", nil, err.Error(), http.StatusBadRequest)
 	}
 
+	if *o.DataRetentionSettings.MessageRetentionDays <= 0 {
+		return NewAppError("Config.IsValid", "model.config.is_valid.data_retention.message_retention_days_too_low.app_error", nil, "", http.StatusBadRequest)
+	}
+
+	if *o.DataRetentionSettings.FileRetentionDays <= 0 {
+		return NewAppError("Config.IsValid", "model.config.is_valid.data_retention.file_retention_days_too_low.app_error", nil, "", http.StatusBadRequest)
+	}
+
+	if _, err := time.Parse("03:04", *o.DataRetentionSettings.DeletionJobStartTime); err != nil {
+		return NewAppError("Config.IsValid", "model.config.is_valid.data_retention.deletion_job_start_time.app_error", nil, err.Error(), http.StatusBadRequest)
+	}
+
 	return nil
 }
 

model/license.go

@@ -54,6 +54,7 @@ type Features struct {
 	Announcement              *bool `json:"announcement"`
 	ThemeManagement           *bool `json:"theme_management"`
 	EmailNotificationContents *bool `json:"email_notification_contents"`
+	DataRetention             *bool `json:"data_retention"`
 
 	// after we enabled more features for webrtc we'll need to control them with this
 	FutureFeatures *bool `json:"future_features"`
@@ -74,6 +75,7 @@ func (f *Features) ToMap() map[string]interface{} {
 		"password":                    *f.PasswordRequirements,
 		"elastic_search":              *f.Elasticsearch,
 		"email_notification_contents": *f.EmailNotificationContents,
+		"data_retention":              *f.DataRetention,
 		"future":                      *f.FutureFeatures,
 	}
 }
@@ -163,6 +165,11 @@ func (f *Features) SetDefaults() {
 		f.EmailNotificationContents = new(bool)
 		*f.EmailNotificationContents = *f.FutureFeatures
 	}
+
+	if f.DataRetention == nil {
+		f.DataRetention = new(bool)
+		*f.DataRetention = *f.FutureFeatures
+	}
 }
 
 func (l *License) IsExpired() bool {

model/license_test.go

@@ -26,8 +26,9 @@ func TestLicenseFeaturesToMap(t *testing.T) {
 	CheckTrue(t, m["saml"].(bool))
 	CheckTrue(t, m["password"].(bool))
 	CheckTrue(t, m["elastic_search"].(bool))
-	CheckTrue(t, m["future"].(bool))
 	CheckTrue(t, m["email_notification_contents"].(bool))
+	CheckTrue(t, m["data_retention"].(bool))
+	CheckTrue(t, m["future"].(bool))
 }
 
 func TestLicenseFeaturesSetDefaults(t *testing.T) {
@@ -48,6 +49,7 @@ func TestLicenseFeaturesSetDefaults(t *testing.T) {
 	CheckTrue(t, *f.PasswordRequirements)
 	CheckTrue(t, *f.Elasticsearch)
 	CheckTrue(t, *f.EmailNotificationContents)
+	CheckTrue(t, *f.DataRetention)
 	CheckTrue(t, *f.FutureFeatures)
 
 	f = Features{}
@@ -67,6 +69,7 @@ func TestLicenseFeaturesSetDefaults(t *testing.T) {
 	*f.SAML = true
 	*f.PasswordRequirements = true
 	*f.Elasticsearch = true
+	*f.DataRetention = true
 	*f.EmailNotificationContents = true
 
 	f.SetDefaults()
@@ -85,6 +88,7 @@ func TestLicenseFeaturesSetDefaults(t *testing.T) {
 	CheckTrue(t, *f.PasswordRequirements)
 	CheckTrue(t, *f.Elasticsearch)
 	CheckTrue(t, *f.EmailNotificationContents)
+	CheckTrue(t, *f.DataRetention)
 	CheckFalse(t, *f.FutureFeatures)
 }
 
@@ -166,6 +170,7 @@ func TestLicenseToFromJson(t *testing.T) {
 	CheckBool(t, *f1.SAML, *f.SAML)
 	CheckBool(t, *f1.PasswordRequirements, *f.PasswordRequirements)
 	CheckBool(t, *f1.Elasticsearch, *f.Elasticsearch)
+	CheckBool(t, *f1.DataRetention, *f.DataRetention)
 	CheckBool(t, *f1.FutureFeatures, *f.FutureFeatures)
 
 	invalid := `{"asdf`