PLT-7503: Create Message Export Scheduled Task and CLI Command (#7612)

* Created message export scheduled task * Added CLI command to immediately kick off an export job * Added email addresses for users joining and leaving the channel to the export * Added support for both MySQL and PostgreSQL * Fixing gofmt error * Added a new ChannelMemberHistory store and associated tests * Updating the ChannelMemberHistory channel as users create/join/leave channels * Added user email to the message export object so it can be included in the actiance export xml * Don't fail to log a leave event if a corresponding join event wasn't logged * Adding copyright notices * Adding message export settings to daily diagnostics report * Added System Console integration for message export * Cleaned up TODOs * Made batch size configurable * Added export from timestamp to CLI command * Made ChannelMemberHistory table updates best effort * Added a context-based timeout option to the message export CLI * Minor PR updates/improvements * Removed unnecessary fields from MessageExport object to reduce query overhead * Removed JSON functions from the message export query in an effort to optimize performance * Changed the way that channel member history queries and purges work to better account for edge cases * Fixing a test I missed with the last refactor * Added file copy functionality to file backend, improved config validation, added default config values * Fixed file copy tests * More concise use of the testing libraries * Fixed context leak error * Changed default export path to correctly place an 'export' directory under the 'data' directory * Can't delete records from a read replica * Fixed copy file tests * Start job workers when license is applied, if configured to do so * Suggestions from the PR * Moar unit tests * Fixed test imports
2017-11-30 09:07:04 -05:00
--- a/model/channel_member_history.go
+++ b/model/channel_member_history.go
@@ -0,0 +1,12 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package model
+
+type ChannelMemberHistory struct {
+	ChannelId string
+	UserId    string
+	UserEmail string `db:"Email"`
+	JoinTime  int64
+	LeaveTime *int64
+}
--- a/model/config.go
+++ b/model/config.go
@@ -8,6 +8,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"path/filepath"
 	"strings"
 	"time"
 )
@@ -1508,6 +1509,36 @@ func (s *PluginSettings) SetDefaults() {
 	}
 }

+type MessageExportSettings struct {
+	EnableExport        *bool
+	DailyRunTime        *string
+	ExportFromTimestamp *int64
+	FileLocation        *string
+	BatchSize           *int
+}
+
+func (s *MessageExportSettings) SetDefaults() {
+	if s.EnableExport == nil {
+		s.EnableExport = NewBool(false)
+	}
+
+	if s.FileLocation == nil {
+		s.FileLocation = NewString("export")
+	}
+
+	if s.DailyRunTime == nil {
+		s.DailyRunTime = NewString("01:00")
+	}
+
+	if s.ExportFromTimestamp == nil {
+		s.ExportFromTimestamp = NewInt64(0)
+	}
+
+	if s.BatchSize == nil {
+		s.BatchSize = NewInt(10000)
+	}
+}
+
 type ConfigFunc func() *Config

 type Config struct {
@@ -1538,6 +1569,7 @@ type Config struct {
 	WebrtcSettings        WebrtcSettings
 	ElasticsearchSettings ElasticsearchSettings
 	DataRetentionSettings DataRetentionSettings
+	MessageExportSettings MessageExportSettings
 	JobSettings           JobSettings
 	PluginSettings        PluginSettings
 }
@@ -1617,6 +1649,7 @@ func (o *Config) SetDefaults() {
 	o.LogSettings.SetDefaults()
 	o.JobSettings.SetDefaults()
 	o.WebrtcSettings.SetDefaults()
+	o.MessageExportSettings.SetDefaults()
 }

 func (o *Config) IsValid() *AppError {
@@ -1680,6 +1713,10 @@ func (o *Config) IsValid() *AppError {
 		return err
 	}

+	if err := o.MessageExportSettings.isValid(o.FileSettings); err != nil {
+		return err
+	}
+
 	return nil
 }

@@ -1998,6 +2035,35 @@ func (ls *LocalizationSettings) isValid() *AppError {
 	return nil
 }

+func (mes *MessageExportSettings) isValid(fs FileSettings) *AppError {
+	if mes.EnableExport == nil {
+		return NewAppError("Config.IsValid", "model.config.is_valid.message_export.enable.app_error", nil, "", http.StatusBadRequest)
+	}
+	if *mes.EnableExport {
+		if mes.ExportFromTimestamp == nil || *mes.ExportFromTimestamp < 0 || *mes.ExportFromTimestamp > time.Now().Unix() {
+			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.export_from.app_error", nil, "", http.StatusBadRequest)
+		} else if mes.DailyRunTime == nil {
+			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.daily_runtime.app_error", nil, "", http.StatusBadRequest)
+		} else if _, err := time.Parse("15:04", *mes.DailyRunTime); err != nil {
+			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.daily_runtime.app_error", nil, err.Error(), http.StatusBadRequest)
+		} else if mes.FileLocation == nil {
+			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.app_error", nil, "", http.StatusBadRequest)
+		} else if mes.BatchSize == nil || *mes.BatchSize < 0 {
+			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.batch_size.app_error", nil, "", http.StatusBadRequest)
+		} else if *fs.DriverName != IMAGE_DRIVER_LOCAL {
+			if absFileDir, err := filepath.Abs(fs.Directory); err != nil {
+				return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.relative", nil, err.Error(), http.StatusBadRequest)
+			} else if absMessageExportDir, err := filepath.Abs(*mes.FileLocation); err != nil {
+				return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.relative", nil, err.Error(), http.StatusBadRequest)
+			} else if !strings.HasPrefix(absMessageExportDir, absFileDir) {
+				// configured export directory must be relative to data directory
+				return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.relative", nil, "", http.StatusBadRequest)
+			}
+		}
+	}
+	return nil
+}
+
 func (o *Config) GetSanitizeOptions() map[string]bool {
 	options := map[string]bool{}
 	options["fullname"] = o.PrivacySettings.ShowFullName
--- a/model/config_test.go
+++ b/model/config_test.go
@@ -5,6 +5,10 @@ package model

 import (
 	"testing"
+
+	"os"
+
+	"github.com/stretchr/testify/require"
 )

 func TestConfigDefaultFileSettingsDirectory(t *testing.T) {
@@ -33,3 +37,109 @@ func TestConfigDefaultFileSettingsS3SSE(t *testing.T) {
 		t.Fatal("FileSettings.AmazonS3SSE should default to false")
 	}
 }
+
+func TestMessageExportSettingsIsValidEnableExportNotSet(t *testing.T) {
+	fs := &FileSettings{}
+	mes := &MessageExportSettings{}
+
+	// should fail fast because mes.EnableExport is not set
+	require.Error(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValidEnableExportFalse(t *testing.T) {
+	fs := &FileSettings{}
+	mes := &MessageExportSettings{
+		EnableExport: NewBool(false),
+	}
+
+	// should fail fast because message export isn't enabled
+	require.Nil(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValidExportFromTimestampInvalid(t *testing.T) {
+	fs := &FileSettings{}
+	mes := &MessageExportSettings{
+		EnableExport: NewBool(true),
+	}
+
+	// should fail fast because export from timestamp isn't set
+	require.Error(t, mes.isValid(*fs))
+
+	mes.ExportFromTimestamp = NewInt64(-1)
+
+	// should fail fast because export from timestamp isn't valid
+	require.Error(t, mes.isValid(*fs))
+
+	mes.ExportFromTimestamp = NewInt64(GetMillis() + 10000)
+
+	// should fail fast because export from timestamp is greater than current time
+	require.Error(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValidDailyRunTimeInvalid(t *testing.T) {
+	fs := &FileSettings{}
+	mes := &MessageExportSettings{
+		EnableExport:        NewBool(true),
+		ExportFromTimestamp: NewInt64(0),
+	}
+
+	// should fail fast because daily runtime isn't set
+	require.Error(t, mes.isValid(*fs))
+
+	mes.DailyRunTime = NewString("33:33:33")
+
+	// should fail fast because daily runtime is invalid format
+	require.Error(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValidBatchSizeInvalid(t *testing.T) {
+	fs := &FileSettings{
+		DriverName: NewString("foo"), // bypass file location check
+	}
+	mes := &MessageExportSettings{
+		EnableExport:        NewBool(true),
+		ExportFromTimestamp: NewInt64(0),
+		DailyRunTime:        NewString("15:04"),
+		FileLocation:        NewString("foo"),
+	}
+
+	// should fail fast because batch size isn't set
+	require.Error(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValidFileLocationInvalid(t *testing.T) {
+	fs := &FileSettings{}
+	mes := &MessageExportSettings{
+		EnableExport:        NewBool(true),
+		ExportFromTimestamp: NewInt64(0),
+		DailyRunTime:        NewString("15:04"),
+		BatchSize:           NewInt(100),
+	}
+
+	// should fail fast because FileLocation isn't set
+	require.Error(t, mes.isValid(*fs))
+
+	// if using the local file driver, there are more rules for FileLocation
+	fs.DriverName = NewString(IMAGE_DRIVER_LOCAL)
+	fs.Directory, _ = os.Getwd()
+	mes.FileLocation = NewString("")
+
+	// should fail fast because file location is not relative to basepath
+	require.Error(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValid(t *testing.T) {
+	fs := &FileSettings{
+		DriverName: NewString("foo"), // bypass file location check
+	}
+	mes := &MessageExportSettings{
+		EnableExport:        NewBool(true),
+		ExportFromTimestamp: NewInt64(0),
+		DailyRunTime:        NewString("15:04"),
+		FileLocation:        NewString("foo"),
+		BatchSize:           NewInt(100),
+	}
+
+	// should pass because everything is valid
+	require.Nil(t, mes.isValid(*fs))
+}
--- a/model/job.go
+++ b/model/job.go
@@ -12,6 +12,7 @@ import (

 const (
 	JOB_TYPE_DATA_RETENTION                 = "data_retention"
+	JOB_TYPE_MESSAGE_EXPORT                 = "message_export"
 	JOB_TYPE_ELASTICSEARCH_POST_INDEXING    = "elasticsearch_post_indexing"
 	JOB_TYPE_ELASTICSEARCH_POST_AGGREGATION = "elasticsearch_post_aggregation"
 	JOB_TYPE_LDAP_SYNC                      = "ldap_sync"
@@ -50,6 +51,7 @@ func (j *Job) IsValid() *AppError {
 	case JOB_TYPE_ELASTICSEARCH_POST_INDEXING:
 	case JOB_TYPE_ELASTICSEARCH_POST_AGGREGATION:
 	case JOB_TYPE_LDAP_SYNC:
+	case JOB_TYPE_MESSAGE_EXPORT:
 	default:
 		return NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+j.Id, http.StatusBadRequest)
 	}
--- a/model/license.go
+++ b/model/license.go
@@ -55,6 +55,7 @@ type Features struct {
 	ThemeManagement           *bool `json:"theme_management"`
 	EmailNotificationContents *bool `json:"email_notification_contents"`
 	DataRetention             *bool `json:"data_retention"`
+	MessageExport             *bool `json:"message_export"`

 	// after we enabled more features for webrtc we'll need to control them with this
 	FutureFeatures *bool `json:"future_features"`
@@ -76,6 +77,7 @@ func (f *Features) ToMap() map[string]interface{} {
 		"elastic_search":              *f.Elasticsearch,
 		"email_notification_contents": *f.EmailNotificationContents,
 		"data_retention":              *f.DataRetention,
+		"message_export":              *f.MessageExport,
 		"future":                      *f.FutureFeatures,
 	}
 }
@@ -152,6 +154,10 @@ func (f *Features) SetDefaults() {
 	if f.DataRetention == nil {
 		f.DataRetention = NewBool(*f.FutureFeatures)
 	}
+
+	if f.MessageExport == nil {
+		f.MessageExport = NewBool(*f.FutureFeatures)
+	}
 }

 func (l *License) IsExpired() bool {
--- a/model/message_export.go
+++ b/model/message_export.go
@@ -0,0 +1,18 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package model
+
+type MessageExport struct {
+	ChannelId          *string
+	ChannelDisplayName *string
+
+	UserId    *string
+	UserEmail *string
+
+	PostId       *string
+	PostCreateAt *int64
+	PostMessage  *string
+	PostType     *string
+	PostFileIds  StringArray
+}