PLT-7503: Create Message Export Scheduled Task and CLI Command (#7612)

* Created message export scheduled task

* Added CLI command to immediately kick off an export job

* Added email addresses for users joining and leaving the channel to the export

* Added support for both MySQL and PostgreSQL

* Fixing gofmt error

* Added a new ChannelMemberHistory store and associated tests

* Updating the ChannelMemberHistory channel as users create/join/leave channels

* Added user email to the message export object so it can be included in the actiance export xml

* Don't fail to log a leave event if a corresponding join event wasn't logged

* Adding copyright notices

* Adding message export settings to daily diagnostics report

* Added System Console integration for message export

* Cleaned up TODOs

* Made batch size configurable

* Added export from timestamp to CLI command

* Made ChannelMemberHistory table updates best effort

* Added a context-based timeout option to the message export CLI

* Minor PR updates/improvements

* Removed unnecessary fields from MessageExport object to reduce query overhead

* Removed JSON functions from the message export query in an effort to optimize performance

* Changed the way that channel member history queries and purges work to better account for edge cases

* Fixing a test I missed with the last refactor

* Added file copy functionality to file backend, improved config validation, added default config values

* Fixed file copy tests

* More concise use of the testing libraries

* Fixed context leak error

* Changed default export path to correctly place an 'export' directory under the 'data' directory

* Can't delete records from a read replica

* Fixed copy file tests

* Start job workers when license is applied, if configured to do so

* Suggestions from the PR

* Moar unit tests

* Fixed test imports

store/storetest/mocks/ChannelMemberHistoryStore.go

@@ -0,0 +1,77 @@
+// Code generated by mockery v1.0.0
+
+// Regenerate this file using `make store-mocks`.
+
+package mocks
+
+import mock "github.com/stretchr/testify/mock"
+import store "github.com/mattermost/mattermost-server/store"
+
+// ChannelMemberHistoryStore is an autogenerated mock type for the ChannelMemberHistoryStore type
+type ChannelMemberHistoryStore struct {
+	mock.Mock
+}
+
+// GetUsersInChannelDuring provides a mock function with given fields: startTime, endTime, channelId
+func (_m *ChannelMemberHistoryStore) GetUsersInChannelDuring(startTime int64, endTime int64, channelId string) store.StoreChannel {
+	ret := _m.Called(startTime, endTime, channelId)
+
+	var r0 store.StoreChannel
+	if rf, ok := ret.Get(0).(func(int64, int64, string) store.StoreChannel); ok {
+		r0 = rf(startTime, endTime, channelId)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(store.StoreChannel)
+		}
+	}
+
+	return r0
+}
+
+// LogJoinEvent provides a mock function with given fields: userId, channelId, joinTime
+func (_m *ChannelMemberHistoryStore) LogJoinEvent(userId string, channelId string, joinTime int64) store.StoreChannel {
+	ret := _m.Called(userId, channelId, joinTime)
+
+	var r0 store.StoreChannel
+	if rf, ok := ret.Get(0).(func(string, string, int64) store.StoreChannel); ok {
+		r0 = rf(userId, channelId, joinTime)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(store.StoreChannel)
+		}
+	}
+
+	return r0
+}
+
+// LogLeaveEvent provides a mock function with given fields: userId, channelId, leaveTime
+func (_m *ChannelMemberHistoryStore) LogLeaveEvent(userId string, channelId string, leaveTime int64) store.StoreChannel {
+	ret := _m.Called(userId, channelId, leaveTime)
+
+	var r0 store.StoreChannel
+	if rf, ok := ret.Get(0).(func(string, string, int64) store.StoreChannel); ok {
+		r0 = rf(userId, channelId, leaveTime)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(store.StoreChannel)
+		}
+	}
+
+	return r0
+}
+
+// PurgeHistoryBefore provides a mock function with given fields: time, channelId
+func (_m *ChannelMemberHistoryStore) PurgeHistoryBefore(time int64, channelId string) store.StoreChannel {
+	ret := _m.Called(time, channelId)
+
+	var r0 store.StoreChannel
+	if rf, ok := ret.Get(0).(func(int64, string) store.StoreChannel); ok {
+		r0 = rf(time, channelId)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(store.StoreChannel)
+		}
+	}
+
+	return r0
+}

store/storetest/mocks/ComplianceStore.go

@@ -61,6 +61,22 @@ func (_m *ComplianceStore) GetAll(offset int, limit int) store.StoreChannel {
 	return r0
 }
 
+// MessageExport provides a mock function with given fields: after, limit
+func (_m *ComplianceStore) MessageExport(after int64, limit int) store.StoreChannel {
+	ret := _m.Called(after, limit)
+
+	var r0 store.StoreChannel
+	if rf, ok := ret.Get(0).(func(int64, int) store.StoreChannel); ok {
+		r0 = rf(after, limit)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(store.StoreChannel)
+		}
+	}
+
+	return r0
+}
+
 // Save provides a mock function with given fields: compliance
 func (_m *ComplianceStore) Save(compliance *model.Compliance) store.StoreChannel {
 	ret := _m.Called(compliance)

store/storetest/mocks/LayeredStoreDatabaseLayer.go

@@ -46,6 +46,22 @@ func (_m *LayeredStoreDatabaseLayer) Channel() store.ChannelStore {
 	return r0
 }
 
+// ChannelMemberHistory provides a mock function with given fields:
+func (_m *LayeredStoreDatabaseLayer) ChannelMemberHistory() store.ChannelMemberHistoryStore {
+	ret := _m.Called()
+
+	var r0 store.ChannelMemberHistoryStore
+	if rf, ok := ret.Get(0).(func() store.ChannelMemberHistoryStore); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(store.ChannelMemberHistoryStore)
+		}
+	}
+
+	return r0
+}
+
 // Close provides a mock function with given fields:
 func (_m *LayeredStoreDatabaseLayer) Close() {
 	_m.Called()

store/storetest/mocks/Store.go

@@ -44,6 +44,22 @@ func (_m *Store) Channel() store.ChannelStore {
 	return r0
 }
 
+// ChannelMemberHistory provides a mock function with given fields:
+func (_m *Store) ChannelMemberHistory() store.ChannelMemberHistoryStore {
+	ret := _m.Called()
+
+	var r0 store.ChannelMemberHistoryStore
+	if rf, ok := ret.Get(0).(func() store.ChannelMemberHistoryStore); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(store.ChannelMemberHistoryStore)
+		}
+	}
+
+	return r0
+}
+
 // Close provides a mock function with given fields:
 func (_m *Store) Close() {
 	_m.Called()