diff --git a/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/compliance/compliance_export_ui_spec.js b/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/compliance/compliance_export_ui_spec.js
index 7d5f565dc2..0023614a45 100644
--- a/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/compliance/compliance_export_ui_spec.js
+++ b/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/compliance/compliance_export_ui_spec.js
@@ -44,7 +44,7 @@ describe('Compliance Export', () => {
});
});
- it('MM-T3435 - Download Compliance Export Files - CSV Format', () => {
+ it.skip('MM-60115 - MM-T3435 - Download Compliance Export Files - CSV Format', () => {
// # Navigate to a team and post an attachment
cy.visit(`/${teamName}/channels/town-square`);
gotoTeamAndPostImage();
@@ -65,7 +65,7 @@ describe('Compliance Export', () => {
});
});
- it('MM-T3438 - Download Compliance Export Files when 0 messages exported', () => {
+ it.skip('MM-60115 - MM-T3438 - Download Compliance Export Files when 0 messages exported', () => {
// # Navigate to a team and post an attachment
cy.visit(`/${teamName}/channels/town-square`);
gotoTeamAndPostImage();
@@ -94,7 +94,7 @@ describe('Compliance Export', () => {
});
});
- it('MM-T1168 - Compliance Export - Run Now, entry appears in job table', () => {
+ it.skip('MM-60115 - MM-T1168 - Compliance Export - Run Now, entry appears in job table', () => {
// # Navigate to a team and post an attachment
cy.visit(`/${teamName}/channels/town-square`);
gotoTeamAndPostImage();
diff --git a/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/compliance/download_bot_compliance_export_file_spec.js b/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/compliance/download_bot_compliance_export_file_spec.js
index fbb2a499a2..eaa17ee2ab 100644
--- a/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/compliance/download_bot_compliance_export_file_spec.js
+++ b/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/compliance/download_bot_compliance_export_file_spec.js
@@ -60,7 +60,7 @@ describe('Compliance Export', () => {
cy.shellRm('-rf', downloadsFolder);
});
- it('MM-T1175_1 - UserType identifies that the message is posted by a bot', () => {
+ it.skip('MM-60115 - MM-T1175_1 - UserType identifies that the message is posted by a bot', () => {
const message = `This is CSV bot message from ${botName} at ${Date.now()}`;
// # Post bot message
@@ -81,7 +81,7 @@ describe('Compliance Export', () => {
);
});
- it('MM-T1175_2 - UserType identifies that the message is posted by a bot', () => {
+ it.skip('MM-60115 - MM-T1175_2 - UserType identifies that the message is posted by a bot', () => {
const message = `This is XML bot message from ${botName} at ${Date.now()}`;
// # Post bot message
diff --git a/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/compliance/download_compliance_export_file_spec.js b/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/compliance/download_compliance_export_file_spec.js
index 5418ff4c74..2f7d00a60e 100644
--- a/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/compliance/download_compliance_export_file_spec.js
+++ b/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/compliance/download_compliance_export_file_spec.js
@@ -52,7 +52,7 @@ describe('Compliance Export', () => {
cy.shellRm('-rf', downloadsFolder);
});
- it('MM-T1172 - Compliance Export - Deleted file is indicated in CSV File Export', () => {
+ it.skip('MM-60115 - MM-T1172 - Compliance Export - Deleted file is indicated in CSV File Export', () => {
// # Go to compliance page and enable export
cy.uiGoToCompliancePage();
cy.uiEnableComplianceExport();
@@ -84,7 +84,7 @@ describe('Compliance Export', () => {
);
});
- it('MM-T1173 - Compliance Export - Deleted file is indicated in Actiance XML File Export', () => {
+ it.skip('MM-60115 - MM-T1173 - Compliance Export - Deleted file is indicated in Actiance XML File Export', () => {
// # Go to compliance page and enable export
cy.uiGoToCompliancePage();
cy.uiEnableComplianceExport(ExportFormatActiance);
@@ -121,7 +121,7 @@ describe('Compliance Export', () => {
});
});
- it('MM-T1176 - Compliance export should include updated post after editing', () => {
+ it.skip('MM-60115 - MM-T1176 - Compliance export should include updated post after editing', () => {
// # Go to compliance page and enable export
cy.uiGoToCompliancePage();
cy.uiEnableComplianceExport(ExportFormatActiance);
@@ -154,7 +154,7 @@ describe('Compliance Export', () => {
);
});
- it('MM-T3305 - Verify Deactivated users are displayed properly in Compliance Exports', () => {
+ it.skip('MM-60115 - MM-T3305 - Verify Deactivated users are displayed properly in Compliance Exports', () => {
// # Post a message by Admin
cy.postMessageAs({
sender: adminUser,
diff --git a/server/channels/app/channel_test.go b/server/channels/app/channel_test.go
index 098a1569af..88e3b267db 100644
--- a/server/channels/app/channel_test.go
+++ b/server/channels/app/channel_test.go
@@ -298,7 +298,7 @@ func TestJoinDefaultChannelsCreatesChannelMemberHistoryRecordTownSquare(t *testi
channel, err := th.App.Srv().Store().Channel().GetByName(th.BasicTeam.Id, "town-square", true)
require.NoError(t, err)
townSquareChannelID := channel.Id
- users, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, townSquareChannelID)
+ users, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{townSquareChannelID})
require.NoError(t, nErr)
initialNumTownSquareUsers := len(users)
@@ -308,7 +308,7 @@ func TestJoinDefaultChannelsCreatesChannelMemberHistoryRecordTownSquare(t *testi
require.Nil(t, appErr)
// there should be a ChannelMemberHistory record for the user
- histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, townSquareChannelID)
+ histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{townSquareChannelID})
require.NoError(t, nErr)
assert.Len(t, histories, initialNumTownSquareUsers+1)
@@ -330,7 +330,7 @@ func TestJoinDefaultChannelsCreatesChannelMemberHistoryRecordOffTopic(t *testing
channel, err := th.App.Srv().Store().Channel().GetByName(th.BasicTeam.Id, "off-topic", true)
require.NoError(t, err)
offTopicChannelId := channel.Id
- users, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, offTopicChannelId)
+ users, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{offTopicChannelId})
require.NoError(t, nErr)
initialNumTownSquareUsers := len(users)
@@ -340,7 +340,7 @@ func TestJoinDefaultChannelsCreatesChannelMemberHistoryRecordOffTopic(t *testing
require.Nil(t, appError)
// there should be a ChannelMemberHistory record for the user
- histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, offTopicChannelId)
+ histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{offTopicChannelId})
require.NoError(t, nErr)
assert.Len(t, histories, initialNumTownSquareUsers+1)
@@ -418,7 +418,7 @@ func TestCreateChannelPublicCreatesChannelMemberHistoryRecord(t *testing.T) {
publicChannel := th.createChannel(th.Context, th.BasicTeam, model.ChannelTypeOpen)
// there should be a ChannelMemberHistory record for the user
- histories, err := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, publicChannel.Id)
+ histories, err := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{publicChannel.Id})
require.NoError(t, err)
assert.Len(t, histories, 1)
assert.Equal(t, th.BasicUser.Id, histories[0].UserId)
@@ -433,7 +433,7 @@ func TestCreateChannelPrivateCreatesChannelMemberHistoryRecord(t *testing.T) {
privateChannel := th.createChannel(th.Context, th.BasicTeam, model.ChannelTypePrivate)
// there should be a ChannelMemberHistory record for the user
- histories, err := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, privateChannel.Id)
+ histories, err := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{privateChannel.Id})
require.NoError(t, err)
assert.Len(t, histories, 1)
assert.Equal(t, th.BasicUser.Id, histories[0].UserId)
@@ -557,7 +557,7 @@ func TestCreateGroupChannelCreatesChannelMemberHistoryRecord(t *testing.T) {
channel, appErr := th.App.CreateGroupChannel(th.Context, groupUserIds, th.BasicUser.Id)
require.Nil(t, appErr, "Failed to create group channel.")
- histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, channel.Id)
+ histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{channel.Id})
require.NoError(t, nErr)
assert.Len(t, histories, 3)
@@ -582,7 +582,7 @@ func TestCreateDirectChannelCreatesChannelMemberHistoryRecord(t *testing.T) {
channel, appErr := th.App.GetOrCreateDirectChannel(th.Context, user1.Id, user2.Id)
require.Nil(t, appErr, "Failed to create direct channel.")
- histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, channel.Id)
+ histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{channel.Id})
require.NoError(t, nErr)
assert.Len(t, histories, 2)
@@ -610,7 +610,7 @@ func TestGetDirectChannelCreatesChannelMemberHistoryRecord(t *testing.T) {
require.Nil(t, appErr, "Failed to create direct channel.")
// there should be a ChannelMemberHistory record for both users
- histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, channel.Id)
+ histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{channel.Id})
require.NoError(t, nErr)
assert.Len(t, histories, 2)
@@ -645,7 +645,7 @@ func TestAddUserToChannelCreatesChannelMemberHistoryRecord(t *testing.T) {
require.Nil(t, appErr, "Failed to add user to channel.")
// there should be a ChannelMemberHistory record for the user
- histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, channel.Id)
+ histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{channel.Id})
require.NoError(t, nErr)
assert.Len(t, histories, 2)
channelMemberHistoryUserIds := make([]string, 0)
@@ -656,6 +656,88 @@ func TestAddUserToChannelCreatesChannelMemberHistoryRecord(t *testing.T) {
assert.Equal(t, groupUserIds, channelMemberHistoryUserIds)
}
+func TestUsersAndPostsCreateActivityInChannel(t *testing.T) {
+ th := Setup(t).InitBasic().DeleteBots()
+ defer th.TearDown()
+
+ user := th.CreateUser()
+ _, err := th.App.AddTeamMember(th.Context, th.BasicTeam.Id, user.Id)
+ require.Nil(t, err, "Failed to add user to team.")
+ user3 := th.CreateUser()
+ _, err = th.App.AddTeamMember(th.Context, th.BasicTeam.Id, user3.Id)
+ require.Nil(t, err, "Failed to add user to team.")
+ user4 := th.CreateUser()
+ _, err = th.App.AddTeamMember(th.Context, th.BasicTeam.Id, user4.Id)
+ require.Nil(t, err, "Failed to add user to team.")
+
+ channel1 := th.createChannel(th.Context, th.BasicTeam, model.ChannelTypeOpen)
+ channel2 := th.createChannel(th.Context, th.BasicTeam, model.ChannelTypeOpen)
+ channel3 := th.createChannel(th.Context, th.BasicTeam, model.ChannelTypeOpen)
+ channel4 := th.createChannel(th.Context, th.BasicTeam, model.ChannelTypeOpen)
+ channel5 := th.createChannel(th.Context, th.BasicTeam, model.ChannelTypeOpen)
+ channel6 := th.createChannel(th.Context, th.BasicTeam, model.ChannelTypeOpen)
+
+ // user3 is already in channel3
+ _, err = th.App.AddUserToChannel(th.Context, user3, channel3, false)
+ require.Nil(t, err, "Failed to add user to channel.")
+ // user4 is already in channel4 (for the second part of the test)
+ _, err = th.App.AddUserToChannel(th.Context, user4, channel4, false)
+ require.Nil(t, err, "Failed to add user to channel.")
+
+ // make sure we don't catch earlier posts
+ time.Sleep(10 * time.Millisecond)
+ testStart := model.GetMillis()
+
+ // Test: previous activity (user3 and 4's adds) aren't showing up:
+ channelIds, nErr := th.App.Srv().Store().ChannelMemberHistory().GetChannelsWithActivityDuring(testStart, testStart+10000)
+ require.NoError(t, nErr)
+ assert.Len(t, channelIds, 0)
+
+ // Posts, adds, and leaves should create activity
+ post := &model.Post{
+ ChannelId: channel1.Id,
+ Message: "root post",
+ UserId: th.BasicUser.Id,
+ }
+ _, err = th.App.CreatePost(th.Context, post, channel1, model.CreatePostFlags{})
+ require.Nil(t, err, "Failed to create post.")
+
+ _, err = th.App.AddUserToChannel(th.Context, user, channel2, false)
+ require.Nil(t, err, "Failed to add user to channel.")
+
+ err = th.App.RemoveUserFromChannel(th.Context, user3.Id, user3.Id, channel3)
+ require.Nil(t, err, "Failed to add user to channel.")
+
+ // Test: there should be a ChannelMemberHistory record for the users and the post
+ channelIds, nErr = th.App.Srv().Store().ChannelMemberHistory().GetChannelsWithActivityDuring(testStart, model.GetMillis())
+ require.NoError(t, nErr)
+ assert.Len(t, channelIds, 3)
+ assert.ElementsMatch(t, []string{channel1.Id, channel2.Id, channel3.Id}, channelIds)
+
+ testEnd := model.GetMillis()
+ // In case the tests are running very fast:
+ time.Sleep(10 * time.Millisecond)
+
+ // Now, we do not find activity for new posts, leaves, or adds after the test is over
+ post2 := &model.Post{
+ ChannelId: channel5.Id,
+ Message: "root post",
+ UserId: th.BasicUser.Id,
+ }
+ err = th.App.RemoveUserFromChannel(th.Context, user4.Id, user4.Id, channel4)
+ require.Nil(t, err, "Failed to create post.")
+ _, err = th.App.CreatePost(th.Context, post2, channel5, model.CreatePostFlags{})
+ require.Nil(t, err, "Failed to create post.")
+ _, err = th.App.AddUserToChannel(th.Context, user, channel6, false)
+ require.Nil(t, err, "Failed to add user to channel.")
+
+ // Test: we get the same three channels as before, not channels 4, 5, 6 which have activity after testEnd
+ channelIds, nErr = th.App.Srv().Store().ChannelMemberHistory().GetChannelsWithActivityDuring(testStart, testEnd)
+ require.NoError(t, nErr)
+ assert.Len(t, channelIds, 3)
+ assert.ElementsMatch(t, []string{channel1.Id, channel2.Id, channel3.Id}, channelIds)
+}
+
func TestLeaveDefaultChannel(t *testing.T) {
th := Setup(t).InitBasic()
defer th.TearDown()
@@ -811,7 +893,7 @@ func TestAddChannelMemberNoUserRequestor(t *testing.T) {
require.Nil(t, appErr, "Failed to add user to channel.")
// there should be a ChannelMemberHistory record for the user
- histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, channel.Id)
+ histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{channel.Id})
require.NoError(t, nErr)
assert.Len(t, histories, 2)
channelMemberHistoryUserIds := make([]string, 0)
diff --git a/server/channels/app/file.go b/server/channels/app/file.go
index 4c05edb79d..6583dac115 100644
--- a/server/channels/app/file.go
+++ b/server/channels/app/file.go
@@ -103,7 +103,7 @@ func (a *App) TestFileStoreConnectionWithConfig(cfg *model.FileSettings) *model.
backend, err = filestore.NewFileBackend(filestore.NewFileBackendSettingsFromConfig(cfg, complianceEnabled, insecure != nil && *insecure))
}
if err != nil {
- return model.NewAppError("FileBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+ return model.NewAppError("FileAttachmentBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
nErr := backend.TestConnection()
if nErr != nil {
diff --git a/server/channels/store/opentracinglayer/opentracinglayer.go b/server/channels/store/opentracinglayer/opentracinglayer.go
index 70e94a15d2..2a1da76375 100644
--- a/server/channels/store/opentracinglayer/opentracinglayer.go
+++ b/server/channels/store/opentracinglayer/opentracinglayer.go
@@ -2933,7 +2933,25 @@ func (s *OpenTracingLayerChannelMemberHistoryStore) GetChannelsLeftSince(userID
return result, err
}
-func (s *OpenTracingLayerChannelMemberHistoryStore) GetUsersInChannelDuring(startTime int64, endTime int64, channelID string) ([]*model.ChannelMemberHistoryResult, error) {
+func (s *OpenTracingLayerChannelMemberHistoryStore) GetChannelsWithActivityDuring(startTime int64, endTime int64) ([]string, error) {
+ origCtx := s.Root.Store.Context()
+ span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "ChannelMemberHistoryStore.GetChannelsWithActivityDuring")
+ s.Root.Store.SetContext(newCtx)
+ defer func() {
+ s.Root.Store.SetContext(origCtx)
+ }()
+
+ defer span.Finish()
+ result, err := s.ChannelMemberHistoryStore.GetChannelsWithActivityDuring(startTime, endTime)
+ if err != nil {
+ span.LogFields(spanlog.Error(err))
+ ext.Error.Set(span, true)
+ }
+
+ return result, err
+}
+
+func (s *OpenTracingLayerChannelMemberHistoryStore) GetUsersInChannelDuring(startTime int64, endTime int64, channelID []string) ([]*model.ChannelMemberHistoryResult, error) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "ChannelMemberHistoryStore.GetUsersInChannelDuring")
s.Root.Store.SetContext(newCtx)
diff --git a/server/channels/store/retrylayer/retrylayer.go b/server/channels/store/retrylayer/retrylayer.go
index ab24d83547..c931566e0e 100644
--- a/server/channels/store/retrylayer/retrylayer.go
+++ b/server/channels/store/retrylayer/retrylayer.go
@@ -3270,7 +3270,28 @@ func (s *RetryLayerChannelMemberHistoryStore) GetChannelsLeftSince(userID string
}
-func (s *RetryLayerChannelMemberHistoryStore) GetUsersInChannelDuring(startTime int64, endTime int64, channelID string) ([]*model.ChannelMemberHistoryResult, error) {
+func (s *RetryLayerChannelMemberHistoryStore) GetChannelsWithActivityDuring(startTime int64, endTime int64) ([]string, error) {
+
+ tries := 0
+ for {
+ result, err := s.ChannelMemberHistoryStore.GetChannelsWithActivityDuring(startTime, endTime)
+ if err == nil {
+ return result, nil
+ }
+ if !isRepeatableError(err) {
+ return result, err
+ }
+ tries++
+ if tries >= 3 {
+ err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+ return result, err
+ }
+ timepkg.Sleep(100 * timepkg.Millisecond)
+ }
+
+}
+
+func (s *RetryLayerChannelMemberHistoryStore) GetUsersInChannelDuring(startTime int64, endTime int64, channelID []string) ([]*model.ChannelMemberHistoryResult, error) {
tries := 0
for {
diff --git a/server/channels/store/sqlstore/channel_member_history_store.go b/server/channels/store/sqlstore/channel_member_history_store.go
index de9ac07106..5596c4c414 100644
--- a/server/channels/store/sqlstore/channel_member_history_store.go
+++ b/server/channels/store/sqlstore/channel_member_history_store.go
@@ -65,27 +65,78 @@ func (s SqlChannelMemberHistoryStore) LogLeaveEvent(userId string, channelId str
return nil
}
-func (s SqlChannelMemberHistoryStore) GetUsersInChannelDuring(startTime int64, endTime int64, channelId string) ([]*model.ChannelMemberHistoryResult, error) {
+func (s SqlChannelMemberHistoryStore) GetChannelsWithActivityDuring(startTime int64, endTime int64) ([]string, error) {
+ // ChannelMemberHistory has been in production for long enough that we are assuming the export period
+ // starts after the ChannelMemberHistory table was first introduced
+ subqueryPosts := s.getSubQueryBuilder().
+ Select("p.ChannelId").
+ Distinct().
+ From("Posts AS p").
+ Where(
+ sq.And{
+ sq.GtOrEq{"p.UpdateAt": startTime},
+ sq.LtOrEq{"p.UpdateAt": endTime},
+ sq.NotLike{"p.Type": "system_%"},
+ })
+
+ subqueryCMH := s.getSubQueryBuilder().
+ Select("cmh.ChannelId").
+ Distinct().
+ From("ChannelMemberHistory AS cmh").
+ Where(
+ sq.Or{
+ sq.And{
+ sq.GtOrEq{"cmh.JoinTime": startTime},
+ sq.LtOrEq{"cmh.JoinTime": endTime},
+ },
+ sq.And{
+ sq.GtOrEq{"cmh.LeaveTime": startTime},
+ sq.LtOrEq{"cmh.LeaveTime": endTime},
+ },
+ })
+
+ unionExpr, args, err := sq.Expr("(? UNION ?) AS cm", subqueryPosts, subqueryCMH).ToSql()
+ if err != nil {
+ return nil, errors.Wrap(err, "GetChannelsWithActivityDuring unionExpr to sql")
+ }
+
+ // no bound args in this expression
+ query, _, err := s.getQueryBuilder().
+ Select("*").
+ From(unionExpr).ToSql()
+ if err != nil {
+ return nil, errors.Wrap(err, "GetChannelsWithActivityDuring query to sql")
+ }
+
+ channelIds := make([]string, 0)
+ if err := s.GetReplica().Select(&channelIds, query, args...); err != nil {
+ return nil, err
+ }
+
+ return channelIds, nil
+}
+
+func (s SqlChannelMemberHistoryStore) GetUsersInChannelDuring(startTime int64, endTime int64, channelIds []string) ([]*model.ChannelMemberHistoryResult, error) {
useChannelMemberHistory, err := s.hasDataAtOrBefore(startTime)
if err != nil {
- return nil, errors.Wrapf(err, "hasDataAtOrBefore startTime=%d endTime=%d channelId=%s", startTime, endTime, channelId)
+ return nil, errors.Wrapf(err, "hasDataAtOrBefore startTime=%d endTime=%d channelId=%v", startTime, endTime, channelIds)
}
if useChannelMemberHistory {
// the export period starts after the ChannelMemberHistory table was first introduced, so we can use the
// data from it for our export
- channelMemberHistories, err2 := s.getFromChannelMemberHistoryTable(startTime, endTime, channelId)
+ channelMemberHistories, err2 := s.getFromChannelMemberHistoryTable(startTime, endTime, channelIds)
if err2 != nil {
- return nil, errors.Wrapf(err2, "getFromChannelMemberHistoryTable startTime=%d endTime=%d channelId=%s", startTime, endTime, channelId)
+ return nil, errors.Wrapf(err2, "getFromChannelMemberHistoryTable startTime=%d endTime=%d channelId=%v", startTime, endTime, channelIds)
}
return channelMemberHistories, nil
}
// the export period starts before the ChannelMemberHistory table was introduced, so we need to fake the
// data by assuming that anybody who has ever joined the channel in question was present during the export period.
// this may not always be true, but it's better than saying that somebody wasn't there when they were
- channelMemberHistories, err := s.getFromChannelMembersTable(startTime, endTime, channelId)
+ channelMemberHistories, err := s.getFromChannelMembersTable(startTime, endTime, channelIds)
if err != nil {
- return nil, errors.Wrapf(err, "getFromChannelMembersTable startTime=%d endTime=%d channelId=%s", startTime, endTime, channelId)
+ return nil, errors.Wrapf(err, "getFromChannelMembersTable startTime=%d endTime=%d channelId=%v", startTime, endTime, channelIds)
}
return channelMemberHistories, nil
}
@@ -108,14 +159,14 @@ func (s SqlChannelMemberHistoryStore) hasDataAtOrBefore(time int64) (bool, error
return false, nil
}
-func (s SqlChannelMemberHistoryStore) getFromChannelMemberHistoryTable(startTime int64, endTime int64, channelId string) ([]*model.ChannelMemberHistoryResult, error) {
+func (s SqlChannelMemberHistoryStore) getFromChannelMemberHistoryTable(startTime int64, endTime int64, channelIds []string) ([]*model.ChannelMemberHistoryResult, error) {
query, args, err := s.getQueryBuilder().
Select(`cmh.*, u.Email AS "Email", u.Username, Bots.UserId IS NOT NULL AS IsBot, u.DeleteAt AS UserDeleteAt`).
From("ChannelMemberHistory cmh").
Join("Users u ON cmh.UserId = u.Id").
LeftJoin("Bots ON Bots.UserId = u.Id").
Where(sq.And{
- sq.Eq{"cmh.ChannelId": channelId},
+ sq.Eq{"cmh.ChannelId": channelIds},
sq.LtOrEq{"cmh.JoinTime": endTime},
sq.Or{
sq.Eq{"cmh.LeaveTime": nil},
@@ -134,14 +185,14 @@ func (s SqlChannelMemberHistoryStore) getFromChannelMemberHistoryTable(startTime
return histories, nil
}
-func (s SqlChannelMemberHistoryStore) getFromChannelMembersTable(startTime int64, endTime int64, channelId string) ([]*model.ChannelMemberHistoryResult, error) {
+func (s SqlChannelMemberHistoryStore) getFromChannelMembersTable(startTime int64, endTime int64, channelIds []string) ([]*model.ChannelMemberHistoryResult, error) {
query, args, err := s.getQueryBuilder().
Select(`ch.ChannelId, ch.UserId, u.Email AS "Email", u.Username, Bots.UserId IS NOT NULL AS IsBot, u.DeleteAt AS UserDeleteAt`).
Distinct().
From("ChannelMembers ch").
Join("Users u ON ch.UserId = u.id").
LeftJoin("Bots ON Bots.UserId = u.id").
- Where(sq.Eq{"ch.ChannelId": channelId}).ToSql()
+ Where(sq.Eq{"ch.ChannelId": channelIds}).ToSql()
if err != nil {
return nil, errors.Wrap(err, "channel_member_history_to_sql")
}
diff --git a/server/channels/store/sqlstore/compliance_store.go b/server/channels/store/sqlstore/compliance_store.go
index 6323260a76..bd902a292e 100644
--- a/server/channels/store/sqlstore/compliance_store.go
+++ b/server/channels/store/sqlstore/compliance_store.go
@@ -286,24 +286,25 @@ func (s SqlComplianceStore) MessageExport(c request.CTX, cursor model.MessageExp
return nil, cursor, errors.Wrap(caseErr, "unable to construct case statement")
}
- query, args, err := s.getQueryBuilder().Select(`Posts.Id AS PostId,
- Posts.CreateAt AS PostCreateAt,
- Posts.UpdateAt AS PostUpdateAt,
- Posts.DeleteAt AS PostDeleteAt,
- Posts.Message AS PostMessage,
- Posts.Type AS PostType,
- Posts.Props AS PostProps,
- Posts.OriginalId AS PostOriginalId,
- Posts.RootId AS PostRootId,
- Posts.FileIds AS PostFileIds,
- Teams.Id AS TeamId,
- Teams.Name AS TeamName,
+ builder := s.getQueryBuilder().Select(`Posts.Id AS PostId,
+ Posts.CreateAt AS PostCreateAt,
+ Posts.UpdateAt AS PostUpdateAt,
+ Posts.DeleteAt AS PostDeleteAt,
+ Posts.EditAt AS PostEditAt,
+ Posts.Message AS PostMessage,
+ Posts.Type AS PostType,
+ Posts.Props AS PostProps,
+ Posts.OriginalId AS PostOriginalId,
+ Posts.RootId AS PostRootId,
+ Posts.FileIds AS PostFileIds,
+ Teams.Id AS TeamId,
+ Teams.Name AS TeamName,
Teams.DisplayName AS TeamDisplayName,
- Channels.Id AS ChannelId,
- Channels.Name AS ChannelName,
- Channels.Type AS ChannelType,
- Users.Id AS UserId,
- Users.Email AS UserEmail,
+ Channels.Id AS ChannelId,
+ Channels.Name AS ChannelName,
+ Channels.Type AS ChannelType,
+ Users.Id AS UserId,
+ Users.Email AS UserEmail,
Users.Username,
Bots.UserId IS NOT NULL AS IsBot`).
Column(caseStmt+" AS ChannelDisplayName", caseArgs...).
@@ -323,8 +324,13 @@ func (s SqlComplianceStore) MessageExport(c request.CTX, cursor model.MessageExp
sq.NotLike{"Posts.Type": "system_%"},
}).
OrderBy("PostUpdateAt, PostId").
- Limit(uint64(limit)).
- ToSql()
+ Limit(uint64(limit))
+
+ if cursor.UntilUpdateAt > 0 {
+ builder = builder.Where(sq.LtOrEq{"Posts.UpdateAt": cursor.UntilUpdateAt})
+ }
+
+ query, args, err := builder.ToSql()
if err != nil {
return nil, cursor, errors.Wrap(err, "unable to construct query to export messages")
}
diff --git a/server/channels/store/sqlstore/post_store.go b/server/channels/store/sqlstore/post_store.go
index 93605e3f8d..d0cb61127d 100644
--- a/server/channels/store/sqlstore/post_store.go
+++ b/server/channels/store/sqlstore/post_store.go
@@ -2404,15 +2404,14 @@ func (s *SqlPostStore) AnalyticsPostCount(options *model.PostCountOptions) (int6
})
}
- queryString, args, err := query.ToSql()
- if err != nil {
- return 0, errors.Wrap(err, "post_tosql")
+ if options.UntilUpdateAt > 0 {
+ query = query.Where(sq.LtOrEq{"p.UpdateAt": options.UntilUpdateAt})
}
var v int64
- err = s.GetReplica().Get(&v, queryString, args...)
+ err := s.GetReplica().GetBuilder(&v, query)
if err != nil {
- return 0, errors.Wrap(err, "failed to count Posts")
+ return 0, fmt.Errorf("post_tosql failed or failed to count Posts: %w", err)
}
return v, nil
diff --git a/server/channels/store/sqlstore/store.go b/server/channels/store/sqlstore/store.go
index 973e1ca23f..88535e4ce6 100644
--- a/server/channels/store/sqlstore/store.go
+++ b/server/channels/store/sqlstore/store.go
@@ -35,6 +35,8 @@ import (
type migrationDirection string
+type Option func(s *SqlStore) error
+
const (
IndexTypeFullText = "full_text"
IndexTypeFullTextFunc = "full_text_func"
@@ -139,12 +141,20 @@ type SqlStore struct {
isBinaryParam bool
pgDefaultTextSearchConfig string
+ skipMigrations bool
quitMonitor chan struct{}
wgMonitor *sync.WaitGroup
}
-func New(settings model.SqlSettings, logger mlog.LoggerIFace, metrics einterfaces.MetricsInterface) (*SqlStore, error) {
+func SkipMigrations() Option {
+ return func(s *SqlStore) error {
+ s.skipMigrations = true
+ return nil
+ }
+}
+
+func New(settings model.SqlSettings, logger mlog.LoggerIFace, metrics einterfaces.MetricsInterface, options ...Option) (*SqlStore, error) {
store := &SqlStore{
rrCounter: 0,
srCounter: 0,
@@ -155,6 +165,12 @@ func New(settings model.SqlSettings, logger mlog.LoggerIFace, metrics einterface
wgMonitor: &sync.WaitGroup{},
}
+ for _, option := range options {
+ if err := option(store); err != nil {
+ return nil, fmt.Errorf("failed to apply option: %w", err)
+ }
+ }
+
err := store.initConnection()
if err != nil {
return nil, errors.Wrap(err, "error setting up connections")
@@ -178,9 +194,11 @@ func New(settings model.SqlSettings, logger mlog.LoggerIFace, metrics einterface
return nil, errors.Wrap(err, "error while checking DB collation")
}
- err = store.migrate(migrationsDirectionUp, false)
- if err != nil {
- return nil, errors.Wrap(err, "failed to apply database migrations")
+ if !store.skipMigrations {
+ err = store.migrate(migrationsDirectionUp, false)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to apply database migrations")
+ }
}
store.isBinaryParam, err = store.computeBinaryParam()
diff --git a/server/channels/store/sqlstore/store_test.go b/server/channels/store/sqlstore/store_test.go
index 344a72e696..d59bb160e2 100644
--- a/server/channels/store/sqlstore/store_test.go
+++ b/server/channels/store/sqlstore/store_test.go
@@ -4,6 +4,7 @@
package sqlstore
import (
+ "database/sql"
"fmt"
"os"
"path/filepath"
@@ -977,3 +978,40 @@ func TestGetAppliedMigrations(t *testing.T) {
})
}
}
+
+func TestSkipMigrationsOption(t *testing.T) {
+ testDrivers := []string{
+ model.DatabaseDriverPostgres,
+ model.DatabaseDriverMysql,
+ }
+
+ logger := mlog.CreateConsoleTestLogger(t)
+
+ for _, d := range testDrivers {
+ driver := d
+ t.Run("Should not apply migrations when skipMigrations=true for "+driver, func(t *testing.T) {
+ t.Parallel()
+ settings, err := makeSqlSettings(driver)
+ if err != nil {
+ t.Skip(err)
+ }
+
+ store, err := New(*settings, logger, nil, SkipMigrations())
+ require.NoError(t, err)
+
+ _, err = store.GetDBSchemaVersion()
+ assert.Error(t, err)
+
+ // Unfortunately there's no error for table doesn't exist, but we know it won't be ErrNoRows:
+ assert.True(t, !errors.Is(err, sql.ErrNoRows))
+
+ // And we know what each db will return:
+ if driver == model.DatabaseDriverPostgres {
+ assert.Contains(t, err.Error(), "pq: relation \"db_migrations\" does not exist")
+ } else {
+ // Mysql includes the random db name, so test the end:
+ assert.Contains(t, err.Error(), ".db_migrations' doesn't exist")
+ }
+ })
+ }
+}
diff --git a/server/channels/store/store.go b/server/channels/store/store.go
index f421709b80..4912e3cf1d 100644
--- a/server/channels/store/store.go
+++ b/server/channels/store/store.go
@@ -318,7 +318,8 @@ type ChannelStore interface {
type ChannelMemberHistoryStore interface {
LogJoinEvent(userID string, channelID string, joinTime int64) error
LogLeaveEvent(userID string, channelID string, leaveTime int64) error
- GetUsersInChannelDuring(startTime int64, endTime int64, channelID string) ([]*model.ChannelMemberHistoryResult, error)
+ GetUsersInChannelDuring(startTime int64, endTime int64, channelID []string) ([]*model.ChannelMemberHistoryResult, error)
+ GetChannelsWithActivityDuring(startTime int64, endTime int64) ([]string, error)
PermanentDeleteBatchForRetentionPolicies(now, globalPolicyEndTime, limit int64, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error)
DeleteOrphanedRows(limit int) (deleted int64, err error)
PermanentDeleteBatch(endTime int64, limit int64) (int64, error)
diff --git a/server/channels/store/storetest/channel_member_history_store.go b/server/channels/store/storetest/channel_member_history_store.go
index 1c5d44c86a..3ee0240847 100644
--- a/server/channels/store/storetest/channel_member_history_store.go
+++ b/server/channels/store/storetest/channel_member_history_store.go
@@ -4,9 +4,9 @@
package storetest
import (
- "testing"
-
"math"
+ "testing"
+ "time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -21,6 +21,7 @@ func TestChannelMemberHistoryStore(t *testing.T, rctx request.CTX, ss store.Stor
t.Run("TestLogLeaveEvent", func(t *testing.T) { testLogLeaveEvent(t, rctx, ss) })
t.Run("TestGetUsersInChannelAtChannelMemberHistory", func(t *testing.T) { testGetUsersInChannelAtChannelMemberHistory(t, rctx, ss) })
t.Run("TestGetUsersInChannelAtChannelMembers", func(t *testing.T) { testGetUsersInChannelAtChannelMembers(t, rctx, ss) })
+ t.Run("TestGetChannelsWithActivityDuring", func(t *testing.T) { testGetChannelsWithActivityDuring(t, rctx, ss) })
t.Run("TestPermanentDeleteBatch", func(t *testing.T) { testPermanentDeleteBatch(t, rctx, ss) })
t.Run("TestPermanentDeleteBatchForRetentionPolicies", func(t *testing.T) { testPermanentDeleteBatchForRetentionPolicies(t, rctx, ss) })
t.Run("TestGetChannelsLeftSince", func(t *testing.T) { testGetChannelsLeftSince(t, rctx, ss) })
@@ -81,6 +82,160 @@ func testLogLeaveEvent(t *testing.T, rctx request.CTX, ss store.Store) {
assert.NoError(t, err)
}
+func testGetChannelsWithActivityDuring(t *testing.T, rctx request.CTX, ss store.Store) {
+ // Need to wait to make sure channels and posts have nothing in them for this test.
+ time.Sleep(101 * time.Millisecond)
+
+ // create three test channels
+ ch1 := &model.Channel{
+ TeamId: model.NewId(),
+ DisplayName: "Display " + model.NewId(),
+ Name: model.NewId(),
+ Type: model.ChannelTypeOpen,
+ }
+ channel1, err := ss.Channel().Save(rctx, ch1, -1)
+ require.NoError(t, err)
+
+ // channel2 will have no activity until case 6 (shouldn't show up until then)
+ ch2 := &model.Channel{
+ TeamId: model.NewId(),
+ DisplayName: "Display " + model.NewId(),
+ Name: model.NewId(),
+ Type: model.ChannelTypeOpen,
+ }
+ channel2, err := ss.Channel().Save(rctx, ch2, -1)
+ require.NoError(t, err)
+
+ // and two test users
+ user1 := model.User{
+ Email: MakeEmail(),
+ Nickname: model.NewId(),
+ Username: model.NewUsername(),
+ }
+ userPtr, err := ss.User().Save(rctx, &user1)
+ require.NoError(t, err)
+ user1 = *userPtr
+
+ user2 := model.User{
+ Email: MakeEmail(),
+ Nickname: model.NewId(),
+ Username: model.NewUsername(),
+ }
+ userPtr, err = ss.User().Save(rctx, &user2)
+ require.NoError(t, err)
+ user2 = *userPtr
+
+ now := model.GetMillis()
+ originalNow := now
+
+ // user2 joins channel2 before test
+ err = ss.ChannelMemberHistory().LogJoinEvent(user2.Id, channel2.Id, now-2000)
+ require.NoError(t, err)
+
+ // case 7: assert no activity for time period before channel activity
+ channelIds, err := ss.ChannelMemberHistory().GetChannelsWithActivityDuring(originalNow-100, originalNow)
+ require.NoError(t, err)
+ assert.Empty(t, channelIds)
+
+ // case 1: user1 was in channel before period, doesn't show activity
+ err = ss.ChannelMemberHistory().LogJoinEvent(user1.Id, channel1.Id, now-1100)
+ require.NoError(t, err)
+
+ channelIds, err = ss.ChannelMemberHistory().GetChannelsWithActivityDuring(now, now+1000)
+ require.NoError(t, err)
+ assert.Empty(t, channelIds)
+
+ // case 2: user1 leaves, shows activity
+ err = ss.ChannelMemberHistory().LogLeaveEvent(user1.Id, channel1.Id, now+1)
+ require.NoError(t, err)
+
+ channelIds, err = ss.ChannelMemberHistory().GetChannelsWithActivityDuring(now, now+1000)
+ require.NoError(t, err)
+ assert.Equal(t, channelIds, []string{channel1.Id})
+
+ // case 3: user1 joins, shows activity
+ err = ss.ChannelMemberHistory().LogJoinEvent(user1.Id, channel1.Id, now+2)
+ require.NoError(t, err)
+
+ channelIds, err = ss.ChannelMemberHistory().GetChannelsWithActivityDuring(now+2, now+1000)
+ require.NoError(t, err)
+ assert.Equal(t, channelIds, []string{channel1.Id})
+
+ // case 4: new post shows activity
+ channelIds, err = ss.ChannelMemberHistory().GetChannelsWithActivityDuring(now+3, now+1000)
+ require.NoError(t, err)
+ assert.Empty(t, channelIds)
+
+ post := &model.Post{
+ ChannelId: channel1.Id,
+ Message: "root post",
+ UserId: user1.Id,
+ CreateAt: now + 4,
+ UpdateAt: now + 4,
+ }
+ post, err = ss.Post().Save(rctx, post)
+ require.NoError(t, err)
+
+ channelIds, err = ss.ChannelMemberHistory().GetChannelsWithActivityDuring(now+3, now+1000)
+ require.NoError(t, err)
+ assert.Equal(t, channelIds, []string{channel1.Id})
+
+ // case 5: update shows activity
+ // need to wait because update uses getMillis
+ time.Sleep(10 * time.Millisecond)
+ now = model.GetMillis()
+
+ channelIds, err = ss.ChannelMemberHistory().GetChannelsWithActivityDuring(now-1, now+1000)
+ require.NoError(t, err)
+ assert.Empty(t, channelIds)
+
+ newPost := post.Clone()
+ newPost.Message = "edited message"
+ _, err = ss.Post().Update(rctx, newPost, post)
+ require.NoError(t, err)
+
+ channelIds, err = ss.ChannelMemberHistory().GetChannelsWithActivityDuring(now-1, now+1000)
+ require.NoError(t, err)
+ assert.Equal(t, channelIds, []string{channel1.Id})
+
+ // case 6: get both activity from posts and from join/leave;
+ // - also, sql deduplicates two channel1 results (from post and channel history tables)
+ time.Sleep(1 * time.Millisecond)
+ now = model.GetMillis()
+
+ channelIds, err = ss.ChannelMemberHistory().GetChannelsWithActivityDuring(now, now+1000)
+ require.NoError(t, err)
+ assert.Empty(t, channelIds)
+
+ post2 := &model.Post{
+ ChannelId: channel1.Id,
+ Message: "root post",
+ UserId: user1.Id,
+ CreateAt: now + 11,
+ UpdateAt: now + 11,
+ }
+ _, err = ss.Post().Save(rctx, post2)
+ require.NoError(t, err)
+ err = ss.ChannelMemberHistory().LogLeaveEvent(user1.Id, channel1.Id, now+12)
+ require.NoError(t, err)
+ err = ss.ChannelMemberHistory().LogLeaveEvent(user2.Id, channel2.Id, now+13)
+ require.NoError(t, err)
+
+ channelIds, err = ss.ChannelMemberHistory().GetChannelsWithActivityDuring(now+10, now+1000)
+ require.NoError(t, err)
+ assert.ElementsMatch(t, channelIds, []string{channel1.Id, channel2.Id})
+
+ // case 7: still no activity for period before tests
+ channelIds, err = ss.ChannelMemberHistory().GetChannelsWithActivityDuring(originalNow-100, originalNow)
+ require.NoError(t, err)
+ assert.Empty(t, channelIds)
+
+ // case 8: no activity for period after tests
+ channelIds, err = ss.ChannelMemberHistory().GetChannelsWithActivityDuring(now+100, now+1000)
+ require.NoError(t, err)
+ assert.Empty(t, channelIds)
+}
+
func testGetUsersInChannelAtChannelMemberHistory(t *testing.T, rctx request.CTX, ss store.Store) {
// create a test channel
ch := &model.Channel{
@@ -119,12 +274,12 @@ func testGetUsersInChannelAtChannelMemberHistory(t *testing.T, rctx request.CTX,
require.NoError(t, err)
// case 1: user joins and leaves the channel before the export period begins
- channelMembers, err := ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime-500, joinTime-100, channel.Id)
+ channelMembers, err := ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime-500, joinTime-100, []string{channel.Id})
require.NoError(t, err)
assert.Empty(t, channelMembers)
// case 2: user joins the channel after the export period begins, but has not yet left the channel when the export period ends
- channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime-100, joinTime+500, channel.Id)
+ channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime-100, joinTime+500, []string{channel.Id})
require.NoError(t, err)
assert.Len(t, channelMembers, 1)
assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
@@ -135,7 +290,7 @@ func testGetUsersInChannelAtChannelMemberHistory(t *testing.T, rctx request.CTX,
assert.Nil(t, channelMembers[0].LeaveTime)
// case 3: user joins the channel before the export period begins, but has not yet left the channel when the export period ends
- channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime+100, joinTime+500, channel.Id)
+ channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime+100, joinTime+500, []string{channel.Id})
require.NoError(t, err)
assert.Len(t, channelMembers, 1)
assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
@@ -150,7 +305,7 @@ func testGetUsersInChannelAtChannelMemberHistory(t *testing.T, rctx request.CTX,
require.NoError(t, err)
// case 4: user joins the channel before the export period begins, but has not yet left the channel when the export period ends
- channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime+100, leaveTime-100, channel.Id)
+ channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime+100, leaveTime-100, []string{channel.Id})
require.NoError(t, err)
assert.Len(t, channelMembers, 1)
assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
@@ -161,7 +316,7 @@ func testGetUsersInChannelAtChannelMemberHistory(t *testing.T, rctx request.CTX,
assert.Equal(t, leaveTime, *channelMembers[0].LeaveTime)
// case 5: user joins the channel after the export period begins, and leaves the channel before the export period ends
- channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime-100, leaveTime+100, channel.Id)
+ channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime-100, leaveTime+100, []string{channel.Id})
require.NoError(t, err)
assert.Len(t, channelMembers, 1)
assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
@@ -172,7 +327,7 @@ func testGetUsersInChannelAtChannelMemberHistory(t *testing.T, rctx request.CTX,
assert.Equal(t, leaveTime, *channelMembers[0].LeaveTime)
// case 6: user has joined and left the channel long before the export period begins
- channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(leaveTime+100, leaveTime+200, channel.Id)
+ channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(leaveTime+100, leaveTime+200, []string{channel.Id})
require.NoError(t, err)
assert.Empty(t, channelMembers)
}
@@ -223,7 +378,7 @@ func testGetUsersInChannelAtChannelMembers(t *testing.T, rctx request.CTX, ss st
// the past, even though the time that they were actually in the channel doesn't necessarily overlap with the export period
// case 1: user joins and leaves the channel before the export period begins
- channelMembers, err := ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime-500, joinTime-100, channel.Id)
+ channelMembers, err := ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime-500, joinTime-100, []string{channel.Id})
require.NoError(t, err)
assert.Len(t, channelMembers, 1)
assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
@@ -234,7 +389,7 @@ func testGetUsersInChannelAtChannelMembers(t *testing.T, rctx request.CTX, ss st
assert.Equal(t, joinTime-100, *channelMembers[0].LeaveTime)
// case 2: user joins the channel after the export period begins, but has not yet left the channel when the export period ends
- channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime-100, joinTime+500, channel.Id)
+ channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime-100, joinTime+500, []string{channel.Id})
require.NoError(t, err)
assert.Len(t, channelMembers, 1)
assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
@@ -245,7 +400,7 @@ func testGetUsersInChannelAtChannelMembers(t *testing.T, rctx request.CTX, ss st
assert.Equal(t, joinTime+500, *channelMembers[0].LeaveTime)
// case 3: user joins the channel before the export period begins, but has not yet left the channel when the export period ends
- channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime+100, joinTime+500, channel.Id)
+ channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime+100, joinTime+500, []string{channel.Id})
require.NoError(t, err)
assert.Len(t, channelMembers, 1)
assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
@@ -256,7 +411,7 @@ func testGetUsersInChannelAtChannelMembers(t *testing.T, rctx request.CTX, ss st
assert.Equal(t, joinTime+500, *channelMembers[0].LeaveTime)
// case 4: user joins the channel before the export period begins, but has not yet left the channel when the export period ends
- channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime+100, leaveTime-100, channel.Id)
+ channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime+100, leaveTime-100, []string{channel.Id})
require.NoError(t, err)
assert.Len(t, channelMembers, 1)
assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
@@ -267,7 +422,7 @@ func testGetUsersInChannelAtChannelMembers(t *testing.T, rctx request.CTX, ss st
assert.Equal(t, leaveTime-100, *channelMembers[0].LeaveTime)
// case 5: user joins the channel after the export period begins, and leaves the channel before the export period ends
- channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime-100, leaveTime+100, channel.Id)
+ channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime-100, leaveTime+100, []string{channel.Id})
require.NoError(t, err)
assert.Len(t, channelMembers, 1)
assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
@@ -278,7 +433,7 @@ func testGetUsersInChannelAtChannelMembers(t *testing.T, rctx request.CTX, ss st
assert.Equal(t, leaveTime+100, *channelMembers[0].LeaveTime)
// case 6: user has joined and left the channel long before the export period begins
- channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(leaveTime+100, leaveTime+200, channel.Id)
+ channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(leaveTime+100, leaveTime+200, []string{channel.Id})
require.NoError(t, err)
assert.Len(t, channelMembers, 1)
assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
@@ -332,7 +487,7 @@ func testPermanentDeleteBatch(t *testing.T, rctx request.CTX, ss store.Store) {
require.NoError(t, err)
// in between the join time and the leave time, both users were members of the channel
- channelMembers, err := ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime+10, leaveTime-10, channel.Id)
+ channelMembers, err := ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime+10, leaveTime-10, []string{channel.Id})
require.NoError(t, err)
assert.Len(t, channelMembers, 2)
@@ -343,7 +498,7 @@ func testPermanentDeleteBatch(t *testing.T, rctx request.CTX, ss store.Store) {
assert.NotEqual(t, int64(0), rowsDeleted)
// after the delete, there should be one less member in the channel
- channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime+10, leaveTime-10, channel.Id)
+ channelMembers, err = ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime+10, leaveTime-10, []string{channel.Id})
require.NoError(t, err)
assert.Len(t, channelMembers, 1)
assert.Equal(t, user2.Id, channelMembers[0].UserId)
@@ -387,7 +542,7 @@ func testPermanentDeleteBatchForRetentionPolicies(t *testing.T, rctx request.CTX
_, _, err = ss.ChannelMemberHistory().PermanentDeleteBatchForRetentionPolicies(
nowMillis, 0, limit, model.RetentionPolicyCursor{})
require.NoError(t, err)
- result, err := ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime, leaveTime, channel.Id)
+ result, err := ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime, leaveTime, []string{channel.Id})
require.NoError(t, err)
require.Empty(t, result, "history should have been deleted by channel policy")
rows, err := ss.RetentionPolicy().GetIdsForDeletionByTableName("ChannelMemberHistory", 1000)
diff --git a/server/channels/store/storetest/compliance_store.go b/server/channels/store/storetest/compliance_store.go
index 7789d02828..9bfbacd40d 100644
--- a/server/channels/store/storetest/compliance_store.go
+++ b/server/channels/store/storetest/compliance_store.go
@@ -58,6 +58,7 @@ func TestComplianceStore(t *testing.T, rctx request.CTX, ss store.Store) {
t.Run("MessageEditAfterExportMessage", func(t *testing.T) { testEditAfterExportMessage(t, rctx, ss) })
t.Run("MessageDeleteExportMessage", func(t *testing.T) { testDeleteExportMessage(t, rctx, ss) })
t.Run("MessageDeleteAfterExportMessage", func(t *testing.T) { testDeleteAfterExportMessage(t, rctx, ss) })
+ t.Run("MessageExport_UntilUpdateAt", func(t *testing.T) { testMessageExportUntilUpdateAt(t, rctx, ss) })
}
func testComplianceStore(t *testing.T, rctx request.CTX, ss store.Store) {
@@ -1174,3 +1175,105 @@ func testDeleteAfterExportMessage(t *testing.T, rctx request.CTX, ss store.Store
assert.Equal(t, user1.Email, *v.UserEmail)
assert.Equal(t, user1.Username, *v.Username)
}
+
+func testMessageExportUntilUpdateAt(t *testing.T, rctx request.CTX, ss store.Store) {
+ defer cleanupStoreState(t, rctx, ss)
+
+ // get the starting number of message export entries
+ startTime := model.GetMillis()
+ messages, _, err := ss.Compliance().MessageExport(rctx, model.MessageExportCursor{LastPostUpdateAt: startTime - 10}, 10)
+ require.NoError(t, err)
+ assert.Equal(t, 0, len(messages))
+
+ // need a team
+ team := &model.Team{
+ DisplayName: "DisplayName",
+ Name: model.NewId(),
+ Email: MakeEmail(),
+ Type: model.TeamOpen,
+ }
+ team, err = ss.Team().Save(team)
+ require.NoError(t, err)
+
+ // and two users that are a part of that team
+ user1 := &model.User{
+ Email: MakeEmail(),
+ Username: model.NewUsername(),
+ }
+ user1, err = ss.User().Save(rctx, user1)
+ require.NoError(t, err)
+ _, nErr := ss.Team().SaveMember(rctx, &model.TeamMember{
+ TeamId: team.Id,
+ UserId: user1.Id,
+ }, -1)
+ require.NoError(t, nErr)
+
+ user2 := &model.User{
+ Email: MakeEmail(),
+ Username: model.NewUsername(),
+ }
+ user2, err = ss.User().Save(rctx, user2)
+ require.NoError(t, err)
+ _, nErr = ss.Team().SaveMember(rctx, &model.TeamMember{
+ TeamId: team.Id,
+ UserId: user2.Id,
+ }, -1)
+ require.NoError(t, nErr)
+
+ // need a public channel
+ channel := &model.Channel{
+ TeamId: team.Id,
+ Name: model.NewId(),
+ DisplayName: "Public Channel",
+ Type: model.ChannelTypeOpen,
+ }
+ channel, nErr = ss.Channel().Save(rctx, channel, -1)
+ require.NoError(t, nErr)
+
+ var posts []*model.Post
+ // user1 posts ten times in the public channel
+ for i := 0; i < 10; i++ {
+ post := &model.Post{
+ ChannelId: channel.Id,
+ UserId: user1.Id,
+ CreateAt: startTime + int64(i),
+ UpdateAt: startTime + int64(i),
+ Message: model.NewId(),
+ }
+ post, err = ss.Post().Save(rctx, post)
+ require.NoError(t, err)
+ posts = append(posts, post)
+ }
+
+ // fetch 5 starting from the third post, using LastPostUpdateAt and UntilUpdateAt.
+ // UntilUpdateAt is inclusive
+ messageExportMap := map[string]model.MessageExport{}
+ messages, _, err = ss.Compliance().MessageExport(rctx, model.MessageExportCursor{LastPostUpdateAt: posts[2].UpdateAt, UntilUpdateAt: posts[2].UpdateAt + 4}, 10000)
+ require.NoError(t, err)
+ assert.Equal(t, 5, len(messages))
+
+ for _, v := range messages {
+ messageExportMap[*v.PostId] = *v
+ }
+
+ for i := 2; i < 7; i++ {
+ assert.Equal(t, posts[i].Id, *messageExportMap[posts[i].Id].PostId)
+ assert.Equal(t, posts[i].CreateAt, *messageExportMap[posts[i].Id].PostCreateAt)
+ assert.Equal(t, posts[i].Message, *messageExportMap[posts[i].Id].PostMessage)
+ assert.Equal(t, channel.Id, *messageExportMap[posts[i].Id].ChannelId)
+ assert.Equal(t, channel.DisplayName, *messageExportMap[posts[i].Id].ChannelDisplayName)
+ assert.Equal(t, user1.Id, *messageExportMap[posts[i].Id].UserId)
+ assert.Equal(t, user1.Email, *messageExportMap[posts[i].Id].UserEmail)
+ assert.Equal(t, user1.Username, *messageExportMap[posts[i].Id].Username)
+ }
+
+ // Also test AnalyticsPostCount because they are used in tandem for MessageExports
+ count, err := ss.Post().AnalyticsPostCount(&model.PostCountOptions{
+ TeamId: channel.TeamId,
+ ExcludeSystemPosts: true,
+ SinceUpdateAt: posts[2].UpdateAt,
+ UntilUpdateAt: posts[2].UpdateAt + 4,
+ })
+ require.NoError(t, err)
+ require.Equal(t, 5, int(count))
+}
diff --git a/server/channels/store/storetest/mocks/ChannelMemberHistoryStore.go b/server/channels/store/storetest/mocks/ChannelMemberHistoryStore.go
index 68139fdb94..c1327afaa3 100644
--- a/server/channels/store/storetest/mocks/ChannelMemberHistoryStore.go
+++ b/server/channels/store/storetest/mocks/ChannelMemberHistoryStore.go
@@ -72,8 +72,38 @@ func (_m *ChannelMemberHistoryStore) GetChannelsLeftSince(userID string, since i
return r0, r1
}
+// GetChannelsWithActivityDuring provides a mock function with given fields: startTime, endTime
+func (_m *ChannelMemberHistoryStore) GetChannelsWithActivityDuring(startTime int64, endTime int64) ([]string, error) {
+ ret := _m.Called(startTime, endTime)
+
+ if len(ret) == 0 {
+ panic("no return value specified for GetChannelsWithActivityDuring")
+ }
+
+ var r0 []string
+ var r1 error
+ if rf, ok := ret.Get(0).(func(int64, int64) ([]string, error)); ok {
+ return rf(startTime, endTime)
+ }
+ if rf, ok := ret.Get(0).(func(int64, int64) []string); ok {
+ r0 = rf(startTime, endTime)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).([]string)
+ }
+ }
+
+ if rf, ok := ret.Get(1).(func(int64, int64) error); ok {
+ r1 = rf(startTime, endTime)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
// GetUsersInChannelDuring provides a mock function with given fields: startTime, endTime, channelID
-func (_m *ChannelMemberHistoryStore) GetUsersInChannelDuring(startTime int64, endTime int64, channelID string) ([]*model.ChannelMemberHistoryResult, error) {
+func (_m *ChannelMemberHistoryStore) GetUsersInChannelDuring(startTime int64, endTime int64, channelID []string) ([]*model.ChannelMemberHistoryResult, error) {
ret := _m.Called(startTime, endTime, channelID)
if len(ret) == 0 {
@@ -82,10 +112,10 @@ func (_m *ChannelMemberHistoryStore) GetUsersInChannelDuring(startTime int64, en
var r0 []*model.ChannelMemberHistoryResult
var r1 error
- if rf, ok := ret.Get(0).(func(int64, int64, string) ([]*model.ChannelMemberHistoryResult, error)); ok {
+ if rf, ok := ret.Get(0).(func(int64, int64, []string) ([]*model.ChannelMemberHistoryResult, error)); ok {
return rf(startTime, endTime, channelID)
}
- if rf, ok := ret.Get(0).(func(int64, int64, string) []*model.ChannelMemberHistoryResult); ok {
+ if rf, ok := ret.Get(0).(func(int64, int64, []string) []*model.ChannelMemberHistoryResult); ok {
r0 = rf(startTime, endTime, channelID)
} else {
if ret.Get(0) != nil {
@@ -93,7 +123,7 @@ func (_m *ChannelMemberHistoryStore) GetUsersInChannelDuring(startTime int64, en
}
}
- if rf, ok := ret.Get(1).(func(int64, int64, string) error); ok {
+ if rf, ok := ret.Get(1).(func(int64, int64, []string) error); ok {
r1 = rf(startTime, endTime, channelID)
} else {
r1 = ret.Error(1)
diff --git a/server/channels/store/timerlayer/timerlayer.go b/server/channels/store/timerlayer/timerlayer.go
index d15801b4cd..22ee23e386 100644
--- a/server/channels/store/timerlayer/timerlayer.go
+++ b/server/channels/store/timerlayer/timerlayer.go
@@ -2696,7 +2696,23 @@ func (s *TimerLayerChannelMemberHistoryStore) GetChannelsLeftSince(userID string
return result, err
}
-func (s *TimerLayerChannelMemberHistoryStore) GetUsersInChannelDuring(startTime int64, endTime int64, channelID string) ([]*model.ChannelMemberHistoryResult, error) {
+func (s *TimerLayerChannelMemberHistoryStore) GetChannelsWithActivityDuring(startTime int64, endTime int64) ([]string, error) {
+ start := time.Now()
+
+ result, err := s.ChannelMemberHistoryStore.GetChannelsWithActivityDuring(startTime, endTime)
+
+ elapsed := float64(time.Since(start)) / float64(time.Second)
+ if s.Root.Metrics != nil {
+ success := "false"
+ if err == nil {
+ success = "true"
+ }
+ s.Root.Metrics.ObserveStoreMethodDuration("ChannelMemberHistoryStore.GetChannelsWithActivityDuring", success, elapsed)
+ }
+ return result, err
+}
+
+func (s *TimerLayerChannelMemberHistoryStore) GetUsersInChannelDuring(startTime int64, endTime int64, channelID []string) ([]*model.ChannelMemberHistoryResult, error) {
start := time.Now()
result, err := s.ChannelMemberHistoryStore.GetUsersInChannelDuring(startTime, endTime, channelID)
diff --git a/server/cmd/mattermost/commands/export.go b/server/cmd/mattermost/commands/export.go
index 5e2894ada4..ece6eebacf 100644
--- a/server/cmd/mattermost/commands/export.go
+++ b/server/cmd/mattermost/commands/export.go
@@ -5,7 +5,6 @@ package commands
import (
"context"
- "fmt"
"os"
"path/filepath"
"time"
@@ -33,30 +32,6 @@ var ScheduleExportCmd = &cobra.Command{
RunE: scheduleExportCmdF,
}
-var CsvExportCmd = &cobra.Command{
- Use: "csv",
- Short: "Export data from Mattermost in CSV format",
- Long: "Export data from Mattermost in CSV format",
- Example: "export csv --exportFrom=12345",
- RunE: buildExportCmdF("csv"),
-}
-
-var ActianceExportCmd = &cobra.Command{
- Use: "actiance",
- Short: "Export data from Mattermost in Actiance format",
- Long: "Export data from Mattermost in Actiance format",
- Example: "export actiance --exportFrom=12345",
- RunE: buildExportCmdF("actiance"),
-}
-
-var GlobalRelayZipExportCmd = &cobra.Command{
- Use: "global-relay-zip",
- Short: "Export data from Mattermost into a zip file containing emails to send to Global Relay for debug and testing purposes only.",
- Long: "Export data from Mattermost into a zip file containing emails to send to Global Relay for debug and testing purposes only. This does not archive any information in Global Relay.",
- Example: "export global-relay-zip --exportFrom=12345",
- RunE: buildExportCmdF("globalrelay-zip"),
-}
-
var BulkExportCmd = &cobra.Command{
Use: "bulk [file]",
Short: "Export bulk data.",
@@ -71,15 +46,6 @@ func init() {
ScheduleExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
ScheduleExportCmd.Flags().Int("timeoutSeconds", -1, "The maximum number of seconds to wait for the job to complete before timing out.")
- CsvExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
- CsvExportCmd.Flags().Int("limit", -1, "The number of posts to export. The default of -1 means no limit.")
-
- ActianceExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
- ActianceExportCmd.Flags().Int("limit", -1, "The number of posts to export. The default of -1 means no limit.")
-
- GlobalRelayZipExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
- GlobalRelayZipExportCmd.Flags().Int("limit", -1, "The number of posts to export. The default of -1 means no limit.")
-
BulkExportCmd.Flags().Bool("all-teams", true, "Export all teams from the server.")
BulkExportCmd.Flags().Bool("with-archived-channels", false, "Also exports archived channels.")
BulkExportCmd.Flags().Bool("with-profile-pictures", false, "Also exports profile pictures.")
@@ -87,9 +53,6 @@ func init() {
BulkExportCmd.Flags().Bool("archive", false, "Outputs a single archive file.")
ExportCmd.AddCommand(ScheduleExportCmd)
- ExportCmd.AddCommand(CsvExportCmd)
- ExportCmd.AddCommand(ActianceExportCmd)
- ExportCmd.AddCommand(GlobalRelayZipExportCmd)
ExportCmd.AddCommand(BulkExportCmd)
RootCmd.AddCommand(ExportCmd)
@@ -158,57 +121,6 @@ func scheduleExportCmdF(command *cobra.Command, args []string) error {
return nil
}
-func buildExportCmdF(format string) func(command *cobra.Command, args []string) error {
- return func(command *cobra.Command, args []string) error {
- a, err := InitDBCommandContextCobra(command, app.SkipPostInitialization())
- license := a.Srv().License()
- if err != nil {
- return err
- }
- defer a.Srv().Shutdown()
-
- rctx := request.EmptyContext(a.Log())
-
- startTime, err := command.Flags().GetInt64("exportFrom")
- if err != nil {
- return errors.New("exportFrom flag error")
- }
- if startTime < 0 {
- return errors.New("exportFrom must be a positive integer")
- }
-
- limit, err := command.Flags().GetInt("limit")
- if err != nil {
- return errors.New("limit flag error")
- }
-
- if a.MessageExport() == nil || license == nil || !*license.Features.MessageExport {
- return errors.New("message export feature not available")
- }
-
- warningsCount, appErr := a.MessageExport().RunExport(rctx, format, startTime, limit)
- if appErr != nil {
- return appErr
- }
- if warningsCount == 0 {
- CommandPrettyPrintln("SUCCESS: Your data was exported.")
- } else {
- if format == model.ComplianceExportTypeGlobalrelay || format == model.ComplianceExportTypeGlobalrelayZip {
- CommandPrettyPrintln(fmt.Sprintf("WARNING: %d warnings encountered, see logs for details.", warningsCount))
- } else {
- CommandPrettyPrintln(fmt.Sprintf("WARNING: %d warnings encountered, see warning.txt for details.", warningsCount))
- }
- }
-
- auditRec := a.MakeAuditRecord(rctx, "buildExport", audit.Success)
- auditRec.AddMeta("format", format)
- auditRec.AddMeta("start", startTime)
- a.LogAuditRec(rctx, auditRec, nil)
-
- return nil
- }
-}
-
func bulkExportCmdF(command *cobra.Command, args []string) error {
a, err := InitDBCommandContextCobra(command, app.SkipPostInitialization())
if err != nil {
diff --git a/server/cmd/mattermost/commands/export_test.go b/server/cmd/mattermost/commands/export_test.go
deleted file mode 100644
index 3d49b356b2..0000000000
--- a/server/cmd/mattermost/commands/export_test.go
+++ /dev/null
@@ -1,74 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.txt for license information.
-
-package commands
-
-import (
- "testing"
-
- "github.com/stretchr/testify/require"
-
- "github.com/mattermost/mattermost/server/public/model"
-)
-
-// There are no tests that actually run the Message Export job, because it can take a long time to complete depending
-// on the size of the database that the config is pointing to. As such, these tests just ensure that the CLI command
-// fails fast if invalid flags are supplied
-
-func TestMessageExportNotEnabled(t *testing.T) {
- th := Setup(t)
- defer th.TearDown()
-
- config := th.Config()
- config.MessageExportSettings.EnableExport = model.NewPointer(false)
- th.SetConfig(config)
-
- // should fail fast because the feature isn't enabled
- require.Error(t, th.RunCommand(t, "export", "schedule"))
-}
-
-func TestMessageExportInvalidFormat(t *testing.T) {
- th := Setup(t)
- defer th.TearDown()
-
- config := th.Config()
- config.MessageExportSettings.EnableExport = model.NewPointer(true)
- th.SetConfig(config)
-
- // should fail fast because format isn't supported
- require.Error(t, th.RunCommand(t, "--format", "not_actiance", "export", "schedule"))
-}
-
-func TestMessageExportNegativeExportFrom(t *testing.T) {
- th := Setup(t)
- defer th.TearDown()
-
- config := th.Config()
- config.MessageExportSettings.EnableExport = model.NewPointer(true)
- th.SetConfig(config)
-
- // should fail fast because export from must be a valid timestamp
- require.Error(t, th.RunCommand(t, "--format", "actiance", "--exportFrom", "-1", "export", "schedule"))
-}
-
-func TestMessageExportNegativeTimeoutSeconds(t *testing.T) {
- th := Setup(t)
- defer th.TearDown()
-
- config := th.Config()
- config.MessageExportSettings.EnableExport = model.NewPointer(true)
- th.SetConfig(config)
-
- // should fail fast because timeout seconds must be a positive int
- require.Error(t, th.RunCommand(t, "--format", "actiance", "--exportFrom", "0", "--timeoutSeconds", "-1", "export", "schedule"))
-}
-
-func TestMessageExportFailsWithoutLicense(t *testing.T) {
- th := Setup(t)
- defer th.TearDown()
-
- actual, _ := th.RunCommandWithOutput(t, "export", "csv", "export.csv", "--exportFrom", "0")
-
- // should fail fast because timeout seconds must be a positive int
- require.Contains(t, actual, "message export feature not available")
-}
diff --git a/server/einterfaces/message_export.go b/server/einterfaces/message_export.go
index fbcff6aece..0e43595e21 100644
--- a/server/einterfaces/message_export.go
+++ b/server/einterfaces/message_export.go
@@ -9,6 +9,5 @@ import (
)
type MessageExportInterface interface {
- StartSynchronizeJob(c request.CTX, exportFromTimestamp int64) (*model.Job, *model.AppError)
- RunExport(c request.CTX, format string, since int64, limit int) (int64, *model.AppError)
+ StartSynchronizeJob(rctx request.CTX, exportFromTimestamp int64) (*model.Job, *model.AppError)
}
diff --git a/server/einterfaces/mocks/MessageExportInterface.go b/server/einterfaces/mocks/MessageExportInterface.go
index c5c09c2ba0..73ba1e4515 100644
--- a/server/einterfaces/mocks/MessageExportInterface.go
+++ b/server/einterfaces/mocks/MessageExportInterface.go
@@ -15,39 +15,9 @@ type MessageExportInterface struct {
mock.Mock
}
-// RunExport provides a mock function with given fields: c, format, since, limit
-func (_m *MessageExportInterface) RunExport(c request.CTX, format string, since int64, limit int) (int64, *model.AppError) {
- ret := _m.Called(c, format, since, limit)
-
- if len(ret) == 0 {
- panic("no return value specified for RunExport")
- }
-
- var r0 int64
- var r1 *model.AppError
- if rf, ok := ret.Get(0).(func(request.CTX, string, int64, int) (int64, *model.AppError)); ok {
- return rf(c, format, since, limit)
- }
- if rf, ok := ret.Get(0).(func(request.CTX, string, int64, int) int64); ok {
- r0 = rf(c, format, since, limit)
- } else {
- r0 = ret.Get(0).(int64)
- }
-
- if rf, ok := ret.Get(1).(func(request.CTX, string, int64, int) *model.AppError); ok {
- r1 = rf(c, format, since, limit)
- } else {
- if ret.Get(1) != nil {
- r1 = ret.Get(1).(*model.AppError)
- }
- }
-
- return r0, r1
-}
-
-// StartSynchronizeJob provides a mock function with given fields: c, exportFromTimestamp
-func (_m *MessageExportInterface) StartSynchronizeJob(c request.CTX, exportFromTimestamp int64) (*model.Job, *model.AppError) {
- ret := _m.Called(c, exportFromTimestamp)
+// StartSynchronizeJob provides a mock function with given fields: rctx, exportFromTimestamp
+func (_m *MessageExportInterface) StartSynchronizeJob(rctx request.CTX, exportFromTimestamp int64) (*model.Job, *model.AppError) {
+ ret := _m.Called(rctx, exportFromTimestamp)
if len(ret) == 0 {
panic("no return value specified for StartSynchronizeJob")
@@ -56,10 +26,10 @@ func (_m *MessageExportInterface) StartSynchronizeJob(c request.CTX, exportFromT
var r0 *model.Job
var r1 *model.AppError
if rf, ok := ret.Get(0).(func(request.CTX, int64) (*model.Job, *model.AppError)); ok {
- return rf(c, exportFromTimestamp)
+ return rf(rctx, exportFromTimestamp)
}
if rf, ok := ret.Get(0).(func(request.CTX, int64) *model.Job); ok {
- r0 = rf(c, exportFromTimestamp)
+ r0 = rf(rctx, exportFromTimestamp)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.Job)
@@ -67,7 +37,7 @@ func (_m *MessageExportInterface) StartSynchronizeJob(c request.CTX, exportFromT
}
if rf, ok := ret.Get(1).(func(request.CTX, int64) *model.AppError); ok {
- r1 = rf(c, exportFromTimestamp)
+ r1 = rf(rctx, exportFromTimestamp)
} else {
if ret.Get(1) != nil {
r1 = ret.Get(1).(*model.AppError)
diff --git a/server/enterprise/internal/file/utils.go b/server/enterprise/internal/file/utils.go
new file mode 100644
index 0000000000..48ed3770db
--- /dev/null
+++ b/server/enterprise/internal/file/utils.go
@@ -0,0 +1,23 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package file
+
+import (
+ "os"
+
+ "github.com/mattermost/mattermost/server/public/shared/mlog"
+)
+
+// DeleteTemp removes a file and logs the error
+// Intended to be called in a defer after the creation of a temp file to ensure cleanup
+func DeleteTemp(logger mlog.LoggerIFace, file *os.File) {
+ err := file.Close()
+ if err != nil {
+ logger.Warn("Failed to close temporary file", mlog.String("filename", file.Name()), mlog.Err(err))
+ }
+ err = os.Remove(file.Name())
+ if err != nil {
+ logger.Warn("Failed to delete temporary file", mlog.String("filename", file.Name()), mlog.Err(err))
+ }
+}
diff --git a/server/enterprise/message_export/actiance_export/actiance_export.go b/server/enterprise/message_export/actiance_export/actiance_export.go
index b87532855d..06d7a13617 100644
--- a/server/enterprise/message_export/actiance_export/actiance_export.go
+++ b/server/enterprise/message_export/actiance_export/actiance_export.go
@@ -4,29 +4,30 @@
package actiance_export
import (
+ "archive/zip"
"bytes"
- "encoding/json"
"encoding/xml"
"fmt"
"io"
- "net/http"
- "path"
- "sort"
+ "os"
+ "slices"
"strings"
+ "time"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/common_export"
+ "github.com/mattermost/mattermost/server/v8/enterprise/internal/file"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
- "github.com/mattermost/mattermost/server/v8/channels/store"
+ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
)
const (
- XMLNS = "http://www.w3.org/2001/XMLSchema-instance"
- ActianceExportFilename = "actiance_export.xml"
- ActianceWarningFilename = "warning.txt"
+ XMLNS = "http://www.w3.org/2001/XMLSchema-instance"
+ ActianceExportFilename = "actiance_export.xml"
+ ActianceWarningFilename = "warning.txt"
+ EnsureLastMessageInASort = "zzzzzzzzzzzzzzzzzzzzzzzzzz"
)
// The root-level element of an actiance export
@@ -39,17 +40,15 @@ type RootNode struct {
// The Conversation element indicates an ad hoc IM conversation or a group chat room.
// The messages from a persistent chat room are exported once a day so that a Conversation entry contains the messages posted to a chat room from 12:00:00 AM to 11:59:59 PM
type ChannelExport struct {
- XMLName xml.Name `xml:"Conversation"`
- Perspective string `xml:"Perspective,attr"` // the value of this attribute doesn't seem to matter. Using the channel name makes the export more human readable
- ChannelId string `xml:"-"` // the unique id of the channel
- RoomId string `xml:"RoomID"`
- StartTime int64 `xml:"StartTimeUTC"` // utc timestamp (seconds), start of export period or create time of channel, whichever is greater. Example: 1366611728.
- JoinEvents []JoinExport // start with a list of all users who were present in the channel during the export period
- Elements []any
- UploadStarts []*FileUploadStartExport
- UploadStops []*FileUploadStopExport
- LeaveEvents []LeaveExport // finish with a list of all users who were present in the channel during the export period
- EndTime int64 `xml:"EndTimeUTC"` // utc timestamp (seconds), end of export period or delete time of channel, whichever is lesser. Example: 1366611728.
+ XMLName xml.Name `xml:"Conversation"`
+ Perspective string `xml:"Perspective,attr"` // the value of this attribute doesn't seem to matter. Using the channel name makes the export more human readable
+ ChannelId string `xml:"-"` // the unique id of the channel
+ RoomId string `xml:"RoomID"`
+ StartTime int64 `xml:"StartTimeUTC"` // utc timestamp (seconds), start of export period or create time of channel, whichever is greater. Example: 1366611728.
+ JoinEvents []JoinExport // start with a list of all users who were present in the channel during the export period
+ Elements []Sortable
+ LeaveEvents []LeaveExport // finish with a list of all users who were present in the channel during the export period
+ EndTime int64 `xml:"EndTimeUTC"` // utc timestamp (seconds), end of export period or delete time of channel, whichever is lesser. Example: 1366611728.
}
// The ParticipantEntered element indicates each user who participates in a conversation.
@@ -72,14 +71,55 @@ type LeaveExport struct {
CorporateEmailID string `xml:"CorporateEmailID"`
}
+type Sortable interface {
+ SortVal() (int64, string)
+}
+
+func SortableSort(a, b Sortable) int {
+ aTimestamp, aId := a.SortVal()
+ bTimestamp, bId := b.SortVal()
+ if aTimestamp == bTimestamp {
+ return strings.Compare(aId, bId)
+ }
+ return int(aTimestamp - bTimestamp)
+}
+
// The Message element indicates the message sent by a user
type PostExport struct {
- XMLName xml.Name `xml:"Message"`
- UserEmail string `xml:"LoginName"` // the email of the person that sent the post
- UserType string `xml:"UserType"` // the type of the person that sent the post
- PostTime int64 `xml:"DateTimeUTC"` // utc timestamp (seconds), time at which the user sent the post. Example: 1366611728
- Message string `xml:"Content"` // the text body of the post
- PreviewsPost string `xml:"PreviewsPost"` // the post id of the post that is previewed by the permalink preview feature
+ XMLName xml.Name `xml:"Message"`
+ MessageId string `xml:"MessageId"` // the message id in the db
+ UserEmail string `xml:"LoginName"` // the email of the person that sent the post
+ UserType shared.UserType `xml:"UserType"` // the type of the person that sent the post: "user" or "bot"
+ CreateAt int64 `xml:"DateTimeUTC"` // utc timestamp (unix milliseconds), the post's createAt
+
+ // Allows us to differentiate between:
+ // - "EditedOriginalMsg": the newly created message (new Id), which holds the pre-edited message contents. The
+ // "EditedNewMsgId" field will point to the message (original Id) which has the post-edited message content.
+ // - "EditedNewMsg": the post-edited message content. This is confusing, so be careful: in the db, this EditedNewMsg
+ // is actually the original messageId because we wanted an edited message to have the same messageId as the
+ // pre-edited message. But for the purposes of exporting and to keep the mental model clear for end-users, we are
+ // calling this the EditedNewMsg and EditedNewMsgId, because this will hold the NEW post-edited message contents,
+ // and that's what's important to the end-user viewing the export.
+ // - "UpdatedNoMsgChange": the message content hasn't changed, but the post was updated for some reason (reaction,
+ // replied-to, a reply was edited, a reply was deleted (as of 10.2), perhaps other reasons)
+ // - "Deleted": the message was deleted.
+ // - "FileDeleted": this message is recording that a file was deleted.
+ UpdatedType shared.PostUpdatedType `xml:"UpdatedType,omitempty"`
+ UpdateAt int64 `xml:"UpdatedDateTimeUTC,omitempty"` // if this is an updated post, this is the updated time (same as deleted time for deleted posts).
+
+ // when a message is edited, the EditedOriginalMsg points to the message Id that now has the newly edited message.
+ EditedNewMsgId string `xml:"EditedNewMsgId,omitempty"`
+
+ Message string `xml:"Content"` // the text body of the post
+ PreviewsPost string `xml:"PreviewsPost,omitempty"` // the post id of the post that is previewed by the permalink preview feature
+}
+
+func (p PostExport) SortVal() (int64, string) {
+ // updated messages are sorted by UpdateAt
+ if p.UpdatedType != "" {
+ return p.UpdateAt, p.MessageId
+ }
+ return p.CreateAt, p.MessageId
}
// The FileTransferStarted element indicates the beginning of a file transfer in a conversation
@@ -91,6 +131,11 @@ type FileUploadStartExport struct {
FilePath string `xml:"FileName"` // the path to the file, as stored on the server
}
+func (f FileUploadStartExport) SortVal() (int64, string) {
+ // file messages should come after the message they were with
+ return f.UploadStartTime, EnsureLastMessageInASort
+}
+
// The FileTransferEnded element indicates the end of a file transfer in a conversation
type FileUploadStopExport struct {
XMLName xml.Name `xml:"FileTransferEnded"`
@@ -101,73 +146,105 @@ type FileUploadStopExport struct {
Status string `xml:"Status"` // set to either "Completed" or "Failed" depending on the outcome of the upload operation
}
-func ActianceExport(rctx request.CTX, posts []*model.MessageExport, db store.Store, exportBackend filestore.FileBackend, fileAttachmentBackend filestore.FileBackend, exportDirectory string) (warningCount int64, appErr *model.AppError) {
- // sort the posts into buckets based on the channel in which they appeared
- membersByChannel := common_export.MembersByChannel{}
- metadata := common_export.Metadata{
- Channels: map[string]common_export.MetadataChannel{},
- MessagesCount: 0,
- AttachmentsCount: 0,
- StartTime: 0,
- EndTime: 0,
+func (f FileUploadStopExport) SortVal() (int64, string) {
+ // file messages should come after the message they were with
+ return f.UploadStopTime, EnsureLastMessageInASort
+}
+
+func ActianceExport(rctx request.CTX, p shared.ExportParams) (shared.RunExportResults, error) {
+ start := time.Now()
+
+ // Build the channel exports for the channels that had post or user join/leave activity this batch.
+ exportData, err := shared.GetGenericExportData(p)
+ if err != nil {
+ return exportData.Results, err
}
- elementsByChannel := map[string][]any{}
- allUploadedFiles := []*model.FileInfo{}
+ var allUploadedFiles []*model.FileInfo
- for _, post := range posts {
- if post == nil {
- rctx.Logger().Warn("ignored a nil post reference in the list")
- continue
+ // Convert the generic shared.ChannelExports to the Actiance-specific ChannelExports data.
+ channelExports := make([]ChannelExport, 0, len(exportData.Exports))
+ for _, channel := range exportData.Exports {
+ joinEvents := make([]JoinExport, 0, len(channel.JoinEvents))
+ leaveEvents := make([]LeaveExport, 0, len(channel.LeaveEvents))
+
+ for _, j := range channel.JoinEvents {
+ joinEvents = append(joinEvents, JoinExport{
+ UserEmail: j.UserEmail,
+ UserType: string(j.UserType),
+ JoinTime: j.JoinTime,
+ CorporateEmailID: j.UserEmail,
+ })
}
- elementsByChannel[*post.ChannelId] = append(elementsByChannel[*post.ChannelId], postToExportEntry(post, post.PostCreateAt, *post.PostMessage))
-
- if post.PostDeleteAt != nil && *post.PostDeleteAt > 0 && post.PostProps != nil {
- props := map[string]any{}
- if json.Unmarshal([]byte(*post.PostProps), &props) == nil {
- if _, ok := props[model.PostPropsDeleteBy]; ok {
- elementsByChannel[*post.ChannelId] = append(elementsByChannel[*post.ChannelId], postToExportEntry(post,
- post.PostDeleteAt, "delete "+*post.PostMessage))
- }
- }
+ for _, l := range channel.LeaveEvents {
+ leaveEvents = append(leaveEvents, LeaveExport{
+ UserEmail: l.UserEmail,
+ UserType: string(l.UserType),
+ LeaveTime: l.LeaveTime,
+ CorporateEmailID: l.UserEmail,
+ })
}
- startUploads, stopUploads, uploadedFiles, deleteFileMessages, err := postToAttachmentsEntries(post, db)
- if err != nil {
- return warningCount, err
+ elements := make([]Sortable, 0, len(channel.Posts)+len(channel.DeletedFiles)+len(channel.UploadStarts)+len(channel.UploadStops))
+ for _, p := range channel.Posts {
+ elements = append(elements, PostExport{
+ MessageId: *p.PostId,
+ UserEmail: *p.UserEmail,
+ UserType: p.UserType,
+ CreateAt: *p.PostCreateAt,
+ UpdatedType: p.UpdatedType,
+ UpdateAt: p.UpdateAt,
+ EditedNewMsgId: p.EditedNewMsgId,
+ Message: p.Message,
+ PreviewsPost: p.PreviewsPost,
+ })
}
- elementsByChannel[*post.ChannelId] = append(elementsByChannel[*post.ChannelId], startUploads...)
- elementsByChannel[*post.ChannelId] = append(elementsByChannel[*post.ChannelId], stopUploads...)
- elementsByChannel[*post.ChannelId] = append(elementsByChannel[*post.ChannelId], deleteFileMessages...)
-
- allUploadedFiles = append(allUploadedFiles, uploadedFiles...)
-
- metadata.Update(post, len(uploadedFiles))
-
- if _, ok := membersByChannel[*post.ChannelId]; !ok {
- membersByChannel[*post.ChannelId] = common_export.ChannelMembers{}
+ for _, p := range channel.DeletedFiles {
+ elements = append(elements, PostExport{
+ MessageId: *p.PostId,
+ UserEmail: *p.UserEmail,
+ UserType: p.UserType,
+ CreateAt: *p.PostCreateAt,
+ UpdatedType: p.UpdatedType,
+ UpdateAt: p.UpdateAt,
+ EditedNewMsgId: p.EditedNewMsgId,
+ Message: p.Message,
+ PreviewsPost: p.PreviewsPost,
+ })
}
- membersByChannel[*post.ChannelId][*post.UserId] = common_export.ChannelMember{
- Email: *post.UserEmail,
- UserId: *post.UserId,
- IsBot: post.IsBot,
- Username: *post.Username,
+ for _, u := range channel.UploadStarts {
+ elements = append(elements, FileUploadStartExport{
+ UserEmail: u.UserEmail,
+ UploadStartTime: u.UploadStartTime,
+ Filename: u.FileInfo.Name,
+ FilePath: u.FileInfo.Path,
+ })
}
- }
-
- rctx.Logger().Info("Exported data for channels", mlog.Int("number_of_channels", len(metadata.Channels)))
-
- channelExports := []ChannelExport{}
- for _, channel := range metadata.Channels {
- channelExport, err := buildChannelExport(
- channel,
- membersByChannel[channel.ChannelId],
- elementsByChannel[channel.ChannelId],
- db,
- )
- if err != nil {
- return warningCount, err
+ for _, u := range channel.UploadStops {
+ elements = append(elements, FileUploadStopExport{
+ UserEmail: u.UserEmail,
+ UploadStopTime: u.UploadStopTime,
+ Filename: u.FileInfo.Name,
+ FilePath: u.FileInfo.Path,
+ Status: u.Status,
+ })
}
- channelExports = append(channelExports, *channelExport)
+
+ // We need to sort all the elements by (updateAt, messageId) because they were added by type above.
+ slices.SortStableFunc(elements, SortableSort)
+
+ channelExports = append(channelExports, ChannelExport{
+ Perspective: channel.DisplayName,
+ ChannelId: channel.ChannelId,
+ RoomId: fmt.Sprintf("%v - %v - %v", shared.ChannelTypeDisplayName(channel.ChannelType),
+ channel.ChannelName, channel.ChannelId),
+ StartTime: channel.StartTime,
+ JoinEvents: joinEvents,
+ Elements: elements,
+ LeaveEvents: leaveEvents,
+ EndTime: channel.EndTime,
+ })
+
+ allUploadedFiles = append(allUploadedFiles, channel.Files...)
}
export := &RootNode{
@@ -175,182 +252,113 @@ func ActianceExport(rctx request.CTX, posts []*model.MessageExport, db store.Sto
Channels: channelExports,
}
- return writeExport(rctx, export, allUploadedFiles, exportDirectory, exportBackend, fileAttachmentBackend)
+ results := exportData.Results
+ results.ProcessingPostsMs = time.Since(start).Milliseconds()
+
+ results.WriteExportResult, err = writeExport(rctx, export, allUploadedFiles, p.ExportBackend, p.FileAttachmentBackend, p.BatchPath)
+ results.NumChannels = len(channelExports)
+ return results, err
}
-func postToExportEntry(post *model.MessageExport, createTime *int64, message string) *PostExport {
- userType := "user"
- if post.IsBot {
- userType = "bot"
- }
- return &PostExport{
- PostTime: *createTime,
- Message: message,
- UserType: userType,
- UserEmail: *post.UserEmail,
- PreviewsPost: post.PreviewID(),
- }
-}
-
-func postToAttachmentsEntries(post *model.MessageExport, db store.Store) ([]any, []any, []*model.FileInfo, []any, *model.AppError) {
- // if the post included any files, we need to add special elements to the export.
- if len(post.PostFileIds) == 0 {
- return nil, nil, nil, nil, nil
- }
-
- fileInfos, err := db.FileInfo().GetForPost(*post.PostId, true, true, false)
- if err != nil {
- return nil, nil, nil, nil, model.NewAppError("postToAttachmentsEntries", "ent.message_export.actiance_export.get_attachment_error", nil, "", http.StatusInternalServerError).Wrap(err)
- }
-
- startUploads := []any{}
- stopUploads := []any{}
- deleteFileMessages := []any{}
-
- uploadedFiles := []*model.FileInfo{}
- for _, fileInfo := range fileInfos {
- // insert a record of the file upload into the export file
- // path to exported file is relative to the xml file, so it's just the name of the exported file
- startUploads = append(startUploads, &FileUploadStartExport{
- UserEmail: *post.UserEmail,
- Filename: fileInfo.Name,
- FilePath: fileInfo.Path,
- UploadStartTime: *post.PostCreateAt,
- })
-
- stopUploads = append(stopUploads, &FileUploadStopExport{
- UserEmail: *post.UserEmail,
- Filename: fileInfo.Name,
- FilePath: fileInfo.Path,
- UploadStopTime: *post.PostCreateAt,
- Status: "Completed",
- })
-
- if fileInfo.DeleteAt > 0 && post.PostDeleteAt != nil {
- deleteFileMessages = append(deleteFileMessages, postToExportEntry(post, post.PostDeleteAt, "delete "+fileInfo.Path))
- }
-
- uploadedFiles = append(uploadedFiles, fileInfo)
- }
- return startUploads, stopUploads, uploadedFiles, deleteFileMessages, nil
-}
-
-func buildChannelExport(channel common_export.MetadataChannel, members common_export.ChannelMembers, elements []any, db store.Store) (*ChannelExport, *model.AppError) {
- channelExport := ChannelExport{
- ChannelId: channel.ChannelId,
- RoomId: fmt.Sprintf("%v - %v - %v", common_export.ChannelTypeDisplayName(channel.ChannelType), channel.ChannelName, channel.ChannelId),
- StartTime: channel.StartTime,
- EndTime: channel.EndTime,
- Perspective: channel.ChannelDisplayName,
- }
-
- channelMembersHistory, err := db.ChannelMemberHistory().GetUsersInChannelDuring(channel.StartTime, channel.EndTime, channel.ChannelId)
- if err != nil {
- return nil, model.NewAppError("buildChannelExport", "ent.get_users_in_channel_during", nil, "", http.StatusInternalServerError).Wrap(err)
- }
-
- joins, leaves := common_export.GetJoinsAndLeavesForChannel(channel.StartTime, channel.EndTime, channelMembersHistory, members)
- type StillJoinedInfo struct {
- Time int64
- Type string
- }
- stillJoined := map[string]StillJoinedInfo{}
- for _, join := range joins {
- userType := "user"
- if join.IsBot {
- userType = "bot"
- }
- channelExport.JoinEvents = append(channelExport.JoinEvents, JoinExport{
- JoinTime: join.Datetime,
- UserEmail: join.Email,
- UserType: userType,
- CorporateEmailID: join.Email,
- })
- if value, ok := stillJoined[join.Email]; !ok {
- stillJoined[join.Email] = StillJoinedInfo{Time: join.Datetime, Type: userType}
- } else {
- if join.Datetime > value.Time {
- stillJoined[join.Email] = StillJoinedInfo{Time: join.Datetime, Type: userType}
- }
- }
- }
- for _, leave := range leaves {
- userType := "user"
- if leave.IsBot {
- userType = "bot"
- }
- channelExport.LeaveEvents = append(channelExport.LeaveEvents, LeaveExport{
- LeaveTime: leave.Datetime,
- UserEmail: leave.Email,
- UserType: userType,
- CorporateEmailID: leave.Email,
- })
- if leave.Datetime > stillJoined[leave.Email].Time {
- delete(stillJoined, leave.Email)
- }
- }
-
- for email := range stillJoined {
- channelExport.LeaveEvents = append(channelExport.LeaveEvents, LeaveExport{
- LeaveTime: channel.EndTime,
- UserEmail: email,
- UserType: stillJoined[email].Type,
- CorporateEmailID: email,
- })
- }
-
- sort.Slice(channelExport.LeaveEvents, func(i, j int) bool {
- if channelExport.LeaveEvents[i].LeaveTime == channelExport.LeaveEvents[j].LeaveTime {
- return channelExport.LeaveEvents[i].UserEmail < channelExport.LeaveEvents[j].UserEmail
- }
- return channelExport.LeaveEvents[i].LeaveTime < channelExport.LeaveEvents[j].LeaveTime
- })
-
- channelExport.Elements = elements
- return &channelExport, nil
-}
-
-func writeExport(rctx request.CTX, export *RootNode, uploadedFiles []*model.FileInfo, exportDirectory string, exportBackend filestore.FileBackend, fileAttachmentBackend filestore.FileBackend) (warningCount int64, appErr *model.AppError) {
+func writeExport(rctx request.CTX, export *RootNode, uploadedFiles []*model.FileInfo, exportBackend filestore.FileBackend, fileAttachmentBackend filestore.FileBackend, batchPath string) (res shared.WriteExportResult, err error) {
+ start := time.Now()
// marshal the export object to xml
xmlData := &bytes.Buffer{}
xmlData.WriteString(xml.Header)
enc := xml.NewEncoder(xmlData)
enc.Indent("", " ")
- if err := enc.Encode(export); err != nil {
- return warningCount, model.NewAppError("ActianceExport.AtianceExport", "ent.actiance.export.marshalToXml.appError", nil, "", 0).Wrap(err)
+ if err = enc.Encode(export); err != nil {
+ return res, fmt.Errorf("unable to convert export to XML: %w", err)
+ }
+ if err = enc.Flush(); err != nil {
+ return res, fmt.Errorf("unable to flush the XML encoder: %w", err)
}
- enc.Flush()
- // Try to disable the write timeout if the backend supports it
- if _, err := filestore.TryWriteFileContext(rctx.Context(), exportBackend, xmlData, path.Join(exportDirectory, ActianceExportFilename)); err != nil {
- return warningCount, model.NewAppError("ActianceExport.AtianceExport", "ent.actiance.export.write_file.appError", nil, "", 0).Wrap(err)
+ // Write this batch to a tmp zip, then copy the zip to the export directory.
+ // Using a 2M buffer because the file backend may be s3 and this optimizes speed and
+ // memory usage, see: https://github.com/mattermost/mattermost/pull/26629
+ buf := make([]byte, 1024*1024*2)
+ temp, err := os.CreateTemp("", "compliance-export-batch-*.zip")
+ if err != nil {
+ return res, fmt.Errorf("unable to create the batch temporary file: %w", err)
}
+ defer file.DeleteTemp(rctx.Logger(), temp)
+
+ zipFile := zip.NewWriter(temp)
+ w, err := zipFile.Create(ActianceExportFilename)
+ if err != nil {
+ return res, fmt.Errorf("unable to create the xml file in the zipFile created with the batch temporary file: %w", err)
+ }
+ if _, err = io.CopyBuffer(w, xmlData, buf); err != nil {
+ return res, fmt.Errorf("unable to write into the zipFile created with the batch temporary file: %w", err)
+ }
+ res.ProcessingXmlMs = time.Since(start).Milliseconds()
+
+ start = time.Now()
var missingFiles []string
for _, fileInfo := range uploadedFiles {
- var attachmentSrc io.ReadCloser
- attachmentSrc, nErr := fileAttachmentBackend.Reader(fileInfo.Path)
- if nErr != nil {
- missingFiles = append(missingFiles, "Warning:"+common_export.MissingFileMessage+" - "+fileInfo.Path)
- rctx.Logger().Warn(common_export.MissingFileMessage, mlog.String("FileName", fileInfo.Path))
+ var attachmentReader io.ReadCloser
+ attachmentReader, err = fileAttachmentBackend.Reader(fileInfo.Path)
+ if err != nil {
+ missingFiles = append(missingFiles, "Warning:"+shared.MissingFileMessage+" - "+fileInfo.Path)
+ rctx.Logger().Warn(shared.MissingFileMessage, mlog.String("filename", fileInfo.Path))
continue
}
- defer attachmentSrc.Close()
- destPath := path.Join(exportDirectory, fileInfo.Path)
+ // There could be many uploadedFiles, so be careful about closing readers.
+ if err = func() error {
+ defer attachmentReader.Close()
+ var zipWriter io.Writer
+ zipWriter, err = zipFile.Create(fileInfo.Path)
+ if err != nil {
+ return err
+ }
- _, nErr = exportBackend.WriteFile(attachmentSrc, destPath)
- if nErr != nil {
- return warningCount, model.NewAppError("ActianceExport.AtianceExport", "ent.actiance.export.write_file.appError", nil, "", 0).Wrap(nErr)
+ // CopyBuffer works with dirty buffers, no need to clear it.
+ if _, err = io.CopyBuffer(zipWriter, attachmentReader, buf); err != nil {
+ return err
+ }
+
+ return nil
+ }(); err != nil {
+ return res, fmt.Errorf("unable to write into the zipFile created with the batch temporary file: %w", err)
}
}
- warningCount = int64(len(missingFiles))
- if warningCount > 0 {
- _, err := filestore.TryWriteFileContext(rctx.Context(), exportBackend, strings.NewReader(strings.Join(missingFiles, "\n")), path.Join(exportDirectory, ActianceWarningFilename))
+
+ res.TransferringFilesMs = time.Since(start).Milliseconds()
+ res.NumWarnings = len(missingFiles)
+ if res.NumWarnings > 0 {
+ var w io.Writer
+ w, err = zipFile.Create(ActianceWarningFilename)
if err != nil {
- appErr = model.NewAppError("ActianceExport.AtianceExport", "ent.actiance.export.write_file.appError", nil, "", 0).Wrap(err)
+ return res, fmt.Errorf("unable to create the warning file in the zipFile created with the batch temporary file: %w", err)
+ }
+ r := strings.NewReader(strings.Join(missingFiles, "\n"))
+ if _, err = io.CopyBuffer(w, r, buf); err != nil {
+ return res, fmt.Errorf("unable to write into the zipFile created with the batch temporary file: %w", err)
}
}
- return warningCount, appErr
+
+ if err = zipFile.Close(); err != nil {
+ return res, fmt.Errorf("unable to close the zipFile created with the batch temporary file: %w", err)
+ }
+
+ _, err = temp.Seek(0, io.SeekStart)
+ if err != nil {
+ return res, fmt.Errorf("unable to seek to the beginning of the the batch temporary file: %w", err)
+ }
+
+ start = time.Now()
+
+ // Try to write the file without a timeout due to the potential size of the file.
+ _, err = filestore.TryWriteFileContext(rctx.Context(), exportBackend, temp, batchPath)
+ if err != nil {
+ return res, fmt.Errorf("unable to transfer the batch zip to the file backend: %w", err)
+ }
+
+ res.TransferringZipMs = time.Since(start).Milliseconds()
+
+ return res, nil
}
diff --git a/server/enterprise/message_export/actiance_export/actiance_export_test.go b/server/enterprise/message_export/actiance_export/actiance_export_test.go
index 2664fbdba4..392508ed00 100644
--- a/server/enterprise/message_export/actiance_export/actiance_export_test.go
+++ b/server/enterprise/message_export/actiance_export/actiance_export_test.go
@@ -4,12 +4,19 @@
package actiance_export
import (
+ "archive/zip"
"bytes"
"encoding/xml"
+ "fmt"
+ "io"
"os"
+ "path"
"strings"
"testing"
+ "github.com/mattermost/mattermost/server/public/shared/i18n"
+ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
+
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
@@ -20,6 +27,14 @@ import (
"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
)
+type MyReporter struct {
+ mock.Mock
+}
+
+func (mr *MyReporter) ReportProgressMessage(message string) {
+ mr.Called(message)
+}
+
func TestActianceExport(t *testing.T) {
t.Run("no dedicated export filestore", func(t *testing.T) {
exportTempDir, err := os.MkdirTemp("", "")
@@ -70,10 +85,14 @@ func TestActianceExport(t *testing.T) {
func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, attachmentBackend filestore.FileBackend) {
rctx := request.TestContext(t)
+ rctx = rctx.WithT(i18n.IdentityTfunc()).(*request.Context)
chanTypeDirect := model.ChannelTypeDirect
- csvExportTests := []struct {
+ actianceExportTests := []struct {
name string
+ jobEndTime int64
+ activity []string
+ channels model.ChannelList
cmhs map[string][]*model.ChannelMemberHistoryResult
posts []*model.MessageExport
attachments map[string][]*model.FileInfo
@@ -82,6 +101,7 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
}{
{
name: "empty",
+ jobEndTime: 30,
cmhs: map[string][]*model.ChannelMemberHistoryResult{},
posts: []*model.MessageExport{},
attachments: map[string][]*model.FileInfo{},
@@ -89,22 +109,31 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
xml.Header,
"",
}, ""),
+ activity: []string{},
expectedFiles: 2,
},
{
- name: "posts",
+ name: "posts",
+ jobEndTime: 500,
cmhs: map[string][]*model.ChannelMemberHistoryResult{
"channel-id": {
- {JoinTime: 0, UserId: "test", UserEmail: "test", Username: "test", LeaveTime: model.NewPointer(int64(400))},
- {JoinTime: 8, UserId: "test2", UserEmail: "test2", Username: "test2", LeaveTime: model.NewPointer(int64(80))},
- {JoinTime: 400, UserId: "test3", UserEmail: "test3", Username: "test3"},
- {JoinTime: 10, UserId: "test_bot", UserEmail: "test_bot", Username: "test_bot", IsBot: true, LeaveTime: model.NewPointer(int64(20))},
+ {JoinTime: 0, ChannelId: "channel-id", UserId: "test", UserEmail: "test@email", Username: "testname", LeaveTime: model.NewPointer(int64(400))},
+ {JoinTime: 8, ChannelId: "channel-id", UserId: "test2", UserEmail: "test2@email", Username: "test2name", LeaveTime: model.NewPointer(int64(80))},
+ {JoinTime: 400, ChannelId: "channel-id", UserId: "test3", UserEmail: "test3@email", Username: "test3name"},
+ {JoinTime: 10, ChannelId: "channel-id", UserId: "test_bot", UserEmail: "test_bot@email", Username: "test_botname", IsBot: true, LeaveTime: model.NewPointer(int64(20))},
},
},
+ activity: []string{"channel-id"},
+ channels: model.ChannelList{{
+ TeamId: "team-id",
+ Id: "channel-id",
+ Name: "channel-name",
+ DisplayName: "channel-display-name",
+ Type: model.ChannelTypeDirect,
+ }},
posts: []*model.MessageExport{
{
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostId: model.NewPointer("post-original-id"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
TeamDisplayName: model.NewPointer("team-display-name"),
@@ -112,8 +141,9 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
+ PostUpdateAt: model.NewPointer(int64(2)),
+ PostEditAt: model.NewPointer(int64(2)),
+ PostMessage: model.NewPointer("edited message"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
Username: model.NewPointer("username"),
@@ -132,16 +162,16 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
PostCreateAt: model.NewPointer(int64(1)),
PostUpdateAt: model.NewPointer(int64(2)),
PostDeleteAt: model.NewPointer(int64(2)),
- PostMessage: model.NewPointer("edit message"),
+ PostMessage: model.NewPointer("original message"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
Username: model.NewPointer("username"),
ChannelType: &chanTypeDirect,
PostFileIds: []string{},
},
+ // deleted post
{
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostId: model.NewPointer("post-id2"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
TeamDisplayName: model.NewPointer("team-display-name"),
@@ -151,7 +181,7 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
PostCreateAt: model.NewPointer(int64(1)),
PostUpdateAt: model.NewPointer(int64(4)),
PostDeleteAt: model.NewPointer(int64(4)),
- PostMessage: model.NewPointer("message"),
+ PostMessage: model.NewPointer("message2"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
Username: model.NewPointer("username"),
@@ -160,8 +190,7 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
PostProps: model.NewPointer("{\"deleteBy\":\"fy8j97gwii84bk4zxprbpc9d9w\"}"),
},
{
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostId: model.NewPointer("post-id3"),
PostRootId: model.NewPointer("post-root-id"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
@@ -171,7 +200,7 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
ChannelDisplayName: model.NewPointer("channel-display-name"),
PostCreateAt: model.NewPointer(int64(100)),
PostUpdateAt: model.NewPointer(int64(100)),
- PostMessage: model.NewPointer("message"),
+ PostMessage: model.NewPointer("message3"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
Username: model.NewPointer("username"),
@@ -187,101 +216,129 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
" direct - channel-name - channel-id\n",
" 1\n",
" \n",
- " test\n",
+ " test@email\n",
" user\n",
" 0\n",
- " test\n",
+ " test@email\n",
" \n",
" \n",
- " test2\n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " test@test.com\n",
+ " \n",
+ " \n",
+ " test2@email\n",
" user\n",
" 8\n",
- " test2\n",
+ " test2@email\n",
" \n",
" \n",
- " test_bot\n",
+ " test_bot@email\n",
" bot\n",
" 10\n",
- " test_bot\n",
+ " test_bot@email\n",
" \n",
" \n",
- " test@test.com\n",
+ " test3@email\n",
" user\n",
- " 1\n",
- " test@test.com\n",
+ " 400\n",
+ " test3@email\n",
" \n",
" \n",
+ " post-id2\n",
" test@test.com\n",
" user\n",
" 1\n",
- " message\n",
- " \n",
+ " message2\n",
" \n",
" \n",
+ " post-id\n",
" test@test.com\n",
" user\n",
" 1\n",
- " edit message\n",
- " \n",
+ " EditedOriginalMsg\n",
+ " 2\n",
+ " post-original-id\n",
+ " original message\n",
" \n",
" \n",
+ " post-original-id\n",
" test@test.com\n",
" user\n",
" 1\n",
- " message\n",
- " \n",
+ " EditedNewMsg\n",
+ " 2\n",
+ " edited message\n",
" \n",
" \n",
+ " post-id2\n",
" test@test.com\n",
" user\n",
- " 4\n",
- " delete message\n",
- " \n",
+ " 1\n",
+ " Deleted\n",
+ " 4\n",
+ " delete message2\n",
" \n",
" \n",
+ " post-id3\n",
" test@test.com\n",
" user\n",
" 100\n",
- " message\n",
- " \n",
+ " message3\n",
" \n",
" \n",
- " test_bot\n",
+ " test_bot@email\n",
" bot\n",
" 20\n",
- " test_bot\n",
+ " test_bot@email\n",
" \n",
" \n",
- " test2\n",
+ " test2@email\n",
" user\n",
" 80\n",
- " test2\n",
+ " test2@email\n",
" \n",
" \n",
- " test\n",
+ " test@email\n",
" user\n",
- " 100\n",
- " test\n",
+ " 400\n",
+ " test@email\n",
+ " \n",
+ " \n",
+ " test3@email\n",
+ " user\n",
+ " 500\n",
+ " test3@email\n",
" \n",
" \n",
" test@test.com\n",
" user\n",
- " 100\n",
+ " 500\n",
" test@test.com\n",
" \n",
- " 100\n",
+ " 500\n",
" \n",
"",
}, ""),
expectedFiles: 2,
},
{
- name: "post with permalink preview",
+ name: "post with permalink preview",
+ jobEndTime: 600,
cmhs: map[string][]*model.ChannelMemberHistoryResult{
"channel-id": {
- {JoinTime: 0, UserId: "test", UserEmail: "test", Username: "test", LeaveTime: model.NewPointer(int64(400))},
+ {JoinTime: 0, ChannelId: "channel-id", UserId: "test", UserEmail: "test@email", Username: "test", LeaveTime: model.NewPointer(int64(400))},
},
},
+ activity: []string{"channel-id"},
+ channels: model.ChannelList{{
+ TeamId: "team-id",
+ Id: "channel-id",
+ Name: "channel-name",
+ DisplayName: "channel-display-name",
+ Type: model.ChannelTypeDirect,
+ }},
posts: []*model.MessageExport{
{
PostId: model.NewPointer("post-id"),
@@ -330,10 +387,10 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
" direct - channel-name - channel-id\n",
" 1\n",
" \n",
- " test\n",
+ " test@email\n",
" user\n",
" 0\n",
- " test\n",
+ " test@email\n",
" \n",
" \n",
" test@test.com\n",
@@ -342,6 +399,7 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
" test@test.com\n",
" \n",
" \n",
+ " post-id\n",
" test@test.com\n",
" user\n",
" 1\n",
@@ -349,45 +407,54 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
" n4w39mc1ff8y5fite4b8hacy1w\n",
" \n",
" \n",
+ " post-id\n",
" test@test.com\n",
" user\n",
" 100\n",
" message\n",
- " \n",
" \n",
" \n",
- " test\n",
+ " test@email\n",
" user\n",
- " 100\n",
- " test\n",
+ " 400\n",
+ " test@email\n",
" \n",
" \n",
" test@test.com\n",
" user\n",
- " 100\n",
+ " 600\n",
" test@test.com\n",
" \n",
- " 100\n",
+ " 600\n",
" \n",
"",
}, ""),
expectedFiles: 0,
},
{
- name: "posts with attachments",
+ name: "posts with attachments",
+ jobEndTime: 700,
cmhs: map[string][]*model.ChannelMemberHistoryResult{
"channel-id": {
{
- JoinTime: 0, UserId: "test", UserEmail: "test", Username: "test", LeaveTime: model.NewPointer(int64(400)),
+ JoinTime: 0, ChannelId: "channel-id", UserId: "test", UserEmail: "test@email", Username: "test", LeaveTime: model.NewPointer(int64(400)),
},
{
- JoinTime: 8, UserId: "test2", UserEmail: "test2", Username: "test2", LeaveTime: model.NewPointer(int64(80)),
+ JoinTime: 8, ChannelId: "channel-id", UserId: "test2", UserEmail: "test2@email", Username: "test2", LeaveTime: model.NewPointer(int64(80)),
},
{
- JoinTime: 400, UserId: "test3", UserEmail: "test3", Username: "test3",
+ JoinTime: 400, ChannelId: "channel-id", UserId: "test3", UserEmail: "test3@email", Username: "test3",
},
},
},
+ activity: []string{"channel-id"},
+ channels: model.ChannelList{{
+ TeamId: "team-id",
+ Id: "channel-id",
+ Name: "channel-name",
+ DisplayName: "channel-display-name",
+ Type: model.ChannelTypeDirect,
+ }},
posts: []*model.MessageExport{
{
PostId: model.NewPointer("post-id-1"),
@@ -399,6 +466,7 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(1)),
PostMessage: model.NewPointer("message"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
@@ -417,6 +485,7 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
PostCreateAt: model.NewPointer(int64(100)),
+ PostUpdateAt: model.NewPointer(int64(100)),
PostMessage: model.NewPointer("message"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
@@ -441,16 +510,10 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
" direct - channel-name - channel-id\n",
" 1\n",
" \n",
- " test\n",
+ " test@email\n",
" user\n",
" 0\n",
- " test\n",
- " \n",
- " \n",
- " test2\n",
- " user\n",
- " 8\n",
- " test2\n",
+ " test@email\n",
" \n",
" \n",
" test@test.com\n",
@@ -458,12 +521,24 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
" 1\n",
" test@test.com\n",
" \n",
+ " \n",
+ " test2@email\n",
+ " user\n",
+ " 8\n",
+ " test2@email\n",
+ " \n",
+ " \n",
+ " test3@email\n",
+ " user\n",
+ " 400\n",
+ " test3@email\n",
+ " \n",
" \n",
+ " post-id-1\n",
" test@test.com\n",
" user\n",
" 1\n",
" message\n",
- " \n",
" \n",
" \n",
" test@test.com\n",
@@ -479,46 +554,569 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
" Completed\n",
" \n",
" \n",
+ " post-id-2\n",
" test@test.com\n",
" user\n",
" 100\n",
" message\n",
- " \n",
" \n",
" \n",
- " test2\n",
+ " test2@email\n",
" user\n",
" 80\n",
- " test2\n",
+ " test2@email\n",
" \n",
" \n",
- " test\n",
+ " test@email\n",
" user\n",
- " 100\n",
- " test\n",
+ " 400\n",
+ " test@email\n",
+ " \n",
+ " \n",
+ " test3@email\n",
+ " user\n",
+ " 700\n",
+ " test3@email\n",
" \n",
" \n",
" test@test.com\n",
" user\n",
- " 100\n",
+ " 700\n",
" test@test.com\n",
" \n",
- " 100\n",
+ " 700\n",
" \n",
"",
}, ""),
expectedFiles: 3,
},
+
+ {
+ name: "posts with deleted attachments, no deleted post, and at different time from original post",
+ jobEndTime: 700,
+ cmhs: map[string][]*model.ChannelMemberHistoryResult{
+ "channel-id": {
+ {
+ JoinTime: 0, ChannelId: "channel-id", UserId: "test", UserEmail: "test@email", Username: "test", LeaveTime: model.NewPointer(int64(400)),
+ },
+ {
+ JoinTime: 8, ChannelId: "channel-id", UserId: "test2", UserEmail: "test2@email", Username: "test2", LeaveTime: model.NewPointer(int64(80)),
+ },
+ {
+ JoinTime: 400, ChannelId: "channel-id", UserId: "test3", UserEmail: "test3@email", Username: "test3",
+ },
+ },
+ },
+ activity: []string{"channel-id"},
+ channels: model.ChannelList{{
+ TeamId: "team-id",
+ Id: "channel-id",
+ Name: "channel-name",
+ DisplayName: "channel-display-name",
+ Type: model.ChannelTypeDirect,
+ }},
+ posts: []*model.MessageExport{
+ {
+ PostId: model.NewPointer("post-id-1"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(1)),
+ PostMessage: model.NewPointer("message1"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("user-id"),
+ Username: model.NewPointer("username"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{"test1"},
+ },
+ },
+ attachments: map[string][]*model.FileInfo{
+ "post-id-1": {
+ {
+ Name: "test1-attachment",
+ Id: "test1-attachment",
+ Path: "test1-attachment",
+ CreateAt: 1,
+ UpdateAt: 2,
+ DeleteAt: 2,
+ },
+ },
+ },
+ expectedData: strings.Join([]string{
+ xml.Header,
+ "\n",
+ " \n",
+ " direct - channel-name - channel-id\n",
+ " 1\n",
+ " \n",
+ " test@email\n",
+ " user\n",
+ " 0\n",
+ " test@email\n",
+ " \n",
+ " \n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " test@test.com\n",
+ " \n",
+ " \n",
+ " test2@email\n",
+ " user\n",
+ " 8\n",
+ " test2@email\n",
+ " \n",
+ " \n",
+ " test3@email\n",
+ " user\n",
+ " 400\n",
+ " test3@email\n",
+ " \n",
+ " \n",
+ " post-id-1\n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " message1\n",
+ " \n",
+ " \n",
+ " test@test.com\n",
+ " 1\n",
+ " test1-attachment\n",
+ " test1-attachment\n",
+ " \n",
+ " \n",
+ " test@test.com\n",
+ " 1\n",
+ " test1-attachment\n",
+ " test1-attachment\n",
+ " Completed\n",
+ " \n",
+ " \n",
+ " post-id-1\n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " FileDeleted\n",
+ " 2\n",
+ " delete test1-attachment\n",
+ " \n",
+ " \n",
+ " test2@email\n",
+ " user\n",
+ " 80\n",
+ " test2@email\n",
+ " \n",
+ " \n",
+ " test@email\n",
+ " user\n",
+ " 400\n",
+ " test@email\n",
+ " \n",
+ " \n",
+ " test3@email\n",
+ " user\n",
+ " 700\n",
+ " test3@email\n",
+ " \n",
+ " \n",
+ " test@test.com\n",
+ " user\n",
+ " 700\n",
+ " test@test.com\n",
+ " \n",
+ " 700\n",
+ " \n",
+ "",
+ }, ""),
+ expectedFiles: 3,
+ },
+ {
+ name: "posts with deleted attachments and deleted post",
+ jobEndTime: 700,
+ cmhs: map[string][]*model.ChannelMemberHistoryResult{
+ "channel-id": {
+ {
+ JoinTime: 0, ChannelId: "channel-id", UserId: "test", UserEmail: "test@email", Username: "test", LeaveTime: model.NewPointer(int64(400)),
+ },
+ {
+ JoinTime: 8, ChannelId: "channel-id", UserId: "test2", UserEmail: "test2@email", Username: "test2", LeaveTime: model.NewPointer(int64(80)),
+ },
+ {
+ JoinTime: 400, ChannelId: "channel-id", UserId: "test3", UserEmail: "test3@email", Username: "test3",
+ },
+ },
+ },
+ activity: []string{"channel-id"},
+ channels: model.ChannelList{{
+ TeamId: "team-id",
+ Id: "channel-id",
+ Name: "channel-name",
+ DisplayName: "channel-display-name",
+ Type: model.ChannelTypeDirect,
+ }},
+ posts: []*model.MessageExport{
+ {
+ PostId: model.NewPointer("post-id-1"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(2)),
+ PostDeleteAt: model.NewPointer(int64(2)),
+ PostMessage: model.NewPointer("message1"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("user-id"),
+ Username: model.NewPointer("username"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{"test1"},
+ PostProps: model.NewPointer("{\"deleteBy\":\"fy8j97gwii84bk4zxprbpc9d9w\"}"),
+ },
+ },
+ attachments: map[string][]*model.FileInfo{
+ "post-id-1": {
+ {
+ Name: "test1-attachment",
+ Id: "test1-attachment",
+ Path: "test1-attachment",
+ CreateAt: 1,
+ UpdateAt: 2,
+ DeleteAt: 2,
+ },
+ },
+ },
+ expectedData: strings.Join([]string{
+ xml.Header,
+ "\n",
+ " \n",
+ " direct - channel-name - channel-id\n",
+ " 1\n",
+ " \n",
+ " test@email\n",
+ " user\n",
+ " 0\n",
+ " test@email\n",
+ " \n",
+ " \n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " test@test.com\n",
+ " \n",
+ " \n",
+ " test2@email\n",
+ " user\n",
+ " 8\n",
+ " test2@email\n",
+ " \n",
+ " \n",
+ " test3@email\n",
+ " user\n",
+ " 400\n",
+ " test3@email\n",
+ " \n",
+ " \n",
+ " post-id-1\n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " message1\n",
+ " \n",
+ " \n",
+ " test@test.com\n",
+ " 1\n",
+ " test1-attachment\n",
+ " test1-attachment\n",
+ " \n",
+ " \n",
+ " test@test.com\n",
+ " 1\n",
+ " test1-attachment\n",
+ " test1-attachment\n",
+ " Completed\n",
+ " \n",
+ " \n",
+ " post-id-1\n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " Deleted\n",
+ " 2\n",
+ " delete message1\n",
+ " \n",
+ " \n",
+ " post-id-1\n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " FileDeleted\n",
+ " 2\n",
+ " delete test1-attachment\n",
+ " \n",
+ " \n",
+ " test2@email\n",
+ " user\n",
+ " 80\n",
+ " test2@email\n",
+ " \n",
+ " \n",
+ " test@email\n",
+ " user\n",
+ " 400\n",
+ " test@email\n",
+ " \n",
+ " \n",
+ " test3@email\n",
+ " user\n",
+ " 700\n",
+ " test3@email\n",
+ " \n",
+ " \n",
+ " test@test.com\n",
+ " user\n",
+ " 700\n",
+ " test@test.com\n",
+ " \n",
+ " 700\n",
+ " \n",
+ "",
+ }, ""),
+ expectedFiles: 3,
+ },
+ {
+ name: "joins and leaves after last post, one batch",
+ jobEndTime: 500,
+ cmhs: map[string][]*model.ChannelMemberHistoryResult{
+ "channel-id": {
+ {JoinTime: 0, ChannelId: "channel-id", UserId: "test", UserEmail: "test@email", Username: "testname", LeaveTime: model.NewPointer(int64(400))},
+ {JoinTime: 8, ChannelId: "channel-id", UserId: "test2", UserEmail: "test2@email", Username: "test2name", LeaveTime: model.NewPointer(int64(80))},
+ {JoinTime: 400, ChannelId: "channel-id", UserId: "test3", UserEmail: "test3@email", Username: "test3name"},
+ {JoinTime: 450, ChannelId: "channel-id", UserId: "test4", UserEmail: "test4@email", Username: "test4name", LeaveTime: model.NewPointer(int64(460))},
+ {JoinTime: 10, ChannelId: "channel-id", UserId: "test_bot", UserEmail: "test_bot@email", Username: "test_botname", IsBot: true, LeaveTime: model.NewPointer(int64(20))},
+ },
+ },
+ activity: []string{"channel-id"},
+ channels: model.ChannelList{{
+ TeamId: "team-id",
+ Id: "channel-id",
+ Name: "channel-name",
+ DisplayName: "channel-display-name",
+ Type: model.ChannelTypeDirect,
+ }},
+ posts: []*model.MessageExport{
+ {
+ PostId: model.NewPointer("post-id1"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(1)),
+ PostMessage: model.NewPointer("message"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("user-id"),
+ Username: model.NewPointer("username"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{},
+ },
+ {
+ PostId: model.NewPointer("post-id2"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(2)),
+ PostDeleteAt: model.NewPointer(int64(2)),
+ PostMessage: model.NewPointer("edit message"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("user-id"),
+ Username: model.NewPointer("username"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{},
+ },
+ {
+ PostId: model.NewPointer("post-id3"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(4)),
+ PostDeleteAt: model.NewPointer(int64(4)),
+ PostMessage: model.NewPointer("message"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("user-id"),
+ Username: model.NewPointer("username"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{},
+ PostProps: model.NewPointer("{\"deleteBy\":\"fy8j97gwii84bk4zxprbpc9d9w\"}"),
+ },
+ {
+ PostId: model.NewPointer("post-id4"),
+ PostRootId: model.NewPointer("post-root-id"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostCreateAt: model.NewPointer(int64(100)),
+ PostUpdateAt: model.NewPointer(int64(100)),
+ PostMessage: model.NewPointer("message"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("user-id"),
+ Username: model.NewPointer("username"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{},
+ },
+ },
+ attachments: map[string][]*model.FileInfo{},
+ expectedData: strings.Join([]string{
+ xml.Header,
+ "\n",
+ " \n",
+ " direct - channel-name - channel-id\n",
+ " 1\n",
+ " \n",
+ " test@email\n",
+ " user\n",
+ " 0\n",
+ " test@email\n",
+ " \n",
+ " \n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " test@test.com\n",
+ " \n",
+ " \n",
+ " test2@email\n",
+ " user\n",
+ " 8\n",
+ " test2@email\n",
+ " \n",
+ " \n",
+ " test_bot@email\n",
+ " bot\n",
+ " 10\n",
+ " test_bot@email\n",
+ " \n",
+ " \n",
+ " test3@email\n",
+ " user\n",
+ " 400\n",
+ " test3@email\n",
+ " \n",
+ " \n",
+ " test4@email\n",
+ " user\n",
+ " 450\n",
+ " test4@email\n",
+ " \n",
+ " \n",
+ " post-id1\n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " message\n",
+ " \n",
+ " \n",
+ " post-id3\n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " message\n",
+ " \n",
+ " \n",
+ " post-id2\n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " UpdatedNoMsgChange\n",
+ " 2\n",
+ " edit message\n",
+ " \n",
+ " \n",
+ " post-id3\n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " Deleted\n",
+ " 4\n",
+ " delete message\n",
+ " \n",
+ " \n",
+ " post-id4\n",
+ " test@test.com\n",
+ " user\n",
+ " 100\n",
+ " message\n",
+ " \n",
+ " \n",
+ " test_bot@email\n",
+ " bot\n",
+ " 20\n",
+ " test_bot@email\n",
+ " \n",
+ " \n",
+ " test2@email\n",
+ " user\n",
+ " 80\n",
+ " test2@email\n",
+ " \n",
+ " \n",
+ " test@email\n",
+ " user\n",
+ " 400\n",
+ " test@email\n",
+ " \n",
+ " \n",
+ " test4@email\n",
+ " user\n",
+ " 460\n",
+ " test4@email\n",
+ " \n",
+ " \n",
+ " test3@email\n",
+ " user\n",
+ " 500\n",
+ " test3@email\n",
+ " \n",
+ " \n",
+ " test@test.com\n",
+ " user\n",
+ " 500\n",
+ " test@test.com\n",
+ " \n",
+ " 500\n",
+ " \n",
+ "",
+ }, ""),
+ expectedFiles: 2,
+ },
}
- for _, tt := range csvExportTests {
+ for _, tt := range actianceExportTests {
t.Run(tt.name, func(t *testing.T) {
mockStore := &storetest.Store{}
defer mockStore.AssertExpectations(t)
if len(tt.attachments) > 0 {
for post_id, attachments := range tt.attachments {
- attachments := attachments // TODO: Remove once go1.22 is used
call := mockStore.FileInfoStore.On("GetForPost", post_id, true, true, false)
call.Run(func(args mock.Arguments) {
call.Return(attachments, nil)
@@ -535,26 +1133,530 @@ func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, at
if len(tt.cmhs) > 0 {
for channelId, cmhs := range tt.cmhs {
- mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", int64(1), int64(100), channelId).Return(cmhs, nil)
+ mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", int64(1), tt.jobEndTime, []string{channelId}).
+ Return(cmhs, nil)
}
}
- warnings, appErr := ActianceExport(rctx, tt.posts, mockStore, exportBackend, attachmentBackend, "test")
- assert.Nil(t, appErr)
- assert.Equal(t, int64(0), warnings)
+ if tt.activity != nil {
+ mockStore.ChannelMemberHistoryStore.On("GetChannelsWithActivityDuring", mock.Anything, mock.Anything).
+ Return(tt.activity, nil)
+ }
- data, nErr := exportBackend.ReadFile("test/actiance_export.xml")
- assert.NoError(t, nErr)
- assert.Equal(t, tt.expectedData, string(data))
+ if tt.channels != nil {
+ mockStore.ChannelStore.On("GetMany", tt.activity, true).
+ Return(tt.channels, nil)
+ }
+
+ myMockReporter := MyReporter{}
+ defer myMockReporter.AssertExpectations(t)
+ if len(tt.activity) > 0 {
+ myMockReporter.On("ReportProgressMessage", "ent.message_export.actiance_export.calculate_channel_exports.channel_message")
+ }
+ if len(tt.cmhs) > 0 {
+ myMockReporter.On("ReportProgressMessage", "ent.message_export.actiance_export.calculate_channel_exports.activity_message")
+ }
+
+ channelMetadata, channelMemberHistories, err := shared.CalculateChannelExports(rctx,
+ shared.ChannelExportsParams{
+ Store: shared.NewMessageExportStore(mockStore),
+ ExportPeriodStartTime: 1,
+ ExportPeriodEndTime: tt.jobEndTime,
+ ChannelBatchSize: 100,
+ ChannelHistoryBatchSize: 100,
+ ReportProgressMessage: myMockReporter.ReportProgressMessage,
+ })
+ assert.NoError(t, err)
+
+ exportFileName := path.Join("export", "jobName", "jobName-batch001.zip")
+ res, err := ActianceExport(rctx, shared.ExportParams{
+ ChannelMetadata: channelMetadata,
+ Posts: tt.posts,
+ ChannelMemberHistories: channelMemberHistories,
+ BatchPath: exportFileName,
+ BatchStartTime: 1,
+ BatchEndTime: tt.jobEndTime,
+ Db: shared.NewMessageExportStore(mockStore),
+ FileAttachmentBackend: attachmentBackend,
+ ExportBackend: exportBackend,
+ })
+ assert.NoError(t, err)
+ assert.Equal(t, 0, res.NumWarnings)
+
+ zipBytes, err := exportBackend.ReadFile(exportFileName)
+ assert.NoError(t, err)
+ zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+ assert.NoError(t, err)
+ actiancexml, err := zipReader.File[0].Open()
+ require.NoError(t, err)
+ defer actiancexml.Close()
+ xmlData, err := io.ReadAll(actiancexml)
+ assert.NoError(t, err)
+
+ // Note: for debugging, better keep this in case we need it again.
+ //t.Logf("<><> actual:\n%s\n", string(xmlData))
+ assert.Equal(t, tt.expectedData, string(xmlData))
t.Cleanup(func() {
- err := exportBackend.RemoveFile("test/actiance_export.xml")
+ err = exportBackend.RemoveFile(exportFileName)
assert.NoError(t, err)
})
})
}
}
+func TestActianceExportMultipleBatches(t *testing.T) {
+ t.Run("no dedicated export filestore", func(t *testing.T) {
+ exportTempDir, err := os.MkdirTemp("", "")
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ err = os.RemoveAll(exportTempDir)
+ assert.NoError(t, err)
+ })
+
+ fileBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
+ DriverName: model.ImageDriverLocal,
+ Directory: exportTempDir,
+ })
+ assert.NoError(t, err)
+
+ runTestActianceExportMultipleBatches(t, fileBackend, fileBackend)
+ })
+
+ t.Run("using dedicated export filestore", func(t *testing.T) {
+ exportTempDir, err := os.MkdirTemp("", "")
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ err = os.RemoveAll(exportTempDir)
+ assert.NoError(t, err)
+ })
+
+ exportBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
+ DriverName: model.ImageDriverLocal,
+ Directory: exportTempDir,
+ })
+ assert.NoError(t, err)
+
+ attachmentTempDir, err := os.MkdirTemp("", "")
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ err = os.RemoveAll(attachmentTempDir)
+ assert.NoError(t, err)
+ })
+
+ attachmentBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
+ DriverName: model.ImageDriverLocal,
+ Directory: attachmentTempDir,
+ })
+
+ runTestActianceExportMultipleBatches(t, exportBackend, attachmentBackend)
+ })
+}
+
+func runTestActianceExportMultipleBatches(t *testing.T, exportBackend filestore.FileBackend, attachmentBackend filestore.FileBackend) {
+ rctx := request.TestContext(t)
+ rctx = rctx.WithT(i18n.IdentityTfunc()).(*request.Context)
+
+ chanTypeDirect := model.ChannelTypeDirect
+ actianceMultiBatchExportTests := []struct {
+ name string
+ jobStartTime int64
+ jobEndTime int64
+ numBatches int
+ activity []string
+ channels model.ChannelList
+ cmhs map[string][]*model.ChannelMemberHistoryResult
+ posts [][]*model.MessageExport
+ attachments map[string][]*model.FileInfo
+ expectedData []string
+ expectedFiles int
+ }{
+ {
+ name: "joins and leaves after last post, and before second batch, two batches",
+ jobStartTime: 1,
+ jobEndTime: 500,
+ numBatches: 2,
+ cmhs: map[string][]*model.ChannelMemberHistoryResult{
+ "channel-id": {
+ // will be included in both batches:
+ {JoinTime: 0, ChannelId: "channel-id", UserId: "test", UserEmail: "test@email", Username: "testname", LeaveTime: model.NewPointer(int64(400))},
+ // Only first batch:
+ {JoinTime: 2, ChannelId: "channel-id", UserId: "testA", UserEmail: "testA@email", Username: "testAname", LeaveTime: model.NewPointer(int64(3))},
+ // Only second batch:
+ {JoinTime: 8, ChannelId: "channel-id", UserId: "test2", UserEmail: "test2@email", Username: "test2name", LeaveTime: model.NewPointer(int64(80))},
+ {JoinTime: 10, ChannelId: "channel-id", UserId: "test_bot", UserEmail: "test_bot@email", Username: "test_botname", IsBot: true, LeaveTime: model.NewPointer(int64(20))},
+ {JoinTime: 400, ChannelId: "channel-id", UserId: "test3", UserEmail: "test3@email", Username: "test3name"},
+ {JoinTime: 450, ChannelId: "channel-id", UserId: "test4", UserEmail: "test4@email", Username: "test4name", LeaveTime: model.NewPointer(int64(460))},
+ },
+ },
+ activity: []string{"channel-id"},
+ channels: model.ChannelList{{
+ TeamId: "team-id",
+ Id: "channel-id",
+ Name: "channel-name",
+ DisplayName: "channel-display-name",
+ Type: model.ChannelTypeDirect,
+ }},
+ posts: [][]*model.MessageExport{
+ {
+ {
+ PostId: model.NewPointer("post-id1"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(1)),
+ PostMessage: model.NewPointer("message"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("user-id"),
+ Username: model.NewPointer("username"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{},
+ },
+ {
+ PostId: model.NewPointer("post-id2"),
+ PostOriginalId: model.NewPointer("post-original-id"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(4)),
+ PostDeleteAt: model.NewPointer(int64(4)),
+ PostMessage: model.NewPointer("edit message"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("user-id"),
+ Username: model.NewPointer("username"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{},
+ },
+ },
+ {
+ {
+ PostId: model.NewPointer("post-id3"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(5)),
+ PostDeleteAt: model.NewPointer(int64(5)),
+ PostMessage: model.NewPointer("message2"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("user-id"),
+ Username: model.NewPointer("username"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{},
+ PostProps: model.NewPointer("{\"deleteBy\":\"fy8j97gwii84bk4zxprbpc9d9w\"}"),
+ },
+ {
+ PostId: model.NewPointer("post-id4"),
+ PostRootId: model.NewPointer("post-root-id"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostCreateAt: model.NewPointer(int64(100)),
+ PostUpdateAt: model.NewPointer(int64(100)),
+ PostMessage: model.NewPointer("message3"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("user-id"),
+ Username: model.NewPointer("username"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{},
+ },
+ }},
+ attachments: map[string][]*model.FileInfo{},
+ expectedData: []string{
+ strings.Join([]string{
+ xml.Header,
+ "\n",
+ " \n",
+ " direct - channel-name - channel-id\n",
+ " 1\n",
+ " \n",
+ " test@email\n",
+ " user\n",
+ " 0\n",
+ " test@email\n",
+ " \n",
+ " \n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " test@test.com\n",
+ " \n",
+ " \n",
+ " testA@email\n",
+ " user\n",
+ " 2\n",
+ " testA@email\n",
+ " \n",
+ " \n",
+ " post-id1\n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " message\n",
+ " \n",
+ " \n",
+ " post-id2\n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " EditedOriginalMsg\n",
+ " 4\n",
+ " post-original-id\n",
+ " edit message\n",
+ " \n",
+ " \n",
+ " testA@email\n",
+ " user\n",
+ " 3\n",
+ " testA@email\n",
+ " \n",
+ " \n",
+ " test@email\n",
+ " user\n",
+ " 4\n",
+ " test@email\n",
+ " \n",
+ " \n",
+ " test@test.com\n",
+ " user\n",
+ " 4\n",
+ " test@test.com\n",
+ " \n",
+ " 4\n",
+ " \n",
+ "",
+ }, ""),
+ strings.Join([]string{
+ xml.Header,
+ "\n",
+ " \n",
+ " direct - channel-name - channel-id\n",
+ " 4\n",
+ " \n",
+ " test@email\n",
+ " user\n",
+ " 0\n",
+ " test@email\n",
+ " \n",
+ " \n",
+ " test@test.com\n",
+ " user\n",
+ " 4\n",
+ " test@test.com\n",
+ " \n",
+ " \n",
+ " test2@email\n",
+ " user\n",
+ " 8\n",
+ " test2@email\n",
+ " \n",
+ " \n",
+ " test_bot@email\n",
+ " bot\n",
+ " 10\n",
+ " test_bot@email\n",
+ " \n",
+ " \n",
+ " test3@email\n",
+ " user\n",
+ " 400\n",
+ " test3@email\n",
+ " \n",
+ " \n",
+ " test4@email\n",
+ " user\n",
+ " 450\n",
+ " test4@email\n",
+ " \n",
+ " \n",
+ " post-id3\n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " message2\n",
+ " \n",
+ " \n",
+ " post-id3\n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " Deleted\n",
+ " 5\n",
+ " delete message2\n",
+ " \n",
+ " \n",
+ " post-id4\n",
+ " test@test.com\n",
+ " user\n",
+ " 100\n",
+ " message3\n",
+ " \n",
+ " \n",
+ " test_bot@email\n",
+ " bot\n",
+ " 20\n",
+ " test_bot@email\n",
+ " \n",
+ " \n",
+ " test2@email\n",
+ " user\n",
+ " 80\n",
+ " test2@email\n",
+ " \n",
+ " \n",
+ " test@email\n",
+ " user\n",
+ " 400\n",
+ " test@email\n",
+ " \n",
+ " \n",
+ " test4@email\n",
+ " user\n",
+ " 460\n",
+ " test4@email\n",
+ " \n",
+ " \n",
+ " test3@email\n",
+ " user\n",
+ " 500\n",
+ " test3@email\n",
+ " \n",
+ " \n",
+ " test@test.com\n",
+ " user\n",
+ " 500\n",
+ " test@test.com\n",
+ " \n",
+ " 500\n",
+ " \n",
+ "",
+ }, ""),
+ },
+ expectedFiles: 2,
+ },
+ }
+
+ for _, tt := range actianceMultiBatchExportTests {
+ t.Run(tt.name, func(t *testing.T) {
+ mockStore := &storetest.Store{}
+ defer mockStore.AssertExpectations(t)
+
+ if len(tt.attachments) > 0 {
+ for post_id, attachments := range tt.attachments {
+ call := mockStore.FileInfoStore.On("GetForPost", post_id, true, true, false)
+ call.Run(func(args mock.Arguments) {
+ call.Return(attachments, nil)
+ })
+ _, err := attachmentBackend.WriteFile(bytes.NewReader([]byte{}), attachments[0].Path)
+ require.NoError(t, err)
+
+ t.Cleanup(func() {
+ err = attachmentBackend.RemoveFile(attachments[0].Path)
+ require.NoError(t, err)
+ })
+ }
+ }
+
+ if len(tt.cmhs) > 0 {
+ for channelId, cmhs := range tt.cmhs {
+ mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", int64(1), tt.jobEndTime, []string{channelId}).
+ Return(cmhs, nil)
+ }
+ }
+
+ if tt.activity != nil {
+ mockStore.ChannelMemberHistoryStore.On("GetChannelsWithActivityDuring", mock.Anything, mock.Anything).
+ Return(tt.activity, nil)
+ }
+
+ if tt.channels != nil {
+ mockStore.ChannelStore.On("GetMany", tt.activity, true).
+ Return(tt.channels, nil)
+ }
+
+ myMockReporter := MyReporter{}
+ defer myMockReporter.AssertExpectations(t)
+ if len(tt.activity) > 0 {
+ myMockReporter.On("ReportProgressMessage", "ent.message_export.actiance_export.calculate_channel_exports.channel_message")
+ }
+ if len(tt.cmhs) > 0 {
+ myMockReporter.On("ReportProgressMessage", "ent.message_export.actiance_export.calculate_channel_exports.activity_message")
+ }
+
+ channelMetadata, channelMemberHistories, err := shared.CalculateChannelExports(rctx,
+ shared.ChannelExportsParams{
+ Store: shared.NewMessageExportStore(mockStore),
+ ExportPeriodStartTime: 1,
+ ExportPeriodEndTime: tt.jobEndTime,
+ ChannelBatchSize: 100,
+ ChannelHistoryBatchSize: 100,
+ ReportProgressMessage: myMockReporter.ReportProgressMessage,
+ })
+ assert.NoError(t, err)
+
+ batchStartTime := int64(1)
+
+ for batch := 0; batch < tt.numBatches; batch++ {
+ var batchEndTime int64
+ if batch == tt.numBatches-1 {
+ batchEndTime = tt.jobEndTime
+ } else {
+ batchEndTime = *tt.posts[batch][len(tt.posts[batch])-1].PostUpdateAt
+ }
+ exportFileName := path.Join("export", "jobName",
+ fmt.Sprintf("jobName-batch00%d.zip", batch+1))
+
+ res, err := ActianceExport(rctx, shared.ExportParams{
+ ChannelMetadata: channelMetadata,
+ Posts: tt.posts[batch],
+ ChannelMemberHistories: channelMemberHistories,
+ BatchPath: exportFileName,
+ BatchStartTime: batchStartTime,
+ BatchEndTime: batchEndTime,
+ Db: shared.NewMessageExportStore(mockStore),
+ FileAttachmentBackend: attachmentBackend,
+ ExportBackend: exportBackend,
+ })
+ assert.NoError(t, err)
+ assert.Equal(t, 0, res.NumWarnings)
+
+ zipBytes, err := exportBackend.ReadFile(exportFileName)
+ assert.NoError(t, err)
+ zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+ assert.NoError(t, err)
+ actiancexml, err := zipReader.File[0].Open()
+ require.NoError(t, err)
+ xmlData, err := io.ReadAll(actiancexml)
+ actiancexml.Close()
+ assert.NoError(t, err)
+
+ assert.Equal(t, tt.expectedData[batch], string(xmlData), fmt.Sprintf("batch %v", batch))
+
+ batchStartTime = *tt.posts[batch][len(tt.posts[batch])-1].PostUpdateAt
+
+ t.Cleanup(func() {
+ err = exportBackend.RemoveFile(exportFileName)
+ assert.NoError(t, err)
+ })
+ }
+ })
+ }
+}
+
func TestMultipleActianceExport(t *testing.T) {
t.Run("no dedicated export filestore", func(t *testing.T) {
exportTempDir, err := os.MkdirTemp("", "")
@@ -605,10 +1707,14 @@ func TestMultipleActianceExport(t *testing.T) {
func runTestMultipleActianceExport(t *testing.T, exportBackend filestore.FileBackend, attachmentBackend filestore.FileBackend) {
rctx := request.TestContext(t)
+ rctx = rctx.WithT(i18n.IdentityTfunc()).(*request.Context)
chanTypeDirect := model.ChannelTypeDirect
- actianceExportTests := []struct {
+ multActianceExportTests := []struct {
name string
+ jobEndTime int64
+ activity []string
+ channels model.ChannelList
cmhs map[string][]*model.ChannelMemberHistoryResult
posts map[string][]*model.MessageExport
attachments map[string][]*model.FileInfo
@@ -616,10 +1722,19 @@ func runTestMultipleActianceExport(t *testing.T, exportBackend filestore.FileBac
expectedFiles int
}{
{
- name: "post,export,delete,export",
+ name: "post,export,delete,export",
+ jobEndTime: 500,
+ activity: []string{"channel-id"},
+ channels: model.ChannelList{{
+ TeamId: "team-id",
+ Id: "channel-id",
+ Name: "channel-name",
+ DisplayName: "channel-display-name",
+ Type: model.ChannelTypeDirect,
+ }},
cmhs: map[string][]*model.ChannelMemberHistoryResult{
"channel-id": {
- {JoinTime: 0, UserId: "user-id", UserEmail: "test@test.com", Username: "username", LeaveTime: model.NewPointer(int64(400))},
+ {JoinTime: 0, ChannelId: "channel-id", UserId: "user-id", UserEmail: "test@test.com", Username: "username", LeaveTime: model.NewPointer(int64(400))},
},
},
posts: map[string][]*model.MessageExport{
@@ -646,7 +1761,6 @@ func runTestMultipleActianceExport(t *testing.T, exportBackend filestore.FileBac
"step2": {
{
PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
TeamDisplayName: model.NewPointer("team-display-name"),
@@ -680,22 +1794,23 @@ func runTestMultipleActianceExport(t *testing.T, exportBackend filestore.FileBac
" test@test.com\n",
" \n",
" \n",
+ " post-id\n",
" test@test.com\n",
" user\n",
" 1\n",
" message\n",
- " \n",
" \n",
" \n",
" test@test.com\n",
" user\n",
- " 1\n",
+ " 400\n",
" test@test.com\n",
" \n",
- " 1\n",
+ " 500\n",
" \n",
"",
}, ""),
+ // We're redoing the export completely, so we'll get the original message then the deleted record
"step2": strings.Join([]string{
xml.Header,
"\n",
@@ -709,26 +1824,28 @@ func runTestMultipleActianceExport(t *testing.T, exportBackend filestore.FileBac
" test@test.com\n",
" \n",
" \n",
+ " post-id\n",
" test@test.com\n",
" user\n",
" 1\n",
" message\n",
- " \n",
" \n",
" \n",
+ " post-id\n",
" test@test.com\n",
" user\n",
- " 2\n",
+ " 1\n",
+ " Deleted\n",
+ " 2\n",
" delete message\n",
- " \n",
" \n",
" \n",
" test@test.com\n",
" user\n",
- " 1\n",
+ " 400\n",
" test@test.com\n",
" \n",
- " 1\n",
+ " 500\n",
" \n",
"",
}, ""),
@@ -736,17 +1853,25 @@ func runTestMultipleActianceExport(t *testing.T, exportBackend filestore.FileBac
expectedFiles: 2,
},
{
- name: "post,export,edit,export",
+ name: "post,export,edit,export",
+ jobEndTime: 600,
cmhs: map[string][]*model.ChannelMemberHistoryResult{
"channel-id": {
- {JoinTime: 0, UserId: "user-id", UserEmail: "test@test.com", Username: "username", LeaveTime: model.NewPointer(int64(400))},
+ {JoinTime: 0, ChannelId: "channel-id", UserId: "user-id", UserEmail: "test@test.com", Username: "username", LeaveTime: model.NewPointer(int64(450))},
},
},
+ activity: []string{"channel-id"},
+ channels: model.ChannelList{{
+ TeamId: "team-id",
+ Id: "channel-id",
+ Name: "channel-name",
+ DisplayName: "channel-display-name",
+ Type: model.ChannelTypeDirect,
+ }},
posts: map[string][]*model.MessageExport{
"step1": {
{
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostId: model.NewPointer("post-original-id"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
TeamDisplayName: model.NewPointer("team-display-name"),
@@ -764,8 +1889,9 @@ func runTestMultipleActianceExport(t *testing.T, exportBackend filestore.FileBac
},
},
"step2": {
+ // new post which holds the original message contents
{
- PostId: model.NewPointer("post-id"),
+ PostId: model.NewPointer("post-id-new"),
PostOriginalId: model.NewPointer("post-original-id"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
@@ -776,6 +1902,25 @@ func runTestMultipleActianceExport(t *testing.T, exportBackend filestore.FileBac
PostCreateAt: model.NewPointer(int64(1)),
PostUpdateAt: model.NewPointer(int64(2)),
PostDeleteAt: model.NewPointer(int64(2)),
+ PostMessage: model.NewPointer("message"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("user-id"),
+ Username: model.NewPointer("username"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{},
+ },
+ // old post which has been edited
+ {
+ PostId: model.NewPointer("post-original-id"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(2)),
+ PostEditAt: model.NewPointer(int64(2)),
PostMessage: model.NewPointer("edit message"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
@@ -799,22 +1944,23 @@ func runTestMultipleActianceExport(t *testing.T, exportBackend filestore.FileBac
" test@test.com\n",
" \n",
" \n",
+ " post-original-id\n",
" test@test.com\n",
" user\n",
" 1\n",
" message\n",
- " \n",
" \n",
" \n",
" test@test.com\n",
" user\n",
- " 1\n",
+ " 450\n",
" test@test.com\n",
" \n",
- " 1\n",
+ " 600\n",
" \n",
"",
}, ""),
+ // We're redoing the export completely, so we'll get the original message then the deleted record
"step2": strings.Join([]string{
xml.Header,
"\n",
@@ -828,19 +1974,31 @@ func runTestMultipleActianceExport(t *testing.T, exportBackend filestore.FileBac
" test@test.com\n",
" \n",
" \n",
+ " post-id-new\n",
" test@test.com\n",
" user\n",
" 1\n",
+ " EditedOriginalMsg\n",
+ " 2\n",
+ " post-original-id\n",
+ " message\n",
+ " \n",
+ " \n",
+ " post-original-id\n",
+ " test@test.com\n",
+ " user\n",
+ " 1\n",
+ " EditedNewMsg\n",
+ " 2\n",
" edit message\n",
- " \n",
" \n",
" \n",
" test@test.com\n",
" user\n",
- " 1\n",
+ " 450\n",
" test@test.com\n",
" \n",
- " 1\n",
+ " 600\n",
" \n",
"",
}, ""),
@@ -849,191 +2007,109 @@ func runTestMultipleActianceExport(t *testing.T, exportBackend filestore.FileBac
},
}
- for _, tt := range actianceExportTests {
+ for _, tt := range multActianceExportTests {
t.Run(tt.name, func(t *testing.T) {
mockStore := &storetest.Store{}
defer mockStore.AssertExpectations(t)
if len(tt.cmhs) > 0 {
for channelId, cmhs := range tt.cmhs {
- mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", int64(1), int64(1), channelId).Return(cmhs, nil)
+ mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", int64(1), tt.jobEndTime, []string{channelId}).
+ Return(cmhs, nil)
}
}
- warnings, appErr := ActianceExport(rctx, tt.posts["step1"], mockStore, exportBackend, attachmentBackend, "test")
- assert.Nil(t, appErr)
- assert.Equal(t, int64(0), warnings)
+ if tt.activity != nil {
+ mockStore.ChannelMemberHistoryStore.On("GetChannelsWithActivityDuring", mock.Anything, mock.Anything).
+ Return(tt.activity, nil)
+ }
- data, err := exportBackend.ReadFile("test/actiance_export.xml")
+ if tt.channels != nil {
+ mockStore.ChannelStore.On("GetMany", tt.activity, true).
+ Return(tt.channels, nil)
+ }
+
+ myMockReporter := MyReporter{}
+ defer myMockReporter.AssertExpectations(t)
+ if len(tt.activity) > 0 {
+ myMockReporter.On("ReportProgressMessage", "ent.message_export.actiance_export.calculate_channel_exports.channel_message")
+ }
+ if len(tt.cmhs) > 0 {
+ myMockReporter.On("ReportProgressMessage", "ent.message_export.actiance_export.calculate_channel_exports.activity_message")
+ }
+
+ channelMetadata, channelMemberHistories, err := shared.CalculateChannelExports(rctx,
+ shared.ChannelExportsParams{
+ Store: shared.NewMessageExportStore(mockStore),
+ ExportPeriodStartTime: 1,
+ ExportPeriodEndTime: tt.jobEndTime,
+ ChannelBatchSize: 100,
+ ChannelHistoryBatchSize: 100,
+ ReportProgressMessage: myMockReporter.ReportProgressMessage,
+ })
assert.NoError(t, err)
- assert.Equal(t, tt.expectedData["step1"], string(data))
- warnings, appErr = ActianceExport(rctx, tt.posts["step2"], mockStore, exportBackend, attachmentBackend, "test")
- assert.Nil(t, appErr)
- assert.Equal(t, int64(0), warnings)
+ exportFileName := path.Join("export", "jobName", "jobName-batch001.zip")
+ res, err := ActianceExport(rctx, shared.ExportParams{
+ ChannelMetadata: channelMetadata,
+ Posts: tt.posts["step1"],
+ ChannelMemberHistories: channelMemberHistories,
+ BatchPath: exportFileName,
+ BatchStartTime: 1,
+ BatchEndTime: tt.jobEndTime,
+ Db: shared.NewMessageExportStore(mockStore),
+ FileAttachmentBackend: attachmentBackend,
+ ExportBackend: exportBackend,
+ })
- data, err = exportBackend.ReadFile("test/actiance_export.xml")
assert.NoError(t, err)
- assert.Equal(t, tt.expectedData["step2"], string(data))
+ assert.Equal(t, 0, res.NumWarnings)
+
+ zipBytes, err := exportBackend.ReadFile(exportFileName)
+ assert.NoError(t, err)
+ zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+ assert.NoError(t, err)
+ actiancexml, err := zipReader.File[0].Open()
+ require.NoError(t, err)
+ defer actiancexml.Close()
+ xmlData, err := io.ReadAll(actiancexml)
+ assert.NoError(t, err)
+
+ assert.Equal(t, tt.expectedData["step1"], string(xmlData))
+
+ res, err = ActianceExport(rctx, shared.ExportParams{
+ ChannelMetadata: channelMetadata,
+ Posts: tt.posts["step2"],
+ ChannelMemberHistories: channelMemberHistories,
+ BatchPath: exportFileName,
+ BatchStartTime: 1,
+ BatchEndTime: tt.jobEndTime,
+ Db: shared.NewMessageExportStore(mockStore),
+ FileAttachmentBackend: attachmentBackend,
+ ExportBackend: exportBackend,
+ })
+ assert.NoError(t, err)
+ assert.Equal(t, 0, res.NumWarnings)
+
+ zipBytes, err = exportBackend.ReadFile(exportFileName)
+ assert.NoError(t, err)
+ zipReader, err = zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+ assert.NoError(t, err)
+ actiancexml, err = zipReader.File[0].Open()
+ require.NoError(t, err)
+ defer actiancexml.Close()
+ xmlData, err = io.ReadAll(actiancexml)
+ assert.NoError(t, err)
+
+ assert.Equal(t, tt.expectedData["step2"], string(xmlData))
t.Cleanup(func() {
- err = exportBackend.RemoveFile("test/actiance_export.xml")
+ err = exportBackend.RemoveFile(exportFileName)
assert.NoError(t, err)
})
})
}
}
-func TestPostToAttachmentsEntries(t *testing.T) {
- chanTypeDirect := model.ChannelTypeDirect
- tt := []struct {
- name string
- post model.MessageExport
- attachments []*model.FileInfo
- expectedStarts []any
- expectedStops []any
- expectedFileInfos []*model.FileInfo
- expectedDeleteFileMessages []any
- expectError bool
- }{
- {
- name: "no-attachments",
- post: model.MessageExport{
- ChannelId: model.NewPointer("Test"),
- ChannelDisplayName: model.NewPointer("Test"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("Some message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("test"),
- Username: model.NewPointer("test"),
- ChannelType: &chanTypeDirect,
- },
- attachments: nil,
- expectedStarts: nil,
- expectedStops: nil,
- expectedFileInfos: nil,
- expectedDeleteFileMessages: nil,
- expectError: false,
- },
- {
- name: "one-attachment",
- post: model.MessageExport{
- PostId: model.NewPointer("test"),
- ChannelId: model.NewPointer("Test"),
- ChannelDisplayName: model.NewPointer("Test"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("Some message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("test"),
- Username: model.NewPointer("test"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"12345"},
- },
- attachments: []*model.FileInfo{
- {Name: "test", Id: "12345", Path: "filename.txt"},
- },
- expectedStarts: []any{
- &FileUploadStartExport{UserEmail: "test@test.com", UploadStartTime: 1, Filename: "test", FilePath: "filename.txt"},
- },
- expectedStops: []any{
- &FileUploadStopExport{UserEmail: "test@test.com", UploadStopTime: 1, Filename: "test", FilePath: "filename.txt", Status: "Completed"},
- },
- expectedFileInfos: []*model.FileInfo{
- {Name: "test", Id: "12345", Path: "filename.txt"},
- },
- expectedDeleteFileMessages: []any{},
- expectError: false,
- },
- {
- name: "two-attachment",
- post: model.MessageExport{
- PostId: model.NewPointer("test"),
- ChannelId: model.NewPointer("Test"),
- ChannelDisplayName: model.NewPointer("Test"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("Some message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("test"),
- Username: model.NewPointer("test"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"12345", "54321"},
- },
- attachments: []*model.FileInfo{
- {Name: "test", Id: "12345", Path: "filename.txt"},
- {Name: "test2", Id: "54321", Path: "filename2.txt"},
- },
- expectedStarts: []any{
- &FileUploadStartExport{UserEmail: "test@test.com", UploadStartTime: 1, Filename: "test", FilePath: "filename.txt"},
- &FileUploadStartExport{UserEmail: "test@test.com", UploadStartTime: 1, Filename: "test2", FilePath: "filename2.txt"},
- },
- expectedStops: []any{
- &FileUploadStopExport{UserEmail: "test@test.com", UploadStopTime: 1, Filename: "test", FilePath: "filename.txt", Status: "Completed"},
- &FileUploadStopExport{UserEmail: "test@test.com", UploadStopTime: 1, Filename: "test2", FilePath: "filename2.txt", Status: "Completed"},
- },
- expectedFileInfos: []*model.FileInfo{
- {Name: "test", Id: "12345", Path: "filename.txt"},
- {Name: "test2", Id: "54321", Path: "filename2.txt"},
- },
- expectedDeleteFileMessages: []any{},
- expectError: false,
- },
- {
- name: "one-attachment-deleted",
- post: model.MessageExport{
- PostId: model.NewPointer("test"),
- ChannelId: model.NewPointer("Test"),
- ChannelDisplayName: model.NewPointer("Test"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostDeleteAt: model.NewPointer(int64(2)),
- PostMessage: model.NewPointer("Some message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("test"),
- Username: model.NewPointer("test"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"12345", "54321"},
- },
- attachments: []*model.FileInfo{
- {Name: "test", Id: "12345", Path: "filename.txt", DeleteAt: 2},
- },
- expectedStarts: []any{
- &FileUploadStartExport{UserEmail: "test@test.com", UploadStartTime: 1, Filename: "test", FilePath: "filename.txt"},
- },
- expectedStops: []any{
- &FileUploadStopExport{UserEmail: "test@test.com", UploadStopTime: 1, Filename: "test", FilePath: "filename.txt", Status: "Completed"},
- },
- expectedFileInfos: []*model.FileInfo{
- {Name: "test", Id: "12345", Path: "filename.txt", DeleteAt: 2},
- },
- expectedDeleteFileMessages: []any{
- &PostExport{UserEmail: "test@test.com", UserType: "user", PostTime: 2, Message: "delete " + "filename.txt"},
- },
- expectError: false,
- },
- }
-
- for _, tc := range tt {
- t.Run(tc.name, func(t *testing.T) {
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- if len(tc.attachments) > 0 {
- call := mockStore.FileInfoStore.On("GetForPost", *tc.post.PostId, true, true, false)
- call.Run(func(args mock.Arguments) {
- call.Return(tc.attachments, nil)
- })
- }
- uploadStarts, uploadStops, files, deleteFileMessages, err := postToAttachmentsEntries(&tc.post, mockStore)
- if tc.expectError {
- assert.NotNil(t, err)
- } else {
- assert.Nil(t, err)
- }
- assert.Equal(t, tc.expectedStarts, uploadStarts)
- assert.Equal(t, tc.expectedStops, uploadStops)
- assert.Equal(t, tc.expectedFileInfos, files)
- assert.Equal(t, tc.expectedDeleteFileMessages, deleteFileMessages)
- })
- }
-}
func TestWriteExportWarnings(t *testing.T) {
tempDir, err := os.MkdirTemp("", "")
@@ -1066,10 +2142,170 @@ func TestWriteExportWarnings(t *testing.T) {
Channels: []ChannelExport{},
}
- warnings, appErr := writeExport(rctx, export, uploadedFiles, "test", fileBackend, fileBackend)
- assert.Nil(t, appErr)
- assert.Equal(t, int64(2), warnings)
+ exportFileName := path.Join("export", "jobName", "jobName-batch001.zip")
+ res, err := writeExport(rctx, export, uploadedFiles, fileBackend, fileBackend, exportFileName)
+ assert.NoError(t, err)
+ assert.Equal(t, 2, res.NumWarnings)
- err = fileBackend.RemoveFile("test/actiance_export.xml")
+ err = fileBackend.RemoveFile(exportFileName)
require.NoError(t, err)
}
+
+func Test_channelHasActivity(t *testing.T) {
+ tests := []struct {
+ name string
+ cmhs []*model.ChannelMemberHistoryResult
+ startTime int64
+ endTime int64
+ want bool
+ }{
+ {
+ name: "no activity",
+ cmhs: nil,
+ startTime: 1000,
+ endTime: 2000,
+ want: false,
+ },
+ {
+ name: "no activity in bounds (but activity out of bounds)",
+ cmhs: []*model.ChannelMemberHistoryResult{
+ {
+ ChannelId: "channelid",
+ UserId: "testid",
+ JoinTime: 900,
+ LeaveTime: nil,
+ UserEmail: "testemail@email.com",
+ Username: "test_username",
+ IsBot: false,
+ UserDeleteAt: 0,
+ },
+ {
+ ChannelId: "channelid",
+ UserId: "testid2",
+ JoinTime: 2100,
+ LeaveTime: nil,
+ UserEmail: "testemail@email.com",
+ Username: "test_username",
+ IsBot: false,
+ UserDeleteAt: 0,
+ },
+ },
+ startTime: 1000,
+ endTime: 2000,
+ want: false,
+ },
+ {
+ name: "join on lower bounds",
+ cmhs: []*model.ChannelMemberHistoryResult{
+ {
+ ChannelId: "channelid",
+ UserId: "testid",
+ JoinTime: 1000,
+ LeaveTime: nil,
+ UserEmail: "testemail@email.com",
+ Username: "test_username",
+ IsBot: false,
+ UserDeleteAt: 0,
+ },
+ },
+ startTime: 1000,
+ endTime: 2000,
+ want: true,
+ },
+ {
+ name: "join within bounds",
+ cmhs: []*model.ChannelMemberHistoryResult{
+ {
+ ChannelId: "channelid",
+ UserId: "testid",
+ JoinTime: 1500,
+ LeaveTime: nil,
+ UserEmail: "testemail@email.com",
+ Username: "test_username",
+ IsBot: false,
+ UserDeleteAt: 0,
+ },
+ },
+ startTime: 1000,
+ endTime: 2000,
+ want: true,
+ },
+ {
+ name: "join on upper bounds",
+ cmhs: []*model.ChannelMemberHistoryResult{
+ {
+ ChannelId: "channelid",
+ UserId: "testid",
+ JoinTime: 2000,
+ LeaveTime: nil,
+ UserEmail: "testemail@email.com",
+ Username: "test_username",
+ IsBot: false,
+ UserDeleteAt: 0,
+ },
+ },
+ startTime: 1000,
+ endTime: 2000,
+ want: true,
+ },
+ {
+ name: "leave on lower bounds",
+ cmhs: []*model.ChannelMemberHistoryResult{
+ {
+ ChannelId: "channelid",
+ UserId: "testid",
+ JoinTime: 100,
+ LeaveTime: model.NewPointer[int64](1000),
+ UserEmail: "testemail@email.com",
+ Username: "test_username",
+ IsBot: false,
+ UserDeleteAt: 0,
+ },
+ },
+ startTime: 1000,
+ endTime: 2000,
+ want: true,
+ },
+ {
+ name: "leave within bounds",
+ cmhs: []*model.ChannelMemberHistoryResult{
+ {
+ ChannelId: "channelid",
+ UserId: "testid",
+ JoinTime: 100,
+ LeaveTime: model.NewPointer[int64](1500),
+ UserEmail: "testemail@email.com",
+ Username: "test_username",
+ IsBot: false,
+ UserDeleteAt: 0,
+ },
+ },
+ startTime: 1000,
+ endTime: 2000,
+ want: true,
+ },
+ {
+ name: "leave on upper bounds",
+ cmhs: []*model.ChannelMemberHistoryResult{
+ {
+ ChannelId: "channelid",
+ UserId: "testid",
+ JoinTime: 100,
+ LeaveTime: model.NewPointer[int64](2000),
+ UserEmail: "testemail@email.com",
+ Username: "test_username",
+ IsBot: false,
+ UserDeleteAt: 0,
+ },
+ },
+ startTime: 1000,
+ endTime: 2000,
+ want: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ assert.Equalf(t, tt.want, shared.ChannelHasActivity(tt.cmhs, tt.startTime, tt.endTime), "channelHasActivity(%v, %v, %v)", tt.cmhs, tt.startTime, tt.endTime)
+ })
+ }
+}
diff --git a/server/enterprise/message_export/actiance_export/main_test.go b/server/enterprise/message_export/actiance_export/main_test.go
new file mode 100644
index 0000000000..4275303b85
--- /dev/null
+++ b/server/enterprise/message_export/actiance_export/main_test.go
@@ -0,0 +1,21 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package actiance_export
+
+import (
+ "testing"
+
+ "github.com/mattermost/mattermost/server/v8/channels/api4"
+ "github.com/mattermost/mattermost/server/v8/channels/testlib"
+)
+
+var mainHelper *testlib.MainHelper
+
+func TestMain(m *testing.M) {
+ mainHelper = testlib.NewMainHelper()
+ defer mainHelper.Close()
+ api4.SetMainHelper(mainHelper)
+
+ mainHelper.Main(m)
+}
diff --git a/server/enterprise/message_export/actiance_export/test_helpers.go b/server/enterprise/message_export/actiance_export/test_helpers.go
new file mode 100644
index 0000000000..79e28a5957
--- /dev/null
+++ b/server/enterprise/message_export/actiance_export/test_helpers.go
@@ -0,0 +1,49 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package actiance_export
+
+import (
+ "encoding/xml"
+ "io"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+type ChannelExportWithSpecifics struct {
+ XMLName xml.Name `xml:"Conversation"`
+ Perspective string `xml:"Perspective,attr"`
+ ChannelId string `xml:"-"`
+ RoomId string `xml:"RoomID"`
+ StartTime int64 `xml:"StartTimeUTC"`
+ JoinEvents []*JoinExport `xml:"ParticipantEntered"`
+ Messages []*PostExport `xml:"Message"`
+ FileStarts []*FileUploadStartExport `xml:"FileTransferStarted"`
+ FileStops []*FileUploadStopExport `xml:"FileTransferEnded"`
+ LeaveEvents []*LeaveExport `xml:"ParticipantLeft"`
+ EndTime int64 `xml:"EndTimeUTC"`
+}
+
+func GetChannelExports(t *testing.T, r io.Reader) []*ChannelExportWithSpecifics {
+ decoder := xml.NewDecoder(r)
+ var exportedChannels []*ChannelExportWithSpecifics
+ for {
+ token, err := decoder.Token()
+ if token == nil || err != nil {
+ break
+ }
+ switch se := token.(type) {
+ case xml.StartElement:
+ if se.Name.Local == "Conversation" {
+ var a *ChannelExportWithSpecifics
+ err = decoder.DecodeElement(&a, &se)
+ require.NoError(t, err)
+ exportedChannels = append(exportedChannels, a)
+ }
+ default:
+ }
+ }
+
+ return exportedChannels
+}
diff --git a/server/enterprise/message_export/common_export/common_export.go b/server/enterprise/message_export/common_export/common_export.go
index 6c5c882509..fa31e1d739 100644
--- a/server/enterprise/message_export/common_export/common_export.go
+++ b/server/enterprise/message_export/common_export/common_export.go
@@ -5,8 +5,13 @@ package common_export
import (
"fmt"
+ "strconv"
+
+ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
"github.com/mattermost/mattermost/server/public/model"
+ "github.com/mattermost/mattermost/server/public/shared/mlog"
+ "github.com/mattermost/mattermost/server/public/shared/request"
)
const MissingFileMessage = "File missing for post; cannot copy file to archive"
@@ -33,8 +38,6 @@ type ChannelMember struct {
Email string
Username string
}
-type ChannelMembers map[string]ChannelMember
-type MembersByChannel map[string]ChannelMembers
type MetadataChannel struct {
TeamId *string
@@ -52,7 +55,7 @@ type MetadataChannel struct {
}
type Metadata struct {
- Channels map[string]MetadataChannel
+ Channels map[string]*MetadataChannel
MessagesCount int
AttachmentsCount int
StartTime int64
@@ -62,7 +65,7 @@ type Metadata struct {
func (metadata *Metadata) Update(post *model.MessageExport, attachments int) {
channelMetadata, ok := metadata.Channels[*post.ChannelId]
if !ok {
- channelMetadata = MetadataChannel{
+ channelMetadata = &MetadataChannel{
TeamId: post.TeamId,
TeamName: post.TeamName,
TeamDisplayName: post.TeamDisplayName,
@@ -89,11 +92,93 @@ func (metadata *Metadata) Update(post *model.MessageExport, attachments int) {
metadata.Channels[*post.ChannelId] = channelMetadata
}
-func GetJoinsAndLeavesForChannel(startTime int64, endTime int64, channelMembersHistory []*model.ChannelMemberHistoryResult, channelMembers ChannelMembers) ([]ChannelMemberJoin, []ChannelMemberLeave) {
- joins := []ChannelMemberJoin{}
- leaves := []ChannelMemberLeave{}
+type ChannelExportsParams struct {
+ Store shared.MessageExportStore
+ ExportPeriodStartTime int64
+ ExportPeriodEndTime int64
+ ChannelBatchSize int
+ ChannelHistoryBatchSize int
+ ReportProgressMessage func(message string)
+}
- alreadyJoined := map[string]bool{}
+// CalculateChannelExports returns the channel info ( map[channelId]*MetadataChannel ) and the channel user
+// joins/leaves ( map[channelId][]*model.ChannelMemberHistoryResult ) for any channel that has had activity
+// (posts or user join/leaves) between ExportPeriodStartTime and ExportPeriodEndTime.
+func CalculateChannelExports(rctx request.CTX, opt ChannelExportsParams) (map[string]*MetadataChannel, map[string][]*model.ChannelMemberHistoryResult, error) {
+ // Which channels had user activity in the export period?
+ activeChannelIds, err := opt.Store.ChannelMemberHistory().GetChannelsWithActivityDuring(opt.ExportPeriodStartTime, opt.ExportPeriodEndTime)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ if len(activeChannelIds) == 0 {
+ return nil, nil, nil
+ }
+
+ rctx.Logger().Debug("Started CalculateChannelExports", mlog.Int("export_period_start_time", opt.ExportPeriodStartTime), mlog.Int("export_period_end_time", opt.ExportPeriodEndTime), mlog.Int("num_active_channel_ids", len(activeChannelIds)))
+ message := rctx.T("ent.message_export.actiance_export.calculate_channel_exports.channel_message", model.StringMap{"NumChannels": strconv.Itoa(len(activeChannelIds))})
+ opt.ReportProgressMessage(message)
+
+ // For each channel, get its metadata.
+ channelMetadata := make(map[string]*MetadataChannel, len(activeChannelIds))
+
+ // Use batches to reduce db load and network waste.
+ for pos := 0; pos < len(activeChannelIds); pos += opt.ChannelBatchSize {
+ upTo := min(pos+opt.ChannelBatchSize, len(activeChannelIds))
+ batch := activeChannelIds[pos:upTo]
+ channels, err := opt.Store.Channel().GetMany(batch, true)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ for _, channel := range channels {
+ channelMetadata[channel.Id] = &MetadataChannel{
+ TeamId: model.NewPointer(channel.TeamId),
+ ChannelId: channel.Id,
+ ChannelName: channel.Name,
+ ChannelDisplayName: channel.DisplayName,
+ ChannelType: channel.Type,
+ RoomId: fmt.Sprintf("%v - %v", ChannelTypeDisplayName(channel.Type), channel.Id),
+ StartTime: opt.ExportPeriodStartTime,
+ EndTime: opt.ExportPeriodEndTime,
+ }
+ }
+ }
+
+ historiesByChannelId := make(map[string][]*model.ChannelMemberHistoryResult, len(activeChannelIds))
+
+ // Now that we have metadata, get channelMemberHistories for each channel.
+ // Use batches to reduce total db load and network waste.
+ for pos := 0; pos < len(activeChannelIds); pos += opt.ChannelHistoryBatchSize {
+ // This may take a while, so update the system console UI.
+ message := rctx.T("ent.message_export.actiance_export.calculate_channel_exports.activity_message", model.StringMap{
+ "NumChannels": strconv.Itoa(len(activeChannelIds)),
+ "NumCompleted": strconv.Itoa(pos),
+ })
+ opt.ReportProgressMessage(message)
+
+ upTo := min(pos+opt.ChannelHistoryBatchSize, len(activeChannelIds))
+ batch := activeChannelIds[pos:upTo]
+ channelMemberHistories, err := opt.Store.ChannelMemberHistory().GetUsersInChannelDuring(opt.ExportPeriodStartTime, opt.ExportPeriodEndTime, batch)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ // collect the channelMemberHistories by channelId
+ for _, entry := range channelMemberHistories {
+ historiesByChannelId[entry.ChannelId] = append(historiesByChannelId[entry.ChannelId], entry)
+ }
+ }
+
+ return channelMetadata, historiesByChannelId, nil
+}
+
+func GetJoinsAndLeavesForChannel(startTime int64, endTime int64, channelMembersHistory []*model.ChannelMemberHistoryResult,
+ postAuthors map[string]ChannelMember) ([]ChannelMemberJoin, []ChannelMemberLeave) {
+ var joins []ChannelMemberJoin
+ var leaves []ChannelMemberLeave
+
+ alreadyJoined := make(map[string]bool)
for _, cmh := range channelMembersHistory {
if cmh.UserDeleteAt > 0 && cmh.UserDeleteAt < startTime {
continue
@@ -129,7 +214,7 @@ func GetJoinsAndLeavesForChannel(startTime int64, endTime int64, channelMembersH
}
}
- for _, member := range channelMembers {
+ for _, member := range postAuthors {
if alreadyJoined[member.UserId] {
continue
}
diff --git a/server/enterprise/message_export/common_export/common_export_test.go b/server/enterprise/message_export/common_export/common_export_test.go
index a8fc364694..0e6717618e 100644
--- a/server/enterprise/message_export/common_export/common_export_test.go
+++ b/server/enterprise/message_export/common_export/common_export_test.go
@@ -13,7 +13,7 @@ import (
func TestUpdateMetadata(t *testing.T) {
metadata := Metadata{
- Channels: map[string]MetadataChannel{},
+ Channels: map[string]*MetadataChannel{},
MessagesCount: 0,
AttachmentsCount: 0,
StartTime: 0,
@@ -92,7 +92,7 @@ func TestGetJoinsAndLeavesForChannel(t *testing.T) {
name string
channel MetadataChannel
membersHistory []*model.ChannelMemberHistoryResult
- usersInPosts ChannelMembers
+ usersInPosts map[string]ChannelMember
expectedJoins int
expectedLeaves int
}{
@@ -160,9 +160,9 @@ func TestGetJoinsAndLeavesForChannel(t *testing.T) {
name: "implicit-joins",
channel: channel,
membersHistory: nil,
- usersInPosts: ChannelMembers{
- "test1": ChannelMember{UserId: "test1", Email: "test1", Username: "test1"},
- "test2": ChannelMember{UserId: "test2", Email: "test2", Username: "test2"},
+ usersInPosts: map[string]ChannelMember{
+ "test1": {UserId: "test1", Email: "test1", Username: "test1"},
+ "test2": {UserId: "test2", Email: "test2", Username: "test2"},
},
expectedJoins: 2,
expectedLeaves: 0,
@@ -174,9 +174,9 @@ func TestGetJoinsAndLeavesForChannel(t *testing.T) {
{JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test1", UserEmail: "test1", Username: "test1"},
{JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test3", UserEmail: "test3", Username: "test3"},
},
- usersInPosts: ChannelMembers{
- "test1": ChannelMember{UserId: "test1", Email: "test1", Username: "test1"},
- "test2": ChannelMember{UserId: "test2", Email: "test2", Username: "test2"},
+ usersInPosts: map[string]ChannelMember{
+ "test1": {UserId: "test1", Email: "test1", Username: "test1"},
+ "test2": {UserId: "test2", Email: "test2", Username: "test2"},
},
expectedJoins: 3,
expectedLeaves: 2,
diff --git a/server/enterprise/message_export/csv_export/csv_export.go b/server/enterprise/message_export/csv_export/csv_export.go
index 1a319c1a84..74b38b2b55 100644
--- a/server/enterprise/message_export/csv_export/csv_export.go
+++ b/server/enterprise/message_export/csv_export/csv_export.go
@@ -9,18 +9,18 @@ import (
"encoding/json"
"fmt"
"io"
- "net/http"
"os"
"path"
- "sort"
+ "slices"
"strconv"
+ "strings"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/common_export"
+ "github.com/mattermost/mattermost/server/v8/enterprise/internal/file"
+ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
- "github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
)
@@ -28,26 +28,87 @@ const (
EnterPostType = "enter"
LeavePostType = "leave"
PreviouslyJoinedPostType = "previously-joined"
- CSVExportFilename = "csv_export.zip"
CSVWarningFilename = "warning.txt"
)
-func CsvExport(rctx request.CTX, posts []*model.MessageExport, db store.Store, exportBackend filestore.FileBackend, fileAttachmentBackend filestore.FileBackend, exportDirectory string) (warningCount int64, appErr *model.AppError) {
- dest, err := os.CreateTemp("", CSVExportFilename)
+type Row struct {
+ CreateAt int64
+ UpdateAt int64
+ UpdateType shared.PostUpdatedType
+ TeamId string
+ TeamName string
+ TeamDisplayName string
+ ChannelId string
+ ChannelName string
+ ChannelDisplayName string
+ ChannelType string
+ UserId string
+ UserEmail string
+ Username string
+ PostId string
+ EditedByPostId string
+ RepliedToPostId string
+ PostMessage string
+ PostType string
+ UserType string
+ PreviewsPostId string
+}
+
+func CsvExport(rctx request.CTX, p shared.ExportParams) (shared.RunExportResults, error) {
+ // Build the channel exports for the channels that had post or user join/leave activity this batch.
+ exportData, err := shared.GetGenericExportData(p)
+ results := exportData.Results
if err != nil {
- return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.file.creation.appError", nil, "", 0).Wrap(err)
+ return results, err
}
- defer os.Remove(dest.Name())
- zipFile := zip.NewWriter(dest)
+ totalRows := results.CreatedPosts + results.EditedOrigMsgPosts + results.DeletedPosts + results.EditedNewMsgPosts +
+ results.UpdatedPosts + results.UploadedFiles + results.DeletedFiles + results.Joins
+ rows := make([]Row, 0, totalRows)
+ for _, channel := range exportData.Exports {
+ rows = append(rows, getJoinLeavePosts(channel)...)
+ for _, p := range channel.Posts {
+ rows = append(rows, postToRow(p, "message", p.PostCreateAt, p.Message))
+ }
+ for _, u := range channel.UploadStarts {
+ rows = append(rows, attachmentToRow(shared.UploadStartToExportEntry(u)))
+ }
+ for _, d := range channel.DeletedFiles {
+ rows = append(rows, attachmentToRow(d))
+ }
+ }
+
+ // We need to sort all the elements by (CreateAt, PostId) because they were added by type and by channel above.
+ slices.SortStableFunc(rows, func(a, b Row) int {
+ if a.CreateAt == b.CreateAt {
+ return strings.Compare(a.PostId, b.PostId)
+ }
+ return int(a.CreateAt - b.CreateAt)
+ })
+
+ // We've got the data, now its write time:
+
+ // Write this batch to a tmp zip, then copy the zip to the export directory.
+ // Using a 2M buffer because the file backend may be s3 and this optimizes speed and
+ // memory usage, see: https://github.com/mattermost/mattermost/pull/26629
+ buf := make([]byte, 1024*1024*2)
+ temp, err := os.CreateTemp("", "compliance-export-batch-*.zip")
+ if err != nil {
+ return results, fmt.Errorf("unable to create temporary CSV export file: %w", err)
+ }
+ defer file.DeleteTemp(rctx.Logger(), temp)
+
+ zipFile := zip.NewWriter(temp)
csvFile, err := zipFile.Create("posts.csv")
if err != nil {
- return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.zip.creation.appError", nil, "", 0).Wrap(err)
+ return results, fmt.Errorf("unable to create the zip export file: %w", err)
}
csvWriter := csv.NewWriter(csvFile)
err = csvWriter.Write([]string{
"Post Creation Time",
+ "Post Update Time",
+ "Post Update Type",
"Team Id",
"Team Name",
"Team Display Name",
@@ -68,343 +129,229 @@ func CsvExport(rctx request.CTX, posts []*model.MessageExport, db store.Store, e
})
if err != nil {
- return warningCount, model.NewAppError("CsvExportPost", "ent.compliance.csv.header.export.appError", nil, "", 0).Wrap(err)
+ return results, fmt.Errorf("unable to add header to the CSV export: %w", err)
}
- metadata := common_export.Metadata{
- Channels: map[string]common_export.MetadataChannel{},
- MessagesCount: 0,
- AttachmentsCount: 0,
- StartTime: 0,
- EndTime: 0,
- }
-
- membersByChannel := make(common_export.MembersByChannel)
-
- for _, post := range posts {
- attachments, err := getPostAttachments(db, post)
- if err != nil {
- return warningCount, err
- }
-
- if _, ok := membersByChannel[*post.ChannelId]; !ok {
- membersByChannel[*post.ChannelId] = common_export.ChannelMembers{}
- }
-
- membersByChannel[*post.ChannelId][*post.UserId] = common_export.ChannelMember{
- UserId: *post.UserId,
- Username: *post.Username,
- IsBot: post.IsBot,
- Email: *post.UserEmail,
- }
-
- metadata.Update(post, len(attachments))
- }
-
- joinLeavePosts, appErr2 := getJoinLeavePosts(metadata.Channels, membersByChannel, db)
- if appErr2 != nil {
- return warningCount, appErr2
- }
-
- postsGenerator := mergePosts(joinLeavePosts, posts)
-
- for post := postsGenerator(); post != nil; post = postsGenerator() {
- if err = csvWriter.Write(postToRow(post, post.PostCreateAt, *post.PostMessage)); err != nil {
- return warningCount, model.NewAppError("CsvExportPost", "ent.compliance.csv.post.export.appError", nil, "", 0).Wrap(err)
- }
-
- if post.PostDeleteAt != nil && *post.PostDeleteAt > 0 && post.PostProps != nil {
- props := map[string]any{}
- if json.Unmarshal([]byte(*post.PostProps), &props) == nil {
- if _, ok := props[model.PostPropsDeleteBy]; ok {
- if err = csvWriter.Write(postToRow(post, post.PostDeleteAt, "delete "+*post.PostMessage)); err != nil {
- return warningCount, model.NewAppError("CsvExportPost", "ent.compliance.csv.post.export.appError", nil, "", 0).Wrap(err)
- }
- }
- }
- }
-
- var attachments []*model.FileInfo
- attachments, appErr = getPostAttachments(db, post)
- if appErr != nil {
- return warningCount, appErr
- }
-
- for _, attachment := range attachments {
- if err = csvWriter.Write(attachmentToRow(post, attachment)); err != nil {
- return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.attachment.export.appError", nil, "", 0).Wrap(err)
- }
+ for _, row := range rows {
+ if err = csvWriter.Write(rowToStringSlice(row)); err != nil {
+ return results, fmt.Errorf("unable to export a row: %w", err)
}
}
csvWriter.Flush()
var missingFiles []string
- for _, post := range posts {
- attachments, err := getPostAttachments(db, post)
+ for _, post := range p.Posts {
+ var attachments []*model.FileInfo
+ attachments, err = shared.GetPostAttachments(p.Db, post)
if err != nil {
- return warningCount, err
+ return results, err
}
for _, attachment := range attachments {
- var attachmentSrc io.ReadCloser
- attachmentSrc, nErr := fileAttachmentBackend.Reader(attachment.Path)
+ var r io.ReadCloser
+ r, nErr := p.FileAttachmentBackend.Reader(attachment.Path)
if nErr != nil {
- missingFiles = append(missingFiles, "Warning:"+common_export.MissingFileMessage+" - Post: "+*post.PostId+" - "+attachment.Path)
- rctx.Logger().Warn(common_export.MissingFileMessage, mlog.String("PostId", *post.PostId), mlog.String("FileName", attachment.Path))
+ missingFiles = append(missingFiles, "Warning:"+shared.MissingFileMessage+" - Post: "+*post.PostId+" - "+attachment.Path)
+ rctx.Logger().Warn(shared.MissingFileMessage, mlog.String("post_id", *post.PostId), mlog.String("filename", attachment.Path))
continue
}
- defer attachmentSrc.Close()
- attachmentDst, err := zipFile.Create(path.Join("files", *post.PostId, fmt.Sprintf("%s-%s", attachment.Id, path.Base(attachment.Path))))
- if err != nil {
- return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.attachment.copy.appError", nil, "", 0).Wrap(err)
- }
+ // Probably don't need to be this careful (see actiance_export.go), but may as well be consistent.
+ if err = func() error {
+ defer r.Close()
+ var attachmentDst io.Writer
+ attachmentDst, err = zipFile.Create(path.Join("files", *post.PostId, fmt.Sprintf("%s-%s", attachment.Id, path.Base(attachment.Path))))
+ if err != nil {
+ return err
+ }
- _, err = io.Copy(attachmentDst, attachmentSrc)
- if err != nil {
- return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.attachment.copy.appError", nil, "", 0).Wrap(err)
+ _, err = io.CopyBuffer(attachmentDst, r, buf)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ }(); err != nil {
+ return results, fmt.Errorf("unable to copy the attachment into the zip file: %w", err)
}
}
}
- warningCount = int64(len(missingFiles))
- if warningCount > 0 {
+ results.NumWarnings = len(missingFiles)
+ if results.NumWarnings > 0 {
metadataFile, _ := zipFile.Create(CSVWarningFilename)
for _, value := range missingFiles {
_, err = metadataFile.Write([]byte(value + "\n"))
if err != nil {
- appErr = model.NewAppError("CsvExport", "ent.compliance.csv.warning.appError", nil, "", 0).Wrap(err)
- return warningCount, appErr
+ return results, fmt.Errorf("unable to create the warning file: %w", err)
}
}
}
metadataFile, err := zipFile.Create("metadata.json")
if err != nil {
- return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.metadata.json.zipfile.appError", nil, "", 0).Wrap(err)
+ return results, fmt.Errorf("unable to create the zip file: %w", err)
}
- data, err := json.MarshalIndent(metadata, "", " ")
+ data, err := json.MarshalIndent(exportData.Metadata, "", " ")
if err != nil {
- return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.metadata.json.marshalling.appError", nil, "", 0).Wrap(err)
+ return results, fmt.Errorf("unable to convert metadata to json: %w", err)
}
_, err = metadataFile.Write(data)
if err != nil {
- return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.metadata.export.appError", nil, "", 0).Wrap(err)
+ return results, fmt.Errorf("unable to add metadata file to the zip file: %w", err)
}
err = zipFile.Close()
if err != nil {
- return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.metadata.close.appError", nil, "", 0).Wrap(err)
+ return results, fmt.Errorf("unable to close the zip file: %w", err)
}
- _, err = dest.Seek(0, 0)
+ _, err = temp.Seek(0, 0)
if err != nil {
- return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.seek.appError", nil, "", 0).Wrap(err)
+ return results, fmt.Errorf("unable to seek to start of export file: %w", err)
}
+
// Try to write the file without a timeout due to the potential size of the file.
- _, err = filestore.TryWriteFileContext(rctx.Context(), exportBackend, dest, path.Join(exportDirectory, CSVExportFilename))
+ _, err = filestore.TryWriteFileContext(rctx.Context(), p.ExportBackend, temp, p.BatchPath)
if err != nil {
- return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.write_file.appError", nil, "", http.StatusInternalServerError).Wrap(err)
+ return results, fmt.Errorf("unable to write the csv file: %w", err)
}
- return warningCount, appErr
+ return results, nil
}
-func mergePosts(left []*model.MessageExport, right []*model.MessageExport) func() *model.MessageExport {
- leftCursor := 0
- rightCursor := 0
- return func() *model.MessageExport {
- if leftCursor >= len(left) && rightCursor >= len(right) {
- return nil
+func getJoinLeavePosts(channel shared.ChannelExport) []Row {
+ var joinLeavePosts []Row
+
+ for _, join := range channel.JoinEvents {
+ enterMessage := fmt.Sprintf("User %s (%s) joined the channel", join.Username, join.UserEmail)
+ enterPostType := EnterPostType
+ if join.JoinTime <= channel.StartTime {
+ enterPostType = PreviouslyJoinedPostType
+ enterMessage = fmt.Sprintf("User %s (%s) was already in the channel", join.Username, join.UserEmail)
}
-
- if leftCursor >= len(left) {
- rightCursor += 1
- return right[rightCursor-1]
- }
-
- if rightCursor >= len(right) {
- leftCursor += 1
- return left[leftCursor-1]
- }
-
- if *left[leftCursor].PostCreateAt <= *right[rightCursor].PostCreateAt {
- leftCursor += 1
- return left[leftCursor-1]
- }
-
- rightCursor += 1
- return right[rightCursor-1]
- }
-}
-
-func getJoinLeavePosts(channels map[string]common_export.MetadataChannel, membersByChannel common_export.MembersByChannel, db store.Store) ([]*model.MessageExport, *model.AppError) {
- joinLeavePosts := []*model.MessageExport{}
- for _, channel := range channels {
- channelMembersHistory, err := db.ChannelMemberHistory().GetUsersInChannelDuring(channel.StartTime, channel.EndTime, channel.ChannelId)
- if err != nil {
- return nil, model.NewAppError("getJoinLeavePosts", "ent.get_users_in_channel_during", nil, "", http.StatusInternalServerError).Wrap(err)
- }
-
- joins, leaves := common_export.GetJoinsAndLeavesForChannel(channel.StartTime, channel.EndTime, channelMembersHistory, membersByChannel[channel.ChannelId])
-
- for _, join := range joins {
- enterMessage := fmt.Sprintf("User %s (%s) joined the channel", join.Username, join.Email)
- enterPostType := EnterPostType
- createAt := model.NewPointer(join.Datetime)
- channelCopy := channel
- if join.Datetime <= channel.StartTime {
- enterPostType = PreviouslyJoinedPostType
- enterMessage = fmt.Sprintf("User %s (%s) was already in the channel", join.Username, join.Email)
- createAt = model.NewPointer(channel.StartTime)
- }
- joinLeavePosts = append(
- joinLeavePosts,
- &model.MessageExport{
- TeamId: channel.TeamId,
- TeamName: channel.TeamName,
- TeamDisplayName: channel.TeamDisplayName,
-
- ChannelId: &channelCopy.ChannelId,
- ChannelName: &channelCopy.ChannelName,
- ChannelDisplayName: &channelCopy.ChannelDisplayName,
- ChannelType: &channelCopy.ChannelType,
-
- UserId: model.NewPointer(join.UserId),
- UserEmail: model.NewPointer(join.Email),
- Username: model.NewPointer(join.Username),
- IsBot: join.IsBot,
-
- PostId: model.NewPointer(""),
- PostCreateAt: createAt,
- PostMessage: &enterMessage,
- PostType: &enterPostType,
- PostOriginalId: model.NewPointer(""),
- PostFileIds: []string{},
+ joinLeavePosts = append(
+ joinLeavePosts,
+ postToRow(shared.PostExport{
+ MessageExport: model.MessageExport{
+ TeamId: &channel.TeamId,
+ TeamName: &channel.TeamName,
+ TeamDisplayName: &channel.TeamDisplayName,
+ ChannelId: &channel.ChannelId,
+ ChannelName: &channel.ChannelName,
+ ChannelDisplayName: &channel.DisplayName,
+ ChannelType: &channel.ChannelType,
+ UserId: &join.UserId,
+ UserEmail: &join.UserEmail,
+ Username: &join.Username,
+ IsBot: join.UserType == shared.Bot,
+ PostId: model.NewPointer(""),
+ PostCreateAt: &join.JoinTime,
+ PostMessage: &enterMessage,
+ PostType: &enterPostType,
+ PostOriginalId: model.NewPointer(""),
+ PostFileIds: []string{},
},
- )
+ }, enterPostType, &join.JoinTime, enterMessage),
+ )
+ }
+ for _, leave := range channel.LeaveEvents {
+ if leave.ClosedOut {
+ // csv does not record closed-out leaves; see export_data.go for further explanation.
+ continue
}
- for _, leave := range leaves {
- leaveMessage := fmt.Sprintf("User %s (%s) leaved the channel", leave.Username, leave.Email)
- leavePostType := LeavePostType
- channelCopy := channel
+ leaveMessage := fmt.Sprintf("User %s (%s) left the channel", leave.Username, leave.UserEmail)
+ leavePostType := LeavePostType
- joinLeavePosts = append(
- joinLeavePosts,
- &model.MessageExport{
- TeamId: channel.TeamId,
- TeamName: channel.TeamName,
- TeamDisplayName: channel.TeamDisplayName,
-
- ChannelId: &channelCopy.ChannelId,
- ChannelName: &channelCopy.ChannelName,
- ChannelDisplayName: &channelCopy.ChannelDisplayName,
- ChannelType: &channelCopy.ChannelType,
-
- UserId: model.NewPointer(leave.UserId),
- UserEmail: model.NewPointer(leave.Email),
- Username: model.NewPointer(leave.Username),
- IsBot: leave.IsBot,
-
- PostId: model.NewPointer(""),
- PostCreateAt: model.NewPointer(leave.Datetime),
- PostMessage: &leaveMessage,
- PostType: &leavePostType,
- PostOriginalId: model.NewPointer(""),
- PostFileIds: []string{},
+ joinLeavePosts = append(
+ joinLeavePosts,
+ postToRow(shared.PostExport{
+ MessageExport: model.MessageExport{
+ TeamId: &channel.TeamId,
+ TeamName: &channel.TeamName,
+ TeamDisplayName: &channel.TeamDisplayName,
+ ChannelId: &channel.ChannelId,
+ ChannelName: &channel.ChannelName,
+ ChannelDisplayName: &channel.DisplayName,
+ ChannelType: &channel.ChannelType,
+ UserId: &leave.UserId,
+ UserEmail: &leave.UserEmail,
+ Username: &leave.Username,
+ IsBot: leave.UserType == shared.Bot,
+ PostId: model.NewPointer(""),
+ PostCreateAt: &leave.LeaveTime,
+ PostMessage: &leaveMessage,
+ PostType: &leavePostType,
+ PostOriginalId: model.NewPointer(""),
+ PostFileIds: []string{},
},
- )
- }
+ }, leavePostType, &leave.LeaveTime, leaveMessage),
+ )
}
- sort.Slice(joinLeavePosts, func(i, j int) bool {
- return *joinLeavePosts[i].PostCreateAt < *joinLeavePosts[j].PostCreateAt
- })
- return joinLeavePosts, nil
+ return joinLeavePosts
}
-func getPostAttachments(db store.Store, post *model.MessageExport) ([]*model.FileInfo, *model.AppError) {
- // if the post included any files, we need to add special elements to the export.
- if len(post.PostFileIds) == 0 {
- return []*model.FileInfo{}, nil
- }
-
- attachments, err := db.FileInfo().GetForPost(*post.PostId, true, true, false)
- if err != nil {
- return nil, model.NewAppError("getPostAttachments", "ent.message_export.csv_export.get_attachment_error", nil, "", http.StatusInternalServerError).Wrap(err)
- }
- return attachments, nil
-}
-
-func postToRow(post *model.MessageExport, createTime *int64, message string) []string {
- teamId := ""
- teamName := ""
- teamDisplayName := ""
- if post.TeamId != nil {
- teamId = *post.TeamId
- }
- if post.TeamName != nil {
- teamName = *post.TeamName
- }
- if post.TeamDisplayName != nil {
- teamDisplayName = *post.TeamDisplayName
- }
- postType := "message"
- if post.PostType != nil && *post.PostType != "" {
- postType = *post.PostType
- }
- postRootId := ""
- if post.PostRootId != nil {
- postRootId = *post.PostRootId
- }
+func postToRow(p shared.PostExport, postType string, createTime *int64, message string) Row {
userType := "user"
- if post.IsBot {
+ if p.IsBot {
userType = "bot"
}
+ return Row{
+ CreateAt: model.SafeDereference(createTime),
+ UpdateAt: model.SafeDereference(p.PostUpdateAt),
+ UpdateType: p.UpdatedType,
+ TeamId: model.SafeDereference(p.TeamId),
+ TeamName: model.SafeDereference(p.TeamName),
+ TeamDisplayName: model.SafeDereference(p.TeamDisplayName),
+ ChannelId: model.SafeDereference(p.ChannelId),
+ ChannelName: model.SafeDereference(p.ChannelName),
+ ChannelDisplayName: model.SafeDereference(p.ChannelDisplayName),
+ ChannelType: shared.ChannelTypeDisplayName(model.SafeDereference(p.ChannelType)),
+ UserId: model.SafeDereference(p.UserId),
+ UserEmail: model.SafeDereference(p.UserEmail),
+ Username: model.SafeDereference(p.Username),
+ PostId: model.SafeDereference(p.PostId),
+ EditedByPostId: p.EditedNewMsgId,
+ RepliedToPostId: model.SafeDereference(p.PostRootId),
+ PostMessage: message,
+ PostType: postType,
+ UserType: userType,
+ PreviewsPostId: p.PreviewID(),
+ }
+}
+func rowToStringSlice(r Row) []string {
return []string{
- strconv.FormatInt(*createTime, 10),
- teamId,
- teamName,
- teamDisplayName,
- *post.ChannelId,
- *post.ChannelName,
- *post.ChannelDisplayName,
- common_export.ChannelTypeDisplayName(*post.ChannelType),
- *post.UserId,
- *post.UserEmail,
- *post.Username,
- *post.PostId,
- *post.PostOriginalId,
- postRootId,
- message,
- postType,
- userType,
- post.PreviewID(),
+ strconv.FormatInt(r.CreateAt, 10),
+ strconv.FormatInt(r.UpdateAt, 10),
+ string(r.UpdateType),
+ r.TeamId,
+ r.TeamName,
+ r.TeamDisplayName,
+ r.ChannelId,
+ r.ChannelName,
+ r.ChannelDisplayName,
+ r.ChannelType,
+ r.UserId,
+ r.UserEmail,
+ r.Username,
+ r.PostId,
+ r.EditedByPostId,
+ r.RepliedToPostId,
+ r.PostMessage,
+ r.PostType,
+ r.UserType,
+ r.PreviewsPostId,
}
}
-func attachmentToRow(post *model.MessageExport, attachment *model.FileInfo) []string {
- row := postToRow(post, post.PostCreateAt, *post.PostMessage)
-
- attachmentEntry := fmt.Sprintf("%s (files/%s/%s-%s)", attachment.Name, *post.PostId, attachment.Id, path.Base(attachment.Path))
- attachmentMessage := "attachment"
- userType := row[len(row)-2]
-
- if attachment.DeleteAt > 0 && post.PostDeleteAt != nil {
- deleteRow := postToRow(post, post.PostDeleteAt, *post.PostMessage)
- row = append(
- deleteRow[:len(deleteRow)-4],
- attachmentEntry,
- "deleted "+attachmentMessage,
- userType,
- )
- } else {
- row = append(
- row[:len(row)-4],
- attachmentEntry,
- attachmentMessage,
- userType,
- )
+func attachmentToRow(post shared.PostExport) Row {
+ message := strings.TrimSpace(fmt.Sprintf("%s (files/%s/%s-%s)", post.FileInfo.Name, *post.PostId, post.FileInfo.Id, path.Base(post.FileInfo.Path)))
+ postType := "attachment"
+ if post.UpdatedType == shared.FileDeleted {
+ postType = "deleted attachment"
+ post.PostUpdateAt = model.NewPointer(post.FileInfo.DeleteAt)
}
- return row
+
+ return postToRow(post, postType, post.PostCreateAt, message)
}
diff --git a/server/enterprise/message_export/csv_export/csv_export_test.go b/server/enterprise/message_export/csv_export/csv_export_test.go
index 8835925235..3de297f00f 100644
--- a/server/enterprise/message_export/csv_export/csv_export_test.go
+++ b/server/enterprise/message_export/csv_export/csv_export_test.go
@@ -6,8 +6,10 @@ package csv_export
import (
"archive/zip"
"bytes"
+ "fmt"
"io"
"os"
+ "path"
"strings"
"testing"
@@ -15,128 +17,19 @@ import (
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/common_export"
-
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
+ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
)
-func TestMergePosts(t *testing.T) {
- chanTypeDirect := model.ChannelTypeDirect
- // these two posts were made in the same channel
- post1 := &model.MessageExport{
- ChannelId: model.NewPointer("Test"),
- ChannelDisplayName: model.NewPointer("Test"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("Some message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("test"),
- Username: model.NewPointer("test"),
- ChannelType: &chanTypeDirect,
- }
-
- post2 := &model.MessageExport{
- ChannelId: model.NewPointer("Test"),
- ChannelDisplayName: model.NewPointer("Test"),
- PostCreateAt: model.NewPointer(int64(2)),
- PostMessage: model.NewPointer("Some message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("test"),
- Username: model.NewPointer("test"),
- ChannelType: &chanTypeDirect,
- }
-
- post3 := &model.MessageExport{
- ChannelId: model.NewPointer("Test"),
- ChannelDisplayName: model.NewPointer("Test"),
- PostCreateAt: model.NewPointer(int64(3)),
- PostMessage: model.NewPointer("Some message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("test"),
- Username: model.NewPointer("test"),
- ChannelType: &chanTypeDirect,
- }
-
- post4 := &model.MessageExport{
- ChannelId: model.NewPointer("Test"),
- ChannelDisplayName: model.NewPointer("Test"),
- PostCreateAt: model.NewPointer(int64(4)),
- PostMessage: model.NewPointer("Some message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("test"),
- Username: model.NewPointer("test"),
- ChannelType: &chanTypeDirect,
- }
-
- post2other := &model.MessageExport{
- ChannelId: model.NewPointer("Test"),
- ChannelDisplayName: model.NewPointer("Test"),
- PostCreateAt: model.NewPointer(int64(2)),
- PostMessage: model.NewPointer("Some message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("test"),
- Username: model.NewPointer("test"),
- ChannelType: &chanTypeDirect,
- }
-
- var mergetests = []struct {
- name string
- in1 []*model.MessageExport
- in2 []*model.MessageExport
- out []*model.MessageExport
- }{
- {
- "merge all",
- []*model.MessageExport{post1, post2, post3, post4},
- []*model.MessageExport{post1, post2, post3, post4},
- []*model.MessageExport{post1, post1, post2, post2, post3, post3, post4, post4},
- },
- {
- "split and merge 1",
- []*model.MessageExport{post1, post3},
- []*model.MessageExport{post2, post4},
- []*model.MessageExport{post1, post2, post3, post4},
- },
- {
- "split and merge 2",
- []*model.MessageExport{post1, post4},
- []*model.MessageExport{post2, post3},
- []*model.MessageExport{post1, post2, post3, post4},
- },
- {
- "ordered 1",
- []*model.MessageExport{post1, post2},
- []*model.MessageExport{post1, post2other},
- []*model.MessageExport{post1, post1, post2, post2other},
- },
- {
- "ordered 2",
- []*model.MessageExport{post1, post2other},
- []*model.MessageExport{post1, post2},
- []*model.MessageExport{post1, post1, post2other, post2},
- },
- }
-
- for _, tt := range mergetests {
- t.Run(tt.name, func(t *testing.T) {
- next := mergePosts(tt.in1, tt.in2)
- posts := []*model.MessageExport{}
- for post := next(); post != nil; post = next() {
- posts = append(posts, post)
- }
- assert.Equal(t, tt.out, posts)
- })
- }
-}
-
func TestPostToRow(t *testing.T) {
chanTypeDirect := model.ChannelTypeDirect
// these two posts were made in the same channel
- post := &model.MessageExport{
+ post := model.MessageExport{
PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostOriginalId: model.NewPointer(""),
PostRootId: model.NewPointer("post-root-id"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
@@ -145,6 +38,7 @@ func TestPostToRow(t *testing.T) {
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(1)),
PostMessage: model.NewPointer("message"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
@@ -152,14 +46,15 @@ func TestPostToRow(t *testing.T) {
ChannelType: &chanTypeDirect,
}
- post_without_team := &model.MessageExport{
+ post_without_team := model.MessageExport{
PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostOriginalId: model.NewPointer(""),
PostRootId: model.NewPointer("post-root-id"),
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(1)),
PostMessage: model.NewPointer("message"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
@@ -167,9 +62,9 @@ func TestPostToRow(t *testing.T) {
ChannelType: &chanTypeDirect,
}
- post_with_other_type := &model.MessageExport{
+ post_with_other_type := model.MessageExport{
PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostOriginalId: model.NewPointer(""),
PostRootId: model.NewPointer("post-root-id"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
@@ -178,6 +73,7 @@ func TestPostToRow(t *testing.T) {
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(1)),
PostMessage: model.NewPointer("message"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
@@ -186,9 +82,9 @@ func TestPostToRow(t *testing.T) {
PostType: model.NewPointer("other"),
}
- post_with_other_type_bot := &model.MessageExport{
+ post_with_other_type_bot := model.MessageExport{
PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostOriginalId: model.NewPointer(""),
PostRootId: model.NewPointer("post-root-id"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
@@ -197,6 +93,7 @@ func TestPostToRow(t *testing.T) {
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(1)),
PostMessage: model.NewPointer("message"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
@@ -206,9 +103,9 @@ func TestPostToRow(t *testing.T) {
IsBot: true,
}
- post_with_permalink_preview := &model.MessageExport{
+ post_with_permalink_preview := model.MessageExport{
PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostOriginalId: model.NewPointer(""),
PostRootId: model.NewPointer("post-root-id"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
@@ -217,6 +114,7 @@ func TestPostToRow(t *testing.T) {
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(1)),
PostMessage: model.NewPointer("message"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
@@ -226,118 +124,128 @@ func TestPostToRow(t *testing.T) {
}
torowtests := []struct {
name string
- in *model.MessageExport
- out []string
+ in model.MessageExport
+ out Row
}{
{
"simple row",
post,
- []string{"1", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "post-original-id", "post-root-id", "message", "message", "user", ""},
+ Row{1, 1, "", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "", "post-root-id", "message", "message", "user", ""},
},
{
"without team data",
post_without_team,
- []string{"1", "", "", "", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "post-original-id", "post-root-id", "message", "message", "user", ""},
+ Row{1, 1, "", "", "", "", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "", "post-root-id", "message", "message", "user", ""},
},
{
"with special post type",
post_with_other_type,
- []string{"1", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "post-original-id", "post-root-id", "message", "other", "user", ""},
+ Row{1, 1, "", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "", "post-root-id", "message", "other", "user", ""},
},
{
"with special post type from bot",
post_with_other_type_bot,
- []string{"1", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "post-original-id", "post-root-id", "message", "other", "bot", ""},
+ Row{1, 1, "", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "", "post-root-id", "message", "other", "bot", ""},
},
{
"with permalink preview",
post_with_permalink_preview,
- []string{"1", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "post-original-id", "post-root-id", "message", "message", "user", "n4w39mc1ff8y5fite4b8hacy1w"},
+ Row{1, 1, "", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "", "post-root-id", "message", "message", "user", "n4w39mc1ff8y5fite4b8hacy1w"},
},
}
for _, tt := range torowtests {
t.Run(tt.name, func(t *testing.T) {
- assert.Equal(t, tt.out, postToRow(tt.in, tt.in.PostCreateAt, *tt.in.PostMessage))
+ postType := model.SafeDereference(tt.in.PostType)
+ if postType == "" {
+ postType = "message"
+ }
+ in := shared.PostExport{
+ MessageExport: tt.in,
+ }
+ assert.Equal(t, tt.out, postToRow(in, postType, tt.in.PostCreateAt, *tt.in.PostMessage))
})
}
}
func TestAttachmentToRow(t *testing.T) {
chanTypeDirect := model.ChannelTypeDirect
- post := &model.MessageExport{
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
+ post := shared.PostExport{
+ MessageExport: model.MessageExport{
+ PostId: model.NewPointer("post-id"),
+ PostOriginalId: model.NewPointer(""),
+ PostRootId: model.NewPointer("post-root-id"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(1)),
+ PostMessage: model.NewPointer("message"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("user-id"),
+ Username: model.NewPointer("username"),
+ ChannelType: &chanTypeDirect,
+ },
+ FileInfo: &model.FileInfo{
+ Name: "test1",
+ Id: "12345",
+ Path: "filename.txt",
+ },
}
- post_deleted := &model.MessageExport{
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostDeleteAt: model.NewPointer(int64(10)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
+ postDeleted := shared.PostExport{
+ MessageExport: model.MessageExport{
+ PostId: model.NewPointer("post-id"),
+ PostOriginalId: model.NewPointer(""),
+ PostRootId: model.NewPointer("post-root-id"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(10)),
+ PostDeleteAt: model.NewPointer(int64(10)),
+ PostMessage: model.NewPointer("message"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("user-id"),
+ Username: model.NewPointer("username"),
+ ChannelType: &chanTypeDirect,
+ PostProps: model.NewPointer(`{"deleteBy":"user-id"}`),
+ },
+ UpdatedType: shared.FileDeleted,
+ FileInfo: &model.FileInfo{
+ Name: "test2",
+ Id: "12346",
+ Path: "filename.txt",
+ DeleteAt: 10,
+ },
}
- file := &model.FileInfo{
- Name: "test1",
- Id: "12345",
- Path: "filename.txt",
- }
-
- file_deleted := &model.FileInfo{
- Name: "test2",
- Id: "12346",
- Path: "filename.txt",
- DeleteAt: 10,
- }
-
- torowtests := []struct {
- name string
- post *model.MessageExport
- attachment *model.FileInfo
- out []string
+ toRowTests := []struct {
+ name string
+ post shared.PostExport
+ out Row
}{
{
"simple attachment",
post,
- file,
- []string{"1", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "post-original-id", "post-root-id", "test1 (files/post-id/12345-filename.txt)", "attachment", "user"},
+ Row{1, 1, "", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "", "post-root-id", "test1 (files/post-id/12345-filename.txt)", "attachment", "user", ""},
},
{
"simple deleted attachment",
- post_deleted,
- file_deleted,
- []string{"10", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "post-original-id", "post-root-id", "test2 (files/post-id/12346-filename.txt)", "deleted attachment", "user"},
+ postDeleted,
+ Row{1, 10, "FileDeleted", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "", "post-root-id", "test2 (files/post-id/12346-filename.txt)", "deleted attachment", "user", ""},
},
}
- for _, tt := range torowtests {
+ for _, tt := range toRowTests {
t.Run(tt.name, func(t *testing.T) {
- assert.Equal(t, tt.out, attachmentToRow(tt.post, tt.attachment))
+ assert.Equal(t, tt.out, attachmentToRow(tt.post))
})
}
}
@@ -346,7 +254,7 @@ func TestGetPostAttachments(t *testing.T) {
chanTypeDirect := model.ChannelTypeDirect
post := &model.MessageExport{
PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostOriginalId: model.NewPointer(""),
PostRootId: model.NewPointer("post-root-id"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
@@ -366,236 +274,27 @@ func TestGetPostAttachments(t *testing.T) {
mockStore := &storetest.Store{}
defer mockStore.AssertExpectations(t)
- files, appErr := getPostAttachments(mockStore, post)
- assert.Nil(t, appErr)
+ files, err := shared.GetPostAttachments(shared.NewMessageExportStore(mockStore), post)
+ assert.NoError(t, err)
assert.Empty(t, files)
post.PostFileIds = []string{"1", "2"}
mockStore.FileInfoStore.On("GetForPost", *post.PostId, true, true, false).Return([]*model.FileInfo{{Name: "test"}, {Name: "test2"}}, nil)
- files, appErr = getPostAttachments(mockStore, post)
- assert.Nil(t, appErr)
+ files, err = shared.GetPostAttachments(shared.NewMessageExportStore(mockStore), post)
+ assert.NoError(t, err)
assert.Len(t, files, 2)
post.PostId = model.NewPointer("post-id-2")
mockStore.FileInfoStore.On("GetForPost", *post.PostId, true, true, false).Return(nil, model.NewAppError("Test", "test", nil, "", 400))
- files, appErr = getPostAttachments(mockStore, post)
- assert.NotNil(t, appErr)
+ files, err = shared.GetPostAttachments(shared.NewMessageExportStore(mockStore), post)
+ assert.Error(t, err)
assert.Nil(t, files)
}
-func TestGetJoinLeavePosts(t *testing.T) {
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- channels := map[string]common_export.MetadataChannel{"bad-request": {StartTime: 1, EndTime: 2, ChannelId: "bad-request"}}
-
- mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", int64(1), int64(2), "bad-request").Return(nil, model.NewAppError("Test", "test", nil, "", 400))
-
- _, appErr := getJoinLeavePosts(channels, nil, mockStore)
- assert.NotNil(t, appErr)
-
- channels = map[string]common_export.MetadataChannel{
- "good-request-1": {StartTime: 1, EndTime: 7, ChannelId: "good-request-1", TeamId: model.NewPointer("test1"), TeamName: model.NewPointer("test1"), TeamDisplayName: model.NewPointer("test1"), ChannelName: "test1", ChannelDisplayName: "test1", ChannelType: "O"},
- "good-request-2": {StartTime: 2, EndTime: 7, ChannelId: "good-request-2", TeamId: model.NewPointer("test2"), TeamName: model.NewPointer("test2"), TeamDisplayName: model.NewPointer("test2"), ChannelName: "test2", ChannelDisplayName: "test2", ChannelType: "P"},
- }
-
- mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", channels["good-request-1"].StartTime, channels["good-request-1"].EndTime, "good-request-1").Return(
- []*model.ChannelMemberHistoryResult{
- {JoinTime: 1, UserId: "test1", UserEmail: "test1", Username: "test1"},
- {JoinTime: 2, LeaveTime: model.NewPointer(int64(3)), UserId: "test2", UserEmail: "test2", Username: "test2"},
- {JoinTime: 3, UserId: "test3", UserEmail: "test3", Username: "test3"},
- },
- nil,
- )
-
- mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", channels["good-request-2"].StartTime, channels["good-request-2"].EndTime, "good-request-2").Return(
- []*model.ChannelMemberHistoryResult{
- {JoinTime: 4, UserId: "test4", UserEmail: "test4", Username: "test4"},
- {JoinTime: 5, LeaveTime: model.NewPointer(int64(6)), UserId: "test5", UserEmail: "test5", Username: "test5"},
- {JoinTime: 6, UserId: "test6", UserEmail: "test6", Username: "test6"},
- },
- nil,
- )
-
- messages, appErr := getJoinLeavePosts(channels, nil, mockStore)
- assert.Nil(t, appErr)
- assert.Len(t, messages, 8)
- chanTypeOpen := model.ChannelTypeOpen
- chanTypePr := model.ChannelTypePrivate
- assert.Equal(t, &model.MessageExport{
- TeamId: model.NewPointer("test1"),
- TeamName: model.NewPointer("test1"),
- TeamDisplayName: model.NewPointer("test1"),
- ChannelId: model.NewPointer("good-request-1"),
- ChannelName: model.NewPointer("test1"),
- ChannelDisplayName: model.NewPointer("test1"),
- ChannelType: &chanTypeOpen,
- UserId: model.NewPointer("test1"),
- UserEmail: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- IsBot: false,
- PostId: model.NewPointer(""),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("User test1 (test1) was already in the channel"),
- PostType: model.NewPointer("previously-joined"),
- PostRootId: nil,
- PostProps: nil,
- PostOriginalId: model.NewPointer(""),
- PostFileIds: model.StringArray{},
- }, messages[0])
- assert.Equal(t, &model.MessageExport{
- TeamId: model.NewPointer("test1"),
- TeamName: model.NewPointer("test1"),
- TeamDisplayName: model.NewPointer("test1"),
- ChannelId: model.NewPointer("good-request-1"),
- ChannelName: model.NewPointer("test1"),
- ChannelDisplayName: model.NewPointer("test1"),
- ChannelType: &chanTypeOpen,
- UserId: model.NewPointer("test2"),
- UserEmail: model.NewPointer("test2"),
- Username: model.NewPointer("test2"),
- IsBot: false,
- PostId: model.NewPointer(""),
- PostCreateAt: model.NewPointer(int64(2)),
- PostMessage: model.NewPointer("User test2 (test2) joined the channel"),
- PostType: model.NewPointer("enter"),
- PostRootId: nil,
- PostProps: nil,
- PostOriginalId: model.NewPointer(""),
- PostFileIds: model.StringArray{},
- }, messages[1])
- assert.Equal(t, &model.MessageExport{
- TeamId: model.NewPointer("test1"),
- TeamName: model.NewPointer("test1"),
- TeamDisplayName: model.NewPointer("test1"),
- ChannelId: model.NewPointer("good-request-1"),
- ChannelName: model.NewPointer("test1"),
- ChannelDisplayName: model.NewPointer("test1"),
- ChannelType: &chanTypeOpen,
- UserId: model.NewPointer("test3"),
- UserEmail: model.NewPointer("test3"),
- Username: model.NewPointer("test3"),
- IsBot: false,
- PostId: model.NewPointer(""),
- PostCreateAt: model.NewPointer(int64(3)),
- PostMessage: model.NewPointer("User test3 (test3) joined the channel"),
- PostType: model.NewPointer("enter"),
- PostRootId: nil,
- PostProps: nil,
- PostOriginalId: model.NewPointer(""),
- PostFileIds: model.StringArray{},
- }, messages[2])
- assert.Equal(t, &model.MessageExport{
- TeamId: model.NewPointer("test1"),
- TeamName: model.NewPointer("test1"),
- TeamDisplayName: model.NewPointer("test1"),
- ChannelId: model.NewPointer("good-request-1"),
- ChannelName: model.NewPointer("test1"),
- ChannelDisplayName: model.NewPointer("test1"),
- ChannelType: &chanTypeOpen,
- UserId: model.NewPointer("test2"),
- UserEmail: model.NewPointer("test2"),
- Username: model.NewPointer("test2"),
- IsBot: false,
- PostId: model.NewPointer(""),
- PostCreateAt: model.NewPointer(int64(3)),
- PostMessage: model.NewPointer("User test2 (test2) leaved the channel"),
- PostType: model.NewPointer("leave"),
- PostRootId: nil,
- PostProps: nil,
- PostOriginalId: model.NewPointer(""),
- PostFileIds: model.StringArray{},
- }, messages[3])
- assert.Equal(t, &model.MessageExport{
- TeamId: model.NewPointer("test2"),
- TeamName: model.NewPointer("test2"),
- TeamDisplayName: model.NewPointer("test2"),
- ChannelId: model.NewPointer("good-request-2"),
- ChannelName: model.NewPointer("test2"),
- ChannelDisplayName: model.NewPointer("test2"),
- ChannelType: &chanTypePr,
- UserId: model.NewPointer("test4"),
- UserEmail: model.NewPointer("test4"),
- Username: model.NewPointer("test4"),
- IsBot: false,
- PostId: model.NewPointer(""),
- PostCreateAt: model.NewPointer(int64(4)),
- PostMessage: model.NewPointer("User test4 (test4) joined the channel"),
- PostType: model.NewPointer("enter"),
- PostRootId: nil,
- PostProps: nil,
- PostOriginalId: model.NewPointer(""),
- PostFileIds: model.StringArray{},
- }, messages[4])
- assert.Equal(t, &model.MessageExport{
- TeamId: model.NewPointer("test2"),
- TeamName: model.NewPointer("test2"),
- TeamDisplayName: model.NewPointer("test2"),
- ChannelId: model.NewPointer("good-request-2"),
- ChannelName: model.NewPointer("test2"),
- ChannelDisplayName: model.NewPointer("test2"),
- ChannelType: &chanTypePr,
- UserId: model.NewPointer("test5"),
- UserEmail: model.NewPointer("test5"),
- Username: model.NewPointer("test5"),
- IsBot: false,
- PostId: model.NewPointer(""),
- PostCreateAt: model.NewPointer(int64(5)),
- PostMessage: model.NewPointer("User test5 (test5) joined the channel"),
- PostType: model.NewPointer("enter"),
- PostRootId: nil,
- PostProps: nil,
- PostOriginalId: model.NewPointer(""),
- PostFileIds: model.StringArray{},
- }, messages[5])
- assert.Equal(t, &model.MessageExport{
- TeamId: model.NewPointer("test2"),
- TeamName: model.NewPointer("test2"),
- TeamDisplayName: model.NewPointer("test2"),
- ChannelId: model.NewPointer("good-request-2"),
- ChannelName: model.NewPointer("test2"),
- ChannelDisplayName: model.NewPointer("test2"),
- ChannelType: &chanTypePr,
- UserId: model.NewPointer("test6"),
- UserEmail: model.NewPointer("test6"),
- Username: model.NewPointer("test6"),
- IsBot: false,
- PostId: model.NewPointer(""),
- PostCreateAt: model.NewPointer(int64(6)),
- PostMessage: model.NewPointer("User test6 (test6) joined the channel"),
- PostType: model.NewPointer("enter"),
- PostRootId: nil,
- PostProps: nil,
- PostOriginalId: model.NewPointer(""),
- PostFileIds: model.StringArray{},
- }, messages[6])
- assert.Equal(t, &model.MessageExport{
- TeamId: model.NewPointer("test2"),
- TeamName: model.NewPointer("test2"),
- TeamDisplayName: model.NewPointer("test2"),
- ChannelId: model.NewPointer("good-request-2"),
- ChannelName: model.NewPointer("test2"),
- ChannelDisplayName: model.NewPointer("test2"),
- ChannelType: &chanTypePr,
- UserId: model.NewPointer("test5"),
- UserEmail: model.NewPointer("test5"),
- Username: model.NewPointer("test5"),
- IsBot: false,
- PostId: model.NewPointer(""),
- PostCreateAt: model.NewPointer(int64(6)),
- PostMessage: model.NewPointer("User test5 (test5) leaved the channel"),
- PostType: model.NewPointer("leave"),
- PostRootId: nil,
- PostProps: nil,
- PostOriginalId: model.NewPointer(""),
- PostFileIds: model.StringArray{},
- }, messages[7])
-}
-
func TestCsvExport(t *testing.T) {
t.Run("no dedicated export filestore", func(t *testing.T) {
exportTempDir, err := os.MkdirTemp("", "")
@@ -647,12 +346,15 @@ func TestCsvExport(t *testing.T) {
func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filestore.FileBackend, attachmentBackend filestore.FileBackend) {
rctx := request.TestContext(t)
- header := "Post Creation Time,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id\n"
+ header := "Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id\n"
chanTypeDirect := model.ChannelTypeDirect
csvExportTests := []struct {
name string
cmhs map[string][]*model.ChannelMemberHistoryResult
+ metadata map[string]*shared.MetadataChannel
+ startTime int64
+ endTime int64
posts []*model.MessageExport
attachments map[string][]*model.FileInfo
expectedPosts string
@@ -665,7 +367,7 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
posts: []*model.MessageExport{},
attachments: map[string][]*model.FileInfo{},
expectedPosts: header,
- expectedMetadata: "{\n \"Channels\": {},\n \"MessagesCount\": 0,\n \"AttachmentsCount\": 0,\n \"StartTime\": 0,\n \"EndTime\": 0\n}",
+ expectedMetadata: "{\n \"Channels\": null,\n \"MessagesCount\": 0,\n \"AttachmentsCount\": 0,\n \"StartTime\": 0,\n \"EndTime\": 0\n}",
expectedFiles: 2,
},
{
@@ -683,10 +385,26 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
},
},
},
+ metadata: map[string]*shared.MetadataChannel{
+ "channel-id": {
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: "channel-id",
+ ChannelName: "channel-name",
+ ChannelDisplayName: "channel-display-name",
+ ChannelType: chanTypeDirect,
+ RoomId: "direct - channel-id",
+ StartTime: 1,
+ EndTime: 100,
+ },
+ },
+ startTime: 1,
+ endTime: 100,
posts: []*model.MessageExport{
{
PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostOriginalId: model.NewPointer(""),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
TeamDisplayName: model.NewPointer("team-display-name"),
@@ -694,6 +412,7 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
PostCreateAt: model.NewPointer(int64(1)),
+ PostUpdateAt: model.NewPointer(int64(1)),
PostMessage: model.NewPointer("message"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
@@ -703,7 +422,7 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
},
{
PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostOriginalId: model.NewPointer(""),
PostRootId: model.NewPointer("post-root-id"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
@@ -712,6 +431,7 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
PostCreateAt: model.NewPointer(int64(100)),
+ PostUpdateAt: model.NewPointer(int64(100)),
PostMessage: model.NewPointer("message"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
@@ -721,7 +441,7 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
},
{
PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostOriginalId: model.NewPointer(""),
PostRootId: model.NewPointer("post-root-id"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
@@ -730,6 +450,7 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
PostCreateAt: model.NewPointer(int64(100)),
+ PostUpdateAt: model.NewPointer(int64(100)),
PostMessage: model.NewPointer("message"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
@@ -742,13 +463,13 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
attachments: map[string][]*model.FileInfo{},
expectedPosts: strings.Join([]string{
header,
- "1,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test,test,test,,,,User test (test) was already in the channel,previously-joined,user,\n",
- "1,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,,,,User username (test@test.com) was already in the channel,previously-joined,user,\n",
- "1,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,post-original-id,,message,message,user,\n",
- "8,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test2,test2,test2,,,,User test2 (test2) joined the channel,enter,user,\n",
- "80,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test2,test2,test2,,,,User test2 (test2) leaved the channel,leave,user,\n",
- "100,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,post-original-id,post-root-id,message,message,user,\n",
- "100,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,post-original-id,post-root-id,message,message,user,o4w39mc1ff8y5fite4b8hacy1x\n",
+ "0,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test,test,test,,,,User test (test) was already in the channel,previously-joined,user,\n",
+ "1,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,,,,User username (test@test.com) was already in the channel,previously-joined,user,\n",
+ "1,1,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,,,message,message,user,\n",
+ "8,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test2,test2,test2,,,,User test2 (test2) joined the channel,enter,user,\n",
+ "80,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test2,test2,test2,,,,User test2 (test2) left the channel,leave,user,\n",
+ "100,100,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,,post-root-id,message,message,user,\n",
+ "100,100,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,,post-root-id,message,message,user,o4w39mc1ff8y5fite4b8hacy1x\n",
}, ""),
expectedMetadata: "{\n \"Channels\": {\n \"channel-id\": {\n \"TeamId\": \"team-id\",\n \"TeamName\": \"team-name\",\n \"TeamDisplayName\": \"team-display-name\",\n \"ChannelId\": \"channel-id\",\n \"ChannelName\": \"channel-name\",\n \"ChannelDisplayName\": \"channel-display-name\",\n \"ChannelType\": \"D\",\n \"RoomId\": \"direct - channel-id\",\n \"StartTime\": 1,\n \"EndTime\": 100,\n \"MessagesCount\": 3,\n \"AttachmentsCount\": 0\n }\n },\n \"MessagesCount\": 3,\n \"AttachmentsCount\": 0,\n \"StartTime\": 1,\n \"EndTime\": 100\n}",
expectedFiles: 2,
@@ -762,16 +483,33 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
},
},
},
+ metadata: map[string]*shared.MetadataChannel{
+ "channel-id": {
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: "channel-id",
+ ChannelName: "channel-name",
+ ChannelDisplayName: "channel-display-name",
+ ChannelType: chanTypeDirect,
+ RoomId: "direct - channel-id",
+ StartTime: 1,
+ EndTime: 100,
+ },
+ },
+ startTime: 1,
+ endTime: 100,
posts: []*model.MessageExport{
{
PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostOriginalId: model.NewPointer(""),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
TeamDisplayName: model.NewPointer("team-display-name"),
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(1)),
PostCreateAt: model.NewPointer(int64(1)),
PostMessage: model.NewPointer("message"),
UserEmail: model.NewPointer("test@test.com"),
@@ -782,7 +520,7 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
},
{
PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostOriginalId: model.NewPointer(""),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
TeamDisplayName: model.NewPointer("team-display-name"),
@@ -790,6 +528,7 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
PostCreateAt: model.NewPointer(int64(100)),
+ PostUpdateAt: model.NewPointer(int64(101)),
PostDeleteAt: model.NewPointer(int64(101)),
PostMessage: model.NewPointer("message"),
UserEmail: model.NewPointer("test@test.com"),
@@ -803,17 +542,18 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
attachments: map[string][]*model.FileInfo{},
expectedPosts: strings.Join([]string{
header,
- "1,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test,test,test,,,,User test (test) was already in the channel,previously-joined,user,\n",
- "1,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,,,,User username (test@test.com) was already in the channel,previously-joined,user,\n",
- "1,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,post-original-id,,message,message,user,\n",
- "100,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,post-original-id,,message,message,user,\n",
- "101,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,post-original-id,,delete message,message,user,\n",
+ "0,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test,test,test,,,,User test (test) was already in the channel,previously-joined,user,\n",
+ "1,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,,,,User username (test@test.com) was already in the channel,previously-joined,user,\n",
+ "1,1,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,,,message,message,user,\n",
+ "100,101,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,,,message,message,user,\n",
+ "100,101,Deleted,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,,,delete message,message,user,\n",
}, ""),
- expectedMetadata: "{\n \"Channels\": {\n \"channel-id\": {\n \"TeamId\": \"team-id\",\n \"TeamName\": \"team-name\",\n \"TeamDisplayName\": \"team-display-name\",\n \"ChannelId\": \"channel-id\",\n \"ChannelName\": \"channel-name\",\n \"ChannelDisplayName\": \"channel-display-name\",\n \"ChannelType\": \"D\",\n \"RoomId\": \"direct - channel-id\",\n \"StartTime\": 1,\n \"EndTime\": 100,\n \"MessagesCount\": 2,\n \"AttachmentsCount\": 0\n }\n },\n \"MessagesCount\": 2,\n \"AttachmentsCount\": 0,\n \"StartTime\": 1,\n \"EndTime\": 100\n}",
+ expectedMetadata: "{\n \"Channels\": {\n \"channel-id\": {\n \"TeamId\": \"team-id\",\n \"TeamName\": \"team-name\",\n \"TeamDisplayName\": \"team-display-name\",\n \"ChannelId\": \"channel-id\",\n \"ChannelName\": \"channel-name\",\n \"ChannelDisplayName\": \"channel-display-name\",\n \"ChannelType\": \"D\",\n \"RoomId\": \"direct - channel-id\",\n \"StartTime\": 1,\n \"EndTime\": 100,\n \"MessagesCount\": 3,\n \"AttachmentsCount\": 0\n }\n },\n \"MessagesCount\": 3,\n \"AttachmentsCount\": 0,\n \"StartTime\": 1,\n \"EndTime\": 100\n}",
expectedFiles: 2,
},
+
{
- name: "posts with attachments",
+ name: "posts with deleted attachment and deleted post, and at different time from non-deleted original post",
cmhs: map[string][]*model.ChannelMemberHistoryResult{
"channel-id": {
{
@@ -827,18 +567,35 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
},
},
},
+ metadata: map[string]*shared.MetadataChannel{
+ "channel-id": {
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: "channel-id",
+ ChannelName: "channel-name",
+ ChannelDisplayName: "channel-display-name",
+ ChannelType: chanTypeDirect,
+ RoomId: "direct - channel-id",
+ StartTime: 1,
+ EndTime: 100,
+ },
+ },
+ startTime: 1,
+ endTime: 100,
posts: []*model.MessageExport{
{
PostId: model.NewPointer("post-id-1"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostOriginalId: model.NewPointer(""),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
TeamDisplayName: model.NewPointer("team-display-name"),
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(1)),
PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
+ PostMessage: model.NewPointer("message 1"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
Username: model.NewPointer("username"),
@@ -847,7 +604,7 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
},
{
PostId: model.NewPointer("post-id-3"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostOriginalId: model.NewPointer(""),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
TeamDisplayName: model.NewPointer("team-display-name"),
@@ -855,17 +612,19 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
PostCreateAt: model.NewPointer(int64(2)),
+ PostUpdateAt: model.NewPointer(int64(3)),
PostDeleteAt: model.NewPointer(int64(3)),
- PostMessage: model.NewPointer("message"),
+ PostMessage: model.NewPointer("message 3"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
Username: model.NewPointer("username"),
ChannelType: &chanTypeDirect,
PostFileIds: []string{"test2"},
+ PostProps: model.NewPointer("{\"deleteBy\":\"user-id\"}"),
},
{
PostId: model.NewPointer("post-id-2"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostOriginalId: model.NewPointer(""),
PostRootId: model.NewPointer("post-id-1"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
@@ -873,13 +632,15 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(100)),
PostCreateAt: model.NewPointer(int64(100)),
- PostMessage: model.NewPointer("message"),
+ PostMessage: model.NewPointer("message 2"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
Username: model.NewPointer("username"),
ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
+ // NOTE: this post will be deleted, but the post-id-2 is not deleted.
+ PostFileIds: []string{"test3"},
},
},
attachments: map[string][]*model.FileInfo{
@@ -898,21 +659,33 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
DeleteAt: 3,
},
},
+ "post-id-2": {
+ {
+ Name: "test3",
+ Id: "test3",
+ Path: "test3",
+ DeleteAt: 102,
+ },
+ },
},
expectedPosts: strings.Join([]string{
header,
- "1,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test,test,test,,,,User test (test) was already in the channel,previously-joined,user,\n",
- "1,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,,,,User username (test@test.com) was already in the channel,previously-joined,user,\n",
- "1,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-1,post-original-id,,message,message,user,\n",
- "1,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-1,post-original-id,,test1 (files/post-id-1/test1-test1),attachment,user\n",
- "2,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-3,post-original-id,,message,message,user,\n",
- "3,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-3,post-original-id,,test2 (files/post-id-3/test2-test2),deleted attachment,user\n",
- "8,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test2,test2,test2,,,,User test2 (test2) joined the channel,enter,user,\n",
- "80,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test2,test2,test2,,,,User test2 (test2) leaved the channel,leave,user,\n",
- "100,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-2,post-original-id,post-id-1,message,message,user,\n",
+ "0,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test,test,test,,,,User test (test) was already in the channel,previously-joined,user,\n",
+ "1,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,,,,User username (test@test.com) was already in the channel,previously-joined,user,\n",
+ "1,1,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-1,,,message 1,message,user,\n",
+ "1,1,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-1,,,test1 (files/post-id-1/test1-test1),attachment,user,\n",
+ "2,3,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-3,,,message 3,message,user,\n",
+ "2,3,Deleted,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-3,,,delete message 3,message,user,\n",
+ "2,3,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-3,,,test2 (files/post-id-3/test2-test2),attachment,user,\n",
+ "2,3,FileDeleted,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-3,,,test2 (files/post-id-3/test2-test2),deleted attachment,user,\n",
+ "8,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test2,test2,test2,,,,User test2 (test2) joined the channel,enter,user,\n",
+ "80,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test2,test2,test2,,,,User test2 (test2) left the channel,leave,user,\n",
+ "100,100,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-2,,post-id-1,message 2,message,user,\n",
+ "100,100,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-2,,post-id-1,test3 (files/post-id-2/test3-test3),attachment,user,\n",
+ "100,102,FileDeleted,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-2,,post-id-1,test3 (files/post-id-2/test3-test3),deleted attachment,user,\n",
}, ""),
- expectedMetadata: "{\n \"Channels\": {\n \"channel-id\": {\n \"TeamId\": \"team-id\",\n \"TeamName\": \"team-name\",\n \"TeamDisplayName\": \"team-display-name\",\n \"ChannelId\": \"channel-id\",\n \"ChannelName\": \"channel-name\",\n \"ChannelDisplayName\": \"channel-display-name\",\n \"ChannelType\": \"D\",\n \"RoomId\": \"direct - channel-id\",\n \"StartTime\": 1,\n \"EndTime\": 100,\n \"MessagesCount\": 3,\n \"AttachmentsCount\": 2\n }\n },\n \"MessagesCount\": 3,\n \"AttachmentsCount\": 2,\n \"StartTime\": 1,\n \"EndTime\": 100\n}",
- expectedFiles: 4,
+ expectedMetadata: "{\n \"Channels\": {\n \"channel-id\": {\n \"TeamId\": \"team-id\",\n \"TeamName\": \"team-name\",\n \"TeamDisplayName\": \"team-display-name\",\n \"ChannelId\": \"channel-id\",\n \"ChannelName\": \"channel-name\",\n \"ChannelDisplayName\": \"channel-display-name\",\n \"ChannelType\": \"D\",\n \"RoomId\": \"direct - channel-id\",\n \"StartTime\": 1,\n \"EndTime\": 100,\n \"MessagesCount\": 4,\n \"AttachmentsCount\": 3\n }\n },\n \"MessagesCount\": 4,\n \"AttachmentsCount\": 3,\n \"StartTime\": 1,\n \"EndTime\": 100\n}",
+ expectedFiles: 5,
},
}
@@ -922,9 +695,8 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
defer mockStore.AssertExpectations(t)
if len(tt.attachments) > 0 {
- for post_id, attachments := range tt.attachments {
- attachments := attachments // TODO: Remove once go1.22 is used
- call := mockStore.FileInfoStore.On("GetForPost", post_id, true, true, false).Times(3)
+ for postId, attachments := range tt.attachments {
+ call := mockStore.FileInfoStore.On("GetForPost", postId, true, true, false)
call.Run(func(args mock.Arguments) {
call.Return(tt.attachments[args.Get(0).(string)], nil)
})
@@ -937,20 +709,26 @@ func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filest
}
}
- if len(tt.cmhs) > 0 {
- for channelId, cmhs := range tt.cmhs {
- mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", int64(1), int64(100), channelId).Return(cmhs, nil)
- }
- }
+ exportFileName := path.Join("export", "jobName", "jobName-batch001-csv.zip")
+ results, err := CsvExport(rctx, shared.ExportParams{
+ ChannelMetadata: tt.metadata,
+ Posts: tt.posts,
+ ChannelMemberHistories: tt.cmhs,
+ BatchPath: exportFileName,
+ BatchStartTime: tt.startTime,
+ BatchEndTime: tt.endTime,
+ Config: nil,
+ Db: shared.NewMessageExportStore(mockStore),
+ FileAttachmentBackend: attachmentBackend,
+ ExportBackend: exportBackend,
+ })
+ assert.NoError(t, err)
+ assert.Equal(t, 0, results.NumWarnings)
- warningCount, appErr := CsvExport(rctx, tt.posts, mockStore, exportBackend, attachmentBackend, "test")
- assert.Nil(t, appErr)
- assert.Equal(t, int64(0), warningCount)
-
- zipBytes, err := exportBackend.ReadFile("test/csv_export.zip")
+ zipBytes, err := exportBackend.ReadFile(exportFileName)
assert.NoError(t, err)
t.Cleanup(func() {
- err = exportBackend.RemoveFile("test/csv_export.zip")
+ err = exportBackend.RemoveFile(exportFileName)
require.NoError(t, err)
})
@@ -996,17 +774,32 @@ func TestWriteExportWarnings(t *testing.T) {
},
},
}
+ metadata := map[string]*shared.MetadataChannel{
+ "channel-id": {
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: "channel-id",
+ ChannelName: "channel-name",
+ ChannelDisplayName: "channel-display-name",
+ ChannelType: chanTypeDirect,
+ RoomId: "direct - channel-id",
+ StartTime: 1,
+ EndTime: 100,
+ },
+ }
posts := []*model.MessageExport{
{
PostId: model.NewPointer("post-id-1"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostOriginalId: model.NewPointer(""),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
TeamDisplayName: model.NewPointer("team-display-name"),
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(1)),
PostCreateAt: model.NewPointer(int64(1)),
PostMessage: model.NewPointer("message"),
UserEmail: model.NewPointer("test@test.com"),
@@ -1017,7 +810,7 @@ func TestWriteExportWarnings(t *testing.T) {
},
{
PostId: model.NewPointer("post-id-3"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostOriginalId: model.NewPointer(""),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
TeamDisplayName: model.NewPointer("team-display-name"),
@@ -1025,13 +818,14 @@ func TestWriteExportWarnings(t *testing.T) {
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
PostCreateAt: model.NewPointer(int64(2)),
+ PostUpdateAt: model.NewPointer(int64(3)),
PostDeleteAt: model.NewPointer(int64(3)),
PostMessage: model.NewPointer("message"),
UserEmail: model.NewPointer("test@test.com"),
UserId: model.NewPointer("user-id"),
Username: model.NewPointer("username"),
ChannelType: &chanTypeDirect,
- PostFileIds: []string{"post-id-2"},
+ PostFileIds: []string{"post-id-3"},
},
}
@@ -1070,30 +864,49 @@ func TestWriteExportWarnings(t *testing.T) {
mockStore := &storetest.Store{}
defer mockStore.AssertExpectations(t)
- for post_id := range attachments {
- call := mockStore.FileInfoStore.On("GetForPost", post_id, true, true, false).Times(3)
+ for postId := range attachments {
+ call := mockStore.FileInfoStore.On("GetForPost", postId, true, true, false).Times(2)
call.Run(func(args mock.Arguments) {
call.Return(attachments[args.Get(0).(string)], nil)
})
}
- for channelId, cmhs := range cmhs {
- mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", int64(1), int64(2), channelId).Return(cmhs, nil)
- }
+ exportFileName := path.Join("export", "jobName", "jobName-batch001-csv.zip")
+ results, err := CsvExport(rctx, shared.ExportParams{
+ ExportType: "",
+ ChannelMetadata: metadata,
+ Posts: posts,
+ ChannelMemberHistories: cmhs,
+ BatchPath: exportFileName,
+ BatchStartTime: 1,
+ BatchEndTime: 100,
+ Config: nil,
+ Db: shared.NewMessageExportStore(mockStore),
+ FileAttachmentBackend: fileBackend,
+ ExportBackend: fileBackend,
+ })
+ assert.NoError(t, err)
+ assert.Equal(t, 2, results.NumWarnings)
- warningCount, appErr := CsvExport(rctx, posts, mockStore, fileBackend, fileBackend, "test")
- assert.Nil(t, appErr)
- assert.Equal(t, int64(2), warningCount)
-
- zipBytes, err := fileBackend.ReadFile("test/csv_export.zip")
+ zipBytes, err := fileBackend.ReadFile(exportFileName)
assert.NoError(t, err)
t.Cleanup(func() {
- err = fileBackend.RemoveFile("test/csv_export.zip")
+ err = fileBackend.RemoveFile(exportFileName)
assert.NoError(t, err)
})
zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
assert.NoError(t, err)
assert.Len(t, zipReader.File, 3)
+ warningsTxt, err := zipReader.Open("warning.txt")
+ assert.NoError(t, err)
+ data, err := io.ReadAll(warningsTxt)
+ assert.NoError(t, err)
+ warnings := string(data)
+
+ expectedWarnings := fmt.Sprintf("Warning:%[1]s - Post: post-id-1 - test1\nWarning:%[1]s - Post: post-id-3 - test2\n",
+ shared.MissingFileMessage)
+
+ assert.Equal(t, expectedWarnings, warnings)
}
diff --git a/server/enterprise/message_export/global_relay_export/deliver.go b/server/enterprise/message_export/global_relay_export/deliver.go
index 2da7dc7758..329dc0a040 100644
--- a/server/enterprise/message_export/global_relay_export/deliver.go
+++ b/server/enterprise/message_export/global_relay_export/deliver.go
@@ -6,8 +6,8 @@ package global_relay_export
import (
"archive/zip"
"context"
+ "fmt"
"io"
- "net/http"
"net/mail"
"net/smtp"
"os"
@@ -16,14 +16,14 @@ import (
"github.com/mattermost/mattermost/server/public/model"
)
-func Deliver(export *os.File, config *model.Config) *model.AppError {
+func Deliver(export *os.File, config *model.Config) error {
info, err := export.Stat()
if err != nil {
- return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.unable_to_get_file_info.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+ return fmt.Errorf("unable to get the information of the export temporary file: %w", err)
}
zipFile, err := zip.NewReader(export, info.Size())
if err != nil {
- return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.unable_to_open_zip_file_data.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+ return fmt.Errorf("unable to open the export temporary file: %w", err)
}
to := *config.MessageExportSettings.GlobalRelaySettings.EmailAddress
@@ -33,7 +33,7 @@ func Deliver(export *os.File, config *model.Config) *model.AppError {
conn, err := connectToSMTPServer(ctx, config)
if err != nil {
- return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.unable_to_connect_smtp_server.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+ return fmt.Errorf("unable to connect to the smtp server: %w", err)
}
defer conn.Close()
@@ -55,56 +55,56 @@ func Deliver(export *os.File, config *model.Config) *model.AppError {
var nErr error
conn, nErr = connectToSMTPServer(context.Background(), config)
if nErr != nil {
- return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.unable_to_connect_smtp_server.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
+ return fmt.Errorf("unable to connect to the smtp server: %w", nErr)
}
}
}
return nil
}
-func deliverEmail(c *smtp.Client, mailFile *zip.File, from string, to string) *model.AppError {
+func deliverEmail(c *smtp.Client, mailFile *zip.File, from string, to string) error {
mailData, err := mailFile.Open()
if err != nil {
- return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.unable_to_open_email_file.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+ return fmt.Errorf("unable to get the an email from the temporary file: %w", err)
}
defer mailData.Close()
err = c.Mail(from)
if err != nil {
- return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.from_address.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+ return fmt.Errorf("unable to set the email From address: %w", err)
}
err = c.Rcpt(to)
if err != nil {
- return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.to_address.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+ return fmt.Errorf("unable to set the email To address: %w", err)
}
w, err := c.Data()
if err != nil {
- return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.msg_data.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+ return fmt.Errorf("unable to write the email message: %w", err)
}
_, err = io.Copy(w, mailData)
if err != nil {
- return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.msg.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+ return fmt.Errorf("unable to set the email message: %w", err)
}
err = w.Close()
if err != nil {
- return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.close.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+ return fmt.Errorf("unable to deliver the email to Global Relay: %w", err)
}
return nil
}
-func getFrom(mailFile *zip.File) (string, *model.AppError) {
+func getFrom(mailFile *zip.File) (string, error) {
mailData, err := mailFile.Open()
if err != nil {
- return "", model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.unable_to_open_email_file.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+ return "", fmt.Errorf("unable to get the an email from the temporary file: %w", err)
}
defer mailData.Close()
message, err := mail.ReadMessage(mailData)
if err != nil {
- return "", model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.parse_mail.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+ return "", fmt.Errorf("unable to read the email information: %w", err)
}
return message.Header.Get("From"), nil
}
diff --git a/server/enterprise/message_export/global_relay_export/deliver_test.go b/server/enterprise/message_export/global_relay_export/deliver_test.go
index 114cf8206f..28ca0816e2 100644
--- a/server/enterprise/message_export/global_relay_export/deliver_test.go
+++ b/server/enterprise/message_export/global_relay_export/deliver_test.go
@@ -30,8 +30,8 @@ func TestDeliver(t *testing.T) {
defer emptyFile.Close()
defer os.Remove(emptyFile.Name())
- appErr := Deliver(emptyFile, config)
- assert.NotNil(t, appErr)
+ err = Deliver(emptyFile, config)
+ assert.Error(t, err)
})
t.Run("Testing empty zip file", func(t *testing.T) {
@@ -46,8 +46,8 @@ func TestDeliver(t *testing.T) {
err = mail.DeleteMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
require.NoError(t, err)
- appErr := Deliver(emptyZipFile, config)
- assert.Nil(t, appErr)
+ err = Deliver(emptyZipFile, config)
+ assert.NoError(t, err)
_, err = mail.GetMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
require.Error(t, err)
@@ -87,8 +87,8 @@ func TestDeliver(t *testing.T) {
err = mail.DeleteMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
require.NoError(t, err)
- appErr := Deliver(emptyZipFile, config)
- assert.Nil(t, appErr)
+ err = Deliver(emptyZipFile, config)
+ assert.NoError(t, err)
mailbox, err := mail.GetMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
require.NoError(t, err)
@@ -130,8 +130,8 @@ func TestDeliver(t *testing.T) {
err = mail.DeleteMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
require.NoError(t, err)
- appErr := Deliver(emptyZipFile, config)
- assert.Nil(t, appErr)
+ err = Deliver(emptyZipFile, config)
+ assert.NoError(t, err)
mailbox, err := mail.GetMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
require.NoError(t, err)
diff --git a/server/enterprise/message_export/global_relay_export/global_relay_export.go b/server/enterprise/message_export/global_relay_export/global_relay_export.go
index b709e9b4f4..b52dd17cf7 100644
--- a/server/enterprise/message_export/global_relay_export/global_relay_export.go
+++ b/server/enterprise/message_export/global_relay_export/global_relay_export.go
@@ -9,7 +9,7 @@ import (
"fmt"
"io"
"mime"
- "net/http"
+ "os"
"sort"
"strings"
"time"
@@ -17,12 +17,11 @@ import (
"github.com/jaytaylor/html2text"
gomail "gopkg.in/mail.v2"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/common_export"
-
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
- "github.com/mattermost/mattermost/server/v8/channels/store"
+ "github.com/mattermost/mattermost/server/v8/enterprise/internal/file"
+ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
"github.com/mattermost/mattermost/server/v8/platform/shared/templates"
)
@@ -32,221 +31,228 @@ const (
GlobalRelayChannelNameHeader = "X-Mattermost-ChannelName"
GlobalRelayChannelIDHeader = "X-Mattermost-ChannelID"
GlobalRelayChannelTypeHeader = "X-Mattermost-ChannelType"
- MaxEmailBytes = 250 << (10 * 2)
MaxEmailsPerConnection = 400
)
-type AllExport map[string][]*ChannelExport
+// MaxEmailBytes is a var because it needs to be set in tests. Otherwise it shouldn't be touched.
+var MaxEmailBytes int64 = 250 * 1024 * 1024 // 250MB
type ChannelExport struct {
- ChannelId string // the unique id of the channel
- ChannelName string // the name of the channel
- ChannelType model.ChannelType // the channel type
- StartTime int64 // utc timestamp (seconds), start of export period or create time of channel, whichever is greater. Example: 1366611728.
- EndTime int64 // utc timestamp (seconds), end of export period or delete time of channel, whichever is lesser. Example: 1366611728.
- Participants []ParticipantRow // summary information about the conversation participants
- Messages []Message // the messages that were sent during the conversation
- ExportedOn int64 // utc timestamp (seconds), when this export was generated
- numUserMessages map[string]int // key is user id, value is number of messages that they sent during this period
- uploadedFiles []*model.FileInfo // any files that were uploaded to the channel during the export period
- bytes int64
+ TeamId string
+ TeamName string
+ TeamDisplayName string
+ ChannelId string // the unique id of the channel
+ ChannelName string // the name of the channel
+ ChannelDisplayName string
+ ChannelType model.ChannelType // the channel type
+ StartTime int64 // utc timestamp (seconds), start of export period or create time of channel, whichever is greater. Example: 1366611728.
+ EndTime int64 // utc timestamp (seconds), end of export period or delete time of channel, whichever is lesser. Example: 1366611728.
+ Participants []ParticipantRow // summary information about the conversation participants
+ Messages []Message // the messages that were sent during the conversation
+ ExportedOn int64 // utc timestamp (seconds), when this export was generated
+ numUserMessages map[string]int // key is user id, value is number of messages that they sent during this period
+ uploadedFiles []*model.FileInfo // any files that were uploaded to the channel during the export period
+ bytes int64
}
// a row in the summary table at the top of the export
type ParticipantRow struct {
- Username string
- UserType string
- Email string
- JoinTime int64
- LeaveTime int64
+ shared.JoinExport
MessagesSent int
}
type Message struct {
+ Id string
SentTime int64
+ SenderId string
SenderUsername string
- PostType string
PostUsername string
SenderUserType string
SenderEmail string
Message string
PreviewsPost string
+ UpdateAt int64
+ UpdateType shared.PostUpdatedType
+ EditedNewMsgId string
}
-func GlobalRelayExport(rctx request.CTX, posts []*model.MessageExport, db store.Store, fileAttachmentBackend filestore.FileBackend, dest io.Writer, templates *templates.Container) ([]string, int64, *model.AppError) {
- var warningCount int64
- attachmentsRemovedPostIDs := []string{}
- allExports := make(map[string][]*ChannelExport)
-
- zipFile := zip.NewWriter(dest)
-
- membersByChannel := common_export.MembersByChannel{}
- metadata := common_export.Metadata{
- Channels: map[string]common_export.MetadataChannel{},
- MessagesCount: 0,
- AttachmentsCount: 0,
- StartTime: 0,
- EndTime: 0,
+func GlobalRelayExport(rctx request.CTX, p shared.ExportParams) (shared.RunExportResults, error) {
+ exportData, err := shared.GetGenericExportData(p)
+ results := exportData.Results
+ if err != nil {
+ return results, err
}
- for _, post := range posts {
- if _, ok := membersByChannel[*post.ChannelId]; !ok {
- membersByChannel[*post.ChannelId] = common_export.ChannelMembers{}
- }
+ var attachmentsRemovedPostIDs []string
+ allExports := make(map[string][]*ChannelExport, len(exportData.Exports))
- membersByChannel[*post.ChannelId][*post.UserId] = common_export.ChannelMember{
- UserId: *post.UserId,
- Username: *post.Username,
- IsBot: post.IsBot,
- Email: *post.UserEmail,
- }
+ // save a pointer to the joins for each channel, to be used later in getParticipants
+ joinsByChannel := make(map[string][]shared.JoinExport, len(exportData.Exports))
- attachments := []*model.FileInfo{}
- if len(post.PostFileIds) > 0 {
- var err error
- attachments, err = db.FileInfo().GetForPost(*post.PostId, true, true, false)
- if err != nil {
- return nil, warningCount, model.NewAppError("GlobalRelayExport", "ent.message_export.global_relay_export.get_attachment_error", nil, "", http.StatusInternalServerError).Wrap(err)
+ for _, channel := range exportData.Exports {
+ for _, post := range channel.Posts {
+ attachmentsRemoved := addToExports(rctx, allExports, channel, post)
+ attachmentsRemovedPostIDs = append(attachmentsRemovedPostIDs, attachmentsRemoved...)
+ }
+ if len(channel.Posts) == 0 {
+ // channel has no posts, but it was exported anyway, so it must have joins and leaves.
+ if _, present := allExports[channel.ChannelId]; !present {
+ // we found a new channel
+ allExports[channel.ChannelId] = []*ChannelExport{genericChannelToChannelExport(channel)}
}
}
-
- attachmentsRemoved := addToExports(rctx, attachments, allExports, post)
- attachmentsRemovedPostIDs = append(attachmentsRemovedPostIDs, attachmentsRemoved...)
-
- metadata.Update(post, len(attachments))
+ joinsByChannel[channel.ChannelId] = channel.JoinEvents
}
+ tmpFile, err := os.CreateTemp("", "")
+ if err != nil {
+ return results, fmt.Errorf("unable to open the temporary export file: %w", err)
+ }
+ defer file.DeleteTemp(rctx.Logger(), tmpFile)
+ zipFile := zip.NewWriter(tmpFile)
+
+ // export each channelExport (sometimes multiple channelExport per real channel)
for _, channelExportList := range allExports {
for batchId, channelExport := range channelExportList {
- participants, appErr := getParticipants(db, channelExport, membersByChannel[channelExport.ChannelId])
- if appErr != nil {
- return nil, warningCount, appErr
- }
- channelExport.Participants = participants
- channelExport.ExportedOn = time.Now().Unix() * 1000
+ // we need to make the participant list for each channelExport "batch" (multiple "batches"per real channel)
+ // because each batch will have its own number of messages for that participant.
+ channelExport.Participants = getParticipants(channelExport, joinsByChannel[channelExport.ChannelId])
+ channelExport.ExportedOn = p.JobStartTime
- channelExportFile, err := zipFile.Create(fmt.Sprintf("%s - (%s) - %d.eml", channelExport.ChannelName, channelExport.ChannelId, batchId))
+ var channelExportFile io.Writer
+ channelExportFile, err = zipFile.Create(fmt.Sprintf("%s - (%s) - %d.eml", channelExport.ChannelName, channelExport.ChannelId, batchId))
if err != nil {
- return nil, warningCount, model.NewAppError("GlobalRelayExport", "ent.message_export.global_relay.create_file_in_zip.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+ return results, fmt.Errorf("unable to create the eml file: %w", err)
}
- if appErr, warningCount = generateEmail(rctx, fileAttachmentBackend, channelExport, templates, channelExportFile); appErr != nil {
- return nil, warningCount, appErr
+ if results.NumWarnings, err = generateEmail(rctx, p.FileAttachmentBackend, channelExport, p.Templates, channelExportFile); err != nil {
+ return results, err
}
}
}
- err := zipFile.Close()
+ err = zipFile.Close()
if err != nil {
- return nil, warningCount, model.NewAppError("GlobalRelayExport", "ent.message_export.global_relay.close_zip_file.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+ return results, fmt.Errorf("unable to close the zip file using tmpFile.Name: %v, err: %w", tmpFile.Name(), err)
}
- return attachmentsRemovedPostIDs, warningCount, nil
+ _, err = tmpFile.Seek(0, 0)
+ if err != nil {
+ return results, fmt.Errorf("unable to re-read the Global Relay temporary export file using tmpFile.Name: %v, err: %w", tmpFile.Name(), err)
+ }
+
+ if p.ExportType == model.ComplianceExportTypeGlobalrelayZip {
+ // Try to disable the write timeout for the potentially big export file.
+ _, err = filestore.TryWriteFileContext(rctx.Context(), p.ExportBackend, tmpFile, p.BatchPath)
+ if err != nil {
+ return results, fmt.Errorf("unable to write the global relay file, using tmpFile.Name: %v, batchPath: %v, err: %w", tmpFile.Name(), p.BatchPath, err)
+ }
+ } else {
+ err = Deliver(tmpFile, p.Config)
+ if err != nil {
+ return results, fmt.Errorf("unable to deliver tmpFile.Name: %v, err: %w", tmpFile.Name(), err)
+ }
+ }
+
+ if len(attachmentsRemovedPostIDs) > 0 {
+ rctx.Logger().Warn("Global Relay Attachments Removed because they were too large to send to Global Relay",
+ mlog.Int("number_of_attachments_removed", len(attachmentsRemovedPostIDs)))
+ rctx.Logger().Warn("List of posts which had attachments removed",
+ mlog.Array("post_ids", attachmentsRemovedPostIDs))
+ }
+
+ return results, nil
}
-func addToExports(rctx request.CTX, attachments []*model.FileInfo, exports map[string][]*ChannelExport, post *model.MessageExport) []string {
+// addToExports adds the post to the allExports collection. allExports keeps a map of channelId->[]*ChannelExport.
+// If a channelId has an existing []*ChannelExport, it adds post to the last ChannelExport in that list.
+// If the last ChannelExport is too big, it starts a new ChannelExport and appends it to the list (a new "batch").
+func addToExports(rctx request.CTX, allExports map[string][]*ChannelExport, genericChannel shared.ChannelExport,
+ post shared.PostExport) []string {
var channelExport *ChannelExport
- attachmentsRemovedPostIDs := []string{}
- if channelExports, present := exports[*post.ChannelId]; !present {
+ var attachmentsRemovedPostIDs []string
+ if channelExports, present := allExports[*post.ChannelId]; !present {
// we found a new channel
- channelExport = &ChannelExport{
- ChannelId: *post.ChannelId,
- ChannelName: *post.ChannelDisplayName,
- ChannelType: *post.ChannelType,
- StartTime: *post.PostCreateAt,
- Messages: make([]Message, 0),
- Participants: make([]ParticipantRow, 0),
- numUserMessages: make(map[string]int),
- uploadedFiles: make([]*model.FileInfo, 0),
- bytes: 0,
- }
- exports[*post.ChannelId] = []*ChannelExport{channelExport}
+ channelExport = genericChannelToChannelExport(genericChannel)
+ allExports[*post.ChannelId] = []*ChannelExport{channelExport}
} else {
// we already know about this channel
channelExport = channelExports[len(channelExports)-1]
}
- // Create a new ChannelExport if it would be too many bytes to add the post
- fileBytes := fileInfoListBytes(attachments)
msgBytes := int64(len(*post.PostMessage))
- postBytes := fileBytes + msgBytes
- postTooLargeForChannelBatch := channelExport.bytes+postBytes > MaxEmailBytes
- postAloneTooLargeToSend := postBytes > MaxEmailBytes // Attachments must be removed from export, they're too big to send.
- if postAloneTooLargeToSend {
+ // Create a new ChannelExport if it would be too many bytes to add the post.
+ // NOTE: we are only exporting attachment starts.
+ attachmentStarts := make([]*model.FileInfo, 0, len(post.AttachmentCreates))
+ for _, start := range post.AttachmentCreates {
+ attachmentStarts = append(attachmentStarts, start.FileInfo)
+ }
+ fileBytes := fileInfoListBytes(attachmentStarts)
+ postBytes := fileBytes + msgBytes
+ // NOTE: This is only a rough estimate -- we're not including the txt or html portion of the email...
+ attachmentsAloneTooLargeToSend := fileBytes > MaxEmailBytes // Attachments must be removed from export, they're too big to send.
+ if attachmentsAloneTooLargeToSend {
+ postBytes -= fileBytes
+ }
+ postTooLargeForChannelBatch := channelExport.bytes+postBytes > MaxEmailBytes
+
+ if attachmentsAloneTooLargeToSend {
attachmentsRemovedPostIDs = append(attachmentsRemovedPostIDs, *post.PostId)
}
- if postTooLargeForChannelBatch && !postAloneTooLargeToSend {
- channelExport = &ChannelExport{
- ChannelId: *post.ChannelId,
- ChannelName: *post.ChannelDisplayName,
- ChannelType: *post.ChannelType,
- StartTime: *post.PostCreateAt,
- Messages: make([]Message, 0),
- Participants: make([]ParticipantRow, 0),
- numUserMessages: make(map[string]int),
- uploadedFiles: make([]*model.FileInfo, 0),
- bytes: 0,
- }
- exports[*post.ChannelId] = append(exports[*post.ChannelId], channelExport)
+ // new "batch"
+ if postTooLargeForChannelBatch {
+ channelExport = genericChannelToChannelExport(genericChannel)
+ allExports[*post.ChannelId] = append(allExports[*post.ChannelId], channelExport)
}
addPostToChannelExport(rctx, channelExport, post)
// if this post includes files, add them to the collection
- for _, fileInfo := range attachments {
- addAttachmentToChannelExport(channelExport, post, fileInfo, postAloneTooLargeToSend)
- }
+ addAttachmentsToChannelExport(channelExport, post, post.AttachmentCreates, post.AttachmentDeletes, attachmentsAloneTooLargeToSend)
channelExport.bytes += postBytes
return attachmentsRemovedPostIDs
}
-func getParticipants(db store.Store, channelExport *ChannelExport, members common_export.ChannelMembers) ([]ParticipantRow, *model.AppError) {
- participantsMap := map[string]ParticipantRow{}
- channelMembersHistory, err := db.ChannelMemberHistory().GetUsersInChannelDuring(channelExport.StartTime, channelExport.EndTime, channelExport.ChannelId)
- if err != nil {
- return nil, model.NewAppError("getParticipants", "ent.get_users_in_channel_during", nil, "", http.StatusInternalServerError).Wrap(err)
+func genericChannelToChannelExport(genericChannel shared.ChannelExport) *ChannelExport {
+ return &ChannelExport{
+ TeamId: genericChannel.TeamId,
+ TeamName: genericChannel.TeamName,
+ TeamDisplayName: genericChannel.TeamDisplayName,
+ ChannelId: genericChannel.ChannelId,
+ ChannelName: genericChannel.ChannelName,
+ ChannelDisplayName: genericChannel.DisplayName,
+ ChannelType: genericChannel.ChannelType,
+ StartTime: genericChannel.StartTime,
+ EndTime: genericChannel.EndTime,
+ // we can't preallocate sizes here because we don't know how many will be in this "batch"
+ Participants: make([]ParticipantRow, 0),
+ Messages: make([]Message, 0),
+ ExportedOn: 0,
+ numUserMessages: make(map[string]int),
+ uploadedFiles: make([]*model.FileInfo, 0),
+ bytes: 0,
}
+}
- joins, leaves := common_export.GetJoinsAndLeavesForChannel(channelExport.StartTime, channelExport.EndTime, channelMembersHistory, members)
-
- for _, join := range joins {
- userType := "user"
- if join.IsBot {
- userType = "bot"
- }
-
- if _, ok := participantsMap[join.UserId]; !ok {
- participantsMap[join.UserId] = ParticipantRow{
- Username: join.Username,
- UserType: userType,
- Email: join.Email,
- JoinTime: join.Datetime,
- LeaveTime: channelExport.EndTime,
- MessagesSent: channelExport.numUserMessages[join.UserId],
- }
- }
- }
- for _, leave := range leaves {
- if participantRow, ok := participantsMap[leave.UserId]; ok {
- participantRow.LeaveTime = leave.Datetime //nolint:govet
- }
- }
-
- participants := []ParticipantRow{}
- for _, participant := range participantsMap {
- participants = append(participants, participant)
+func getParticipants(channelExport *ChannelExport, joinEvents []shared.JoinExport) []ParticipantRow {
+ participants := make([]ParticipantRow, 0, len(joinEvents))
+ for _, j := range joinEvents {
+ participants = append(participants, ParticipantRow{
+ JoinExport: j,
+ MessagesSent: channelExport.numUserMessages[j.UserId],
+ })
}
sort.Slice(participants, func(i, j int) bool {
return participants[i].Username < participants[j].Username
})
- return participants, nil
+ return participants
}
-func generateEmail(rctx request.CTX, fileAttachmentBackend filestore.FileBackend, channelExport *ChannelExport, templates *templates.Container, w io.Writer) (*model.AppError, int64) {
- var warningCount int64
+func generateEmail(rctx request.CTX, fileAttachmentBackend filestore.FileBackend, channelExport *ChannelExport, templates *templates.Container, w io.Writer) (int, error) {
+ var warningCount int
participantEmailAddresses := getParticipantEmails(channelExport)
// GlobalRelay expects the email to come from the person that initiated the conversation.
@@ -258,10 +264,10 @@ func generateEmail(rctx request.CTX, fileAttachmentBackend filestore.FileBackend
htmlBody, err := channelExportToHTML(rctx, channelExport, templates)
if err != nil {
- return model.NewAppError("GlobalRelayExport", "ent.message_export.global_relay.generate_email.app_error", nil, "", http.StatusInternalServerError).Wrap(err), warningCount
+ return warningCount, fmt.Errorf("unable to generate eml file data: %w", err)
}
- subject := fmt.Sprintf("Mattermost Compliance Export: %s", channelExport.ChannelName)
+ subject := fmt.Sprintf("Mattermost Compliance Export: %s", channelExport.ChannelDisplayName)
htmlMessage := "\r\n" + htmlBody + ""
txtBody, err := html2text.FromString(htmlBody)
@@ -278,9 +284,9 @@ func generateEmail(rctx request.CTX, fileAttachmentBackend filestore.FileBackend
"Auto-Submitted": {"auto-generated"},
"Precedence": {"bulk"},
GlobalRelayMsgTypeHeader: {"Mattermost"},
- GlobalRelayChannelNameHeader: {encodeRFC2047Word(channelExport.ChannelName)},
+ GlobalRelayChannelNameHeader: {encodeRFC2047Word(channelExport.ChannelDisplayName)},
GlobalRelayChannelIDHeader: {encodeRFC2047Word(channelExport.ChannelId)},
- GlobalRelayChannelTypeHeader: {encodeRFC2047Word(common_export.ChannelTypeDisplayName(channelExport.ChannelType))},
+ GlobalRelayChannelTypeHeader: {encodeRFC2047Word(shared.ChannelTypeDisplayName(channelExport.ChannelType))},
}
m := gomail.NewMessage(gomail.SetCharset("UTF-8"))
@@ -293,9 +299,10 @@ func generateEmail(rctx request.CTX, fileAttachmentBackend filestore.FileBackend
path := fileInfo.Path
m.Attach(fileInfo.Name, gomail.SetCopyFunc(func(writer io.Writer) error {
- reader, appErr := fileAttachmentBackend.Reader(path)
- if appErr != nil {
- rctx.Logger().Warn("File not found for export", mlog.String("Filename", path))
+ var reader filestore.ReadCloseSeeker
+ reader, err = fileAttachmentBackend.Reader(path)
+ if err != nil {
+ rctx.Logger().Warn("File not found for export", mlog.String("filename", path))
warningCount += 1
return nil
}
@@ -303,7 +310,7 @@ func generateEmail(rctx request.CTX, fileAttachmentBackend filestore.FileBackend
_, err = io.Copy(writer, reader)
if err != nil {
- return model.NewAppError("GlobalRelayExport", "ent.message_export.global_relay.attach_file.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+ return fmt.Errorf("unable to add attachment to the Global Relay export: %w", err)
}
return nil
}))
@@ -311,15 +318,15 @@ func generateEmail(rctx request.CTX, fileAttachmentBackend filestore.FileBackend
_, err = m.WriteTo(w)
if err != nil {
- return model.NewAppError("GlobalRelayExport", "ent.message_export.global_relay.generate_email.app_error", nil, "", http.StatusInternalServerError).Wrap(err), warningCount
+ return warningCount, fmt.Errorf("unable to generate eml file data: %w", err)
}
- return nil, warningCount
+ return warningCount, nil
}
func getParticipantEmails(channelExport *ChannelExport) []string {
- participantEmails := make([]string, len(channelExport.Participants))
- for i, participant := range channelExport.Participants {
- participantEmails[i] = participant.Email
+ participantEmails := make([]string, 0, len(channelExport.Participants))
+ for _, participant := range channelExport.Participants {
+ participantEmails = append(participantEmails, participant.UserEmail)
}
return participantEmails
}
@@ -332,19 +339,15 @@ func fileInfoListBytes(fileInfoList []*model.FileInfo) int64 {
return totalBytes
}
-func addPostToChannelExport(rctx request.CTX, channelExport *ChannelExport, post *model.MessageExport) {
- userType := "user"
- if post.IsBot {
- userType = "bot"
- }
-
+func addPostToChannelExport(rctx request.CTX, channelExport *ChannelExport, post shared.PostExport) {
strPostProps := post.PostProps
- bytPostProps := []byte(*strPostProps)
+ bytePostProps := []byte(*strPostProps)
// Added to show the username if overridden by a webhook or API integration
- postUserName := ""
+ originalUsername := model.SafeDereference(post.Username)
+ postUserName := originalUsername
var postPropsLocal map[string]any
- err := json.Unmarshal(bytPostProps, &postPropsLocal)
+ err := json.Unmarshal(bytePostProps, &postPropsLocal)
if err != nil {
rctx.Logger().Warn("Failed to unmarshal post Props into JSON. Ignoring username override.", mlog.Err(err))
} else {
@@ -352,67 +355,59 @@ func addPostToChannelExport(rctx request.CTX, channelExport *ChannelExport, post
postUserName = overrideUsername.(string)
}
- if postUserName == "" {
+ if postUserName == originalUsername {
if overrideUsername, ok := postPropsLocal["webhook_display_name"]; ok {
postUserName = overrideUsername.(string)
}
}
}
- element := Message{
- SentTime: *post.PostCreateAt,
- Message: *post.PostMessage,
- SenderUserType: userType,
- PostType: *post.PostType,
- PostUsername: postUserName,
- SenderUsername: *post.Username,
- SenderEmail: *post.UserEmail,
- PreviewsPost: post.PreviewID(),
- }
+ element := postToMessage(post)
+ element.PostUsername = postUserName
channelExport.Messages = append(channelExport.Messages, element)
- channelExport.EndTime = *post.PostCreateAt
channelExport.numUserMessages[*post.UserId] += 1
}
-func addAttachmentToChannelExport(channelExport *ChannelExport, post *model.MessageExport, fileInfo *model.FileInfo, removeAttachments bool) {
- var uploadElement Message
- userType := "user"
- if post.IsBot {
- userType = "bot"
+func postToMessage(post shared.PostExport) Message {
+ return Message{
+ Id: model.SafeDereference(post.PostId),
+ SentTime: model.SafeDereference(post.PostCreateAt),
+ SenderId: model.SafeDereference(post.UserId),
+ SenderUsername: model.SafeDereference(post.Username),
+ PostUsername: model.SafeDereference(post.Username),
+ SenderUserType: string(post.UserType),
+ SenderEmail: model.SafeDereference(post.UserEmail),
+ Message: post.Message,
+ PreviewsPost: post.PreviewID(),
+ UpdateAt: post.UpdateAt,
+ UpdateType: post.UpdatedType,
+ EditedNewMsgId: post.EditedNewMsgId,
}
- if removeAttachments {
- // add "post" message indicating that attachments were not sent
- uploadElement = Message{
- SentTime: fileInfo.CreateAt,
- Message: fmt.Sprintf("Uploaded file '%s' (id '%s') was removed because it was too large to send.", fileInfo.Name, fileInfo.Id),
- SenderUsername: *post.Username,
- SenderUserType: userType,
- SenderEmail: *post.UserEmail,
+}
+
+func addAttachmentsToChannelExport(channelExport *ChannelExport, post shared.PostExport,
+ attachmentStarts []*shared.FileUploadStartExport, attachmentDeletes []shared.PostExport, removeAttachments bool) {
+ for _, start := range attachmentStarts {
+ var message string
+
+ if removeAttachments {
+ message = fmt.Sprintf("Uploaded file %q (id '%s') was removed because it was too large to send.",
+ start.FileInfo.Name, start.FileInfo.Id)
+ } else {
+ channelExport.uploadedFiles = append(channelExport.uploadedFiles, start.FileInfo)
+ message = fmt.Sprintf("Uploaded file %s", start.FileInfo.Name)
}
- if fileInfo.DeleteAt != 0 {
- uploadElement.SentTime = fileInfo.DeleteAt
- uploadElement.Message = fmt.Sprintf("Deleted file '%s' (id '%s') was removed because it was too large to send.", fileInfo.Name, fileInfo.Id)
- }
- } else {
- channelExport.uploadedFiles = append(channelExport.uploadedFiles, fileInfo)
-
- // add an implicit "post" to the export that includes the filename so GlobalRelay knows who uploaded each file
- uploadElement = Message{
- SentTime: fileInfo.CreateAt,
- Message: fmt.Sprintf("Uploaded file %s", fileInfo.Name),
- SenderUsername: *post.Username,
- SenderUserType: userType,
- SenderEmail: *post.UserEmail,
- }
-
- if fileInfo.DeleteAt != 0 {
- uploadElement.SentTime = fileInfo.DeleteAt
- uploadElement.Message = fmt.Sprintf("Deleted file %s", fileInfo.Name)
- }
+ uploadElement := postToMessage(post)
+ uploadElement.Message = message
+ channelExport.Messages = append(channelExport.Messages, uploadElement)
}
- channelExport.Messages = append(channelExport.Messages, uploadElement)
+ for _, deleted := range attachmentDeletes {
+ uploadElement := postToMessage(deleted)
+ uploadElement.Message = fmt.Sprintf("Deleted file %s", deleted.FileInfo.Name)
+ channelExport.Messages = append(channelExport.Messages, uploadElement)
+ }
}
func encodeRFC2047Word(s string) string {
diff --git a/server/enterprise/message_export/global_relay_export/global_relay_export_test.go b/server/enterprise/message_export/global_relay_export/global_relay_export_test.go
index 9ff3be1607..a697583236 100644
--- a/server/enterprise/message_export/global_relay_export/global_relay_export_test.go
+++ b/server/enterprise/message_export/global_relay_export/global_relay_export_test.go
@@ -6,19 +6,23 @@ package global_relay_export
import (
"archive/zip"
"bytes"
+ "encoding/base64"
+ "fmt"
"io"
"os"
+ "path"
"strings"
"testing"
"github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
+ "github.com/mattermost/mattermost/server/public/plugin/plugintest/mock"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/utils/fileutils"
+ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
"github.com/mattermost/mattermost/server/v8/platform/shared/templates"
)
@@ -47,68 +51,89 @@ func TestGlobalRelayExport(t *testing.T) {
fileBackend, err := filestore.NewFileBackend(config)
assert.NoError(t, err)
+ exportBackend := fileBackend
chanTypeDirect := model.ChannelTypeDirect
- csvExportTests := []struct {
- name string
- cmhs map[string][]*model.ChannelMemberHistoryResult
- posts []*model.MessageExport
- attachments map[string][]*model.FileInfo
- expectedHeaders []string
- expectedTexts []string
- expectedHTMLs []string
- expectedFiles int
- expectedWarnings int
+ grExportTests := []struct {
+ name string
+ cmhs map[string][]*model.ChannelMemberHistoryResult
+ metadata map[string]*shared.MetadataChannel
+ startTime int64
+ endTime int64
+ posts []*model.MessageExport
+ attachments map[string][]*model.FileInfo
+ attachmentsContent map[string]string
+ expectedAttachmentContent [][]string
+ maxEmailBytes int64
+ numExpectedEmls int
+ expectedHeaders [][]string
+ expectedTexts [][]string
+ notExpectedTexts [][]string
+ expectedHTMLs [][]string
+ expectedWarnings int
+ empty bool
}{
{
name: "empty",
cmhs: map[string][]*model.ChannelMemberHistoryResult{},
posts: []*model.MessageExport{},
attachments: map[string][]*model.FileInfo{},
- expectedHeaders: []string{},
- expectedTexts: []string{},
- expectedHTMLs: []string{},
- expectedFiles: 0,
expectedWarnings: 0,
+ empty: true,
},
{
name: "posts",
cmhs: map[string][]*model.ChannelMemberHistoryResult{
"channel-id": {
{
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(400)),
+ JoinTime: 0, UserId: "id-test1", UserEmail: "test1@test.com", Username: "test1", LeaveTime: model.NewPointer(int64(100_000)),
},
{
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(80)),
+ JoinTime: 8, UserId: "id-test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
},
{
- JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
+ JoinTime: 400, UserId: "id-test3", UserEmail: "test3@test.com", Username: "test3",
},
},
},
+ metadata: map[string]*shared.MetadataChannel{
+ "channel-id": {
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: "channel-id",
+ ChannelName: "channel-name",
+ ChannelDisplayName: "channel-display-name",
+ ChannelType: chanTypeDirect,
+ RoomId: "direct - channel-id",
+ StartTime: 1,
+ EndTime: 100,
+ },
+ },
+ startTime: 1,
+ endTime: 100_000,
posts: []*model.MessageExport{
{
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostId: model.NewPointer("post-id1"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
TeamDisplayName: model.NewPointer("team-display-name"),
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(1)),
PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
+ PostMessage: model.NewPointer("message 1"),
PostProps: model.NewPointer("{}"),
PostType: model.NewPointer(""),
UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
+ UserId: model.NewPointer("id-test1"),
Username: model.NewPointer("test1"),
ChannelType: &chanTypeDirect,
PostFileIds: []string{},
},
{
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
+ PostId: model.NewPointer("post-id2"),
PostRootId: model.NewPointer("post-root-id"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
@@ -116,19 +141,20 @@ func TestGlobalRelayExport(t *testing.T) {
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(100000)),
- PostMessage: model.NewPointer("message"),
+ PostUpdateAt: model.NewPointer(int64(100_000)),
+ PostCreateAt: model.NewPointer(int64(100_000)),
+ PostMessage: model.NewPointer("message 2"),
PostProps: model.NewPointer("{}"),
PostType: model.NewPointer(""),
UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
+ UserId: model.NewPointer("id-test1"),
Username: model.NewPointer("test1"),
ChannelType: &chanTypeDirect,
PostFileIds: []string{},
},
},
attachments: map[string][]*model.FileInfo{},
- expectedHeaders: []string{
+ expectedHeaders: [][]string{{
"MIME-Version: 1.0",
"X-Mattermost-ChannelType: direct",
"Content-Transfer-Encoding: 8bit",
@@ -141,11 +167,16 @@ func TestGlobalRelayExport(t *testing.T) {
"From: test1@test.com",
"To: test1@test.com,test2@test.com",
"Subject: Mattermost Compliance Export: channel-display-name",
- },
+ }},
- expectedTexts: []string{
+ expectedTexts: [][]string{{
strings.Join([]string{
- "* Channel: channel-display-name",
+ "* TeamId: team-id",
+ "* TeamName: team-name",
+ "* TeamDisplayName: team-display-name",
+ "* ChannelId: channel-id",
+ "* ChannelName: channel-name",
+ "* ChannelDisplayName: channel-display-name",
"* Started: 1970-01-01T00:00:00Z",
"* Ended: 1970-01-01T00:01:40Z",
"* Duration: 2 minutes",
@@ -155,26 +186,33 @@ func TestGlobalRelayExport(t *testing.T) {
"Messages",
"--------",
"",
- "* 1970-01-01T00:00:00Z @test1 user (test1@test.com): message",
- "* 1970-01-01T00:01:40Z @test1 user (test1@test.com): message",
+ "* post-id1 1970-01-01T00:00:00Z @test1 id-test1 @test1 user (test1@test.com=",
+ ") message 1",
+ "* post-id2 1970-01-01T00:01:40Z @test1 id-test1 @test1 user (test1@test.com=",
+ ") message 2",
}, "\r\n"),
- },
+ }},
- expectedHTMLs: []string{
+ expectedHTMLs: [][]string{{
strings.Join([]string{
" ",
- " - Channel: channel-display-name<=",
- "/li>",
- "
- Started: 1970-01-01T00:00:00Z<=",
- "/li>",
- "
- Ended: 1970-01-01T00:01:40Z",
- "
- Duration: 2 minutes
",
+ " - TeamId:team-id
",
+ " - TeamName:team-name
",
+ " - TeamDisplayName:team-display-name=",
+ "li>",
+ "
- ChannelId:channel-id
",
+ " - ChannelName:channel-name
",
+ " - ChannelDisplayName:channel-display-=",
+ "name
",
+ " - Started:1970-01-01T00:00:00Z
",
+ " - Ended:1970-01-01T00:01:40Z
",
+ " - Duration:2 minutes
",
"
",
}, "\r\n"),
strings.Join([]string{
"",
- " | @test | ",
+ " id-test1 | ",
+ " @test1 | ",
" user | ",
" test1@test.com | ",
" 1970-01-01T00:00:00Z | ",
@@ -184,6 +222,7 @@ func TestGlobalRelayExport(t *testing.T) {
"
",
"",
"",
+ " | id-test2 | ",
" @test2 | ",
" user | ",
" test2@test.com | ",
@@ -194,6 +233,7 @@ func TestGlobalRelayExport(t *testing.T) {
"
",
"",
"",
+ " | id-test3 | ",
" @test3 | ",
" user | ",
" test3@test.com | ",
@@ -206,54 +246,1491 @@ func TestGlobalRelayExport(t *testing.T) {
strings.Join([]string{
"",
+ " post-id1",
" 1970-01-01T00:00:00Z",
" @test1",
- " ",
+ " id-test1",
+ " @test1",
" user",
- " (test1@test.com):",
- " message",
- " ",
+ " (test1@test.com)",
+ " message 1",
"",
"",
"",
+ " post-id2",
" 1970-01-01T00:01:40Z",
" @test1",
- " ",
+ " id-test1",
+ " @test1",
" user",
- " (test1@test.com):",
- " message",
- " ",
+ " (test1@test.com)",
+ " message 2",
"",
}, "\r\n"),
- },
- expectedFiles: 1,
+ }},
expectedWarnings: 0,
+ numExpectedEmls: 1,
},
+
{
- name: "posts with attachments",
+ name: "posts with attachments, size ok",
cmhs: map[string][]*model.ChannelMemberHistoryResult{
"channel-id": {
{
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(400)),
+ JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
},
{
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(80)),
+ JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
},
{
JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
},
},
},
+ metadata: map[string]*shared.MetadataChannel{
+ "channel-id": {
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: "channel-id",
+ ChannelName: "channel-name",
+ ChannelDisplayName: "channel-display-name",
+ ChannelType: chanTypeDirect,
+ RoomId: "direct - channel-id",
+ StartTime: 1,
+ EndTime: 100,
+ },
+ },
+ startTime: 1,
+ endTime: 100_000,
posts: []*model.MessageExport{
{
PostId: model.NewPointer("post-id-1"),
- PostOriginalId: model.NewPointer("post-original-id"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
TeamDisplayName: model.NewPointer("team-display-name"),
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(1)),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostMessage: model.NewPointer("message1"),
+ PostProps: model.NewPointer("{}"),
+ PostType: model.NewPointer(""),
+ UserEmail: model.NewPointer("test1@test.com"),
+ UserId: model.NewPointer("test1"),
+ Username: model.NewPointer("test1"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{"test1"},
+ },
+ {
+ PostId: model.NewPointer("post-id-2"),
+ PostRootId: model.NewPointer("post-id-1"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(100_000)),
+ PostCreateAt: model.NewPointer(int64(100_000)),
+ PostMessage: model.NewPointer("message2"),
+ PostProps: model.NewPointer("{}"),
+ PostType: model.NewPointer(""),
+ UserEmail: model.NewPointer("test1@test.com"),
+ UserId: model.NewPointer("test1"),
+ Username: model.NewPointer("test1"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{"test2"},
+ },
+ },
+ attachments: map[string][]*model.FileInfo{
+ "post-id-1": {
+ {
+ Name: "test1-attachment",
+ Id: "test1-attachment",
+ Path: "test1-attachment",
+ CreateAt: 1,
+ },
+ },
+ "post-id-2": {
+ {
+ Name: "test2-attachment",
+ Id: "test2-attachment",
+ Path: "test2-attachment",
+ CreateAt: 1,
+ },
+ },
+ },
+ attachmentsContent: map[string]string{
+ "test1-attachment": "this is the attachment content",
+ "test2-attachment": "this is the second attachment content",
+ },
+ expectedAttachmentContent: [][]string{
+ {base64.StdEncoding.EncodeToString([]byte("this is the attachment content"))},
+ {base64.StdEncoding.EncodeToString([]byte("this is the second attachment content"))},
+ },
+ expectedHeaders: [][]string{{
+ "MIME-Version: 1.0",
+ "X-Mattermost-ChannelType: direct",
+ "Content-Transfer-Encoding: 8bit",
+ "Precedence: bulk",
+ "X-GlobalRelay-MsgType: Mattermost",
+ "X-Mattermost-ChannelID: channel-id",
+ "X-Mattermost-ChannelName: channel-display-name",
+ "Auto-Submitted: auto-generated",
+ "Date: Thu, 01 Jan 1970 00:01:40 +0000",
+ "From: test1@test.com",
+ "To: test1@test.com,test2@test.com",
+ "Subject: Mattermost Compliance Export: channel-display-name",
+ }},
+
+ expectedTexts: [][]string{{
+ strings.Join([]string{
+ "* TeamId: team-id",
+ "* TeamName: team-name",
+ "* TeamDisplayName: team-display-name",
+ "* ChannelId: channel-id",
+ "* ChannelName: channel-name",
+ "* ChannelDisplayName: channel-display-name",
+ "* Started: 1970-01-01T00:00:00Z",
+ "* Ended: 1970-01-01T00:01:40Z",
+ "* Duration: 2 minutes",
+ }, "\r\n"),
+ strings.Join([]string{
+ "--------",
+ "Messages",
+ "--------",
+ "",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "message1",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "Uploaded file test1-attachment",
+ "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
+ "message2",
+ }, "\r\n"),
+ strings.Join([]string{
+ "Content-Disposition: attachment; filename=\"test1-attachment\"",
+ "Content-Transfer-Encoding: base64",
+ "Content-Type: application/octet-stream; name=\"test1-attachment\"",
+ }, "\r\n"),
+ }},
+
+ expectedHTMLs: [][]string{{
+ strings.Join([]string{
+ " ",
+ " - TeamId:team-id
",
+ " - TeamName:team-name
",
+ " - TeamDisplayName:team-display-name=",
+ "li>",
+ "
- ChannelId:channel-id
",
+ " - ChannelName:channel-name
",
+ " - ChannelDisplayName:channel-display-=",
+ "name
",
+ " - Started:1970-01-01T00:00:00Z
",
+ " - Ended:1970-01-01T00:01:40Z
",
+ " - Duration:2 minutes
",
+ "
",
+ }, "\r\n"),
+ strings.Join([]string{
+ "
",
+ " | test1 | ",
+ " @test | ",
+ " user | ",
+ " test1@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 2 | ",
+ "
",
+ "",
+ "",
+ " | test2 | ",
+ " @test2 | ",
+ " user | ",
+ " test2@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 0 | ",
+ "
",
+ "",
+ "",
+ " | test3 | ",
+ " @test3 | ",
+ " user | ",
+ " test3@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 0 | ",
+ "
",
+ }, "\r\n"),
+
+ strings.Join([]string{
+ "",
+ " post-id-1",
+ " 1970-01-01T00:00:00Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " message1",
+ "",
+ "",
+ "",
+ " post-id-1",
+ " 1970-01-01T00:00:00Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " Uploaded file test1-attachment",
+ "",
+ "",
+ "",
+ " post-id-2",
+ " 1970-01-01T00:01:40Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " message2",
+ "",
+ }, "\r\n"),
+ }},
+ expectedWarnings: 0,
+ numExpectedEmls: 1,
+ },
+
+ {
+ name: "posts with attachments, size too large, new channel export needed",
+ cmhs: map[string][]*model.ChannelMemberHistoryResult{
+ "channel-id": {
+ {
+ JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
+ },
+ {
+ JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
+ },
+ {
+ JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
+ },
+ },
+ },
+ metadata: map[string]*shared.MetadataChannel{
+ "channel-id": {
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: "channel-id",
+ ChannelName: "channel-name",
+ ChannelDisplayName: "channel-display-name",
+ ChannelType: chanTypeDirect,
+ RoomId: "direct - channel-id",
+ StartTime: 1,
+ EndTime: 100,
+ },
+ },
+ startTime: 1,
+ endTime: 100_000,
+ posts: []*model.MessageExport{
+ {
+ PostId: model.NewPointer("post-id-1"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(1)),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostMessage: model.NewPointer("message1"),
+ PostProps: model.NewPointer("{}"),
+ PostType: model.NewPointer(""),
+ UserEmail: model.NewPointer("test1@test.com"),
+ UserId: model.NewPointer("test1"),
+ Username: model.NewPointer("test1"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{"test1"},
+ },
+ {
+ PostId: model.NewPointer("post-id-2"),
+ PostRootId: model.NewPointer("post-id-1"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(100_000)),
+ PostCreateAt: model.NewPointer(int64(100_000)),
+ PostMessage: model.NewPointer("message2"),
+ PostProps: model.NewPointer("{}"),
+ PostType: model.NewPointer(""),
+ UserEmail: model.NewPointer("test1@test.com"),
+ UserId: model.NewPointer("test1"),
+ Username: model.NewPointer("test1"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{"test2"},
+ },
+ },
+ attachments: map[string][]*model.FileInfo{
+ "post-id-1": {
+ {
+ Name: "test1-attachment",
+ Id: "test1-attachment",
+ Path: "test1-attachment",
+ Size: 90, // 90 + 8 for message = message of 98 bytes. next one will be too big.
+ },
+ },
+ "post-id-2": {
+ {
+ Name: "test2-attachment",
+ Id: "test2-attachment",
+ Path: "test2-attachment",
+ Size: 56,
+ },
+ },
+ },
+ maxEmailBytes: 100,
+ numExpectedEmls: 2,
+ attachmentsContent: map[string]string{
+ "test1-attachment": "this is the attachment content",
+ "test2-attachment": "this is the second attachment content",
+ },
+ expectedAttachmentContent: [][]string{
+ {base64.StdEncoding.EncodeToString([]byte("this is the attachment content"))},
+ {base64.StdEncoding.EncodeToString([]byte("this is the second attachment content"))},
+ },
+ expectedHeaders: [][]string{
+ // eml 0
+ {
+ "MIME-Version: 1.0",
+ "X-Mattermost-ChannelType: direct",
+ "Content-Transfer-Encoding: 8bit",
+ "Precedence: bulk",
+ "X-GlobalRelay-MsgType: Mattermost",
+ "X-Mattermost-ChannelID: channel-id",
+ "X-Mattermost-ChannelName: channel-display-name",
+ "Auto-Submitted: auto-generated",
+ "Date: Thu, 01 Jan 1970 00:01:40 +0000",
+ "From: test1@test.com",
+ "To: test1@test.com,test2@test.com",
+ "Subject: Mattermost Compliance Export: channel-display-name",
+ },
+ // eml 1
+ {
+ "MIME-Version: 1.0",
+ "X-Mattermost-ChannelType: direct",
+ "Content-Transfer-Encoding: 8bit",
+ "Precedence: bulk",
+ "X-GlobalRelay-MsgType: Mattermost",
+ "X-Mattermost-ChannelID: channel-id",
+ "X-Mattermost-ChannelName: channel-display-name",
+ "Auto-Submitted: auto-generated",
+ "Date: Thu, 01 Jan 1970 00:01:40 +0000",
+ "From: test1@test.com",
+ "To: test1@test.com,test2@test.com",
+ "Subject: Mattermost Compliance Export: channel-display-name",
+ },
+ },
+
+ expectedTexts: [][]string{
+ // eml 0
+ {
+ strings.Join([]string{
+ "* TeamId: team-id",
+ "* TeamName: team-name",
+ "* TeamDisplayName: team-display-name",
+ "* ChannelId: channel-id",
+ "* ChannelName: channel-name",
+ "* ChannelDisplayName: channel-display-name",
+ "* Started: 1970-01-01T00:00:00Z",
+ "* Ended: 1970-01-01T00:01:40Z",
+ "* Duration: 2 minutes",
+ }, "\r\n"),
+ strings.Join([]string{
+ "--------",
+ "Messages",
+ "--------",
+ "",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "message1",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "Uploaded file test1-attachment",
+ }, "\r\n"),
+ strings.Join([]string{
+ "Content-Disposition: attachment; filename=\"test1-attachment\"",
+ "Content-Transfer-Encoding: base64",
+ "Content-Type: application/octet-stream; name=\"test1-attachment\"",
+ }, "\r\n"),
+ },
+ // eml 1
+ {
+ strings.Join([]string{
+ "* TeamId: team-id",
+ "* TeamName: team-name",
+ "* TeamDisplayName: team-display-name",
+ "* ChannelId: channel-id",
+ "* ChannelName: channel-name",
+ "* ChannelDisplayName: channel-display-name",
+ "* Started: 1970-01-01T00:00:00Z",
+ "* Ended: 1970-01-01T00:01:40Z",
+ "* Duration: 2 minutes",
+ }, "\r\n"),
+ strings.Join([]string{
+ "--------",
+ "Messages",
+ "--------",
+ "",
+ "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
+ "message2",
+ "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
+ "Uploaded file test2-attachment",
+ }, "\r\n"),
+ strings.Join([]string{
+ "Content-Disposition: attachment; filename=\"test2-attachment\"",
+ "Content-Transfer-Encoding: base64",
+ "Content-Type: application/octet-stream; name=\"test2-attachment\"",
+ }, "\r\n"),
+ },
+ },
+
+ expectedHTMLs: [][]string{
+ // eml 0
+ {
+ strings.Join([]string{
+ " ",
+ " - TeamId:team-id
",
+ " - TeamName:team-name
",
+ " - TeamDisplayName:team-display-name=",
+ "li>",
+ "
- ChannelId:channel-id
",
+ " - ChannelName:channel-name
",
+ " - ChannelDisplayName:channel-display-=",
+ "name
",
+ " - Started:1970-01-01T00:00:00Z
",
+ " - Ended:1970-01-01T00:01:40Z
",
+ " - Duration:2 minutes
",
+ "
",
+ }, "\r\n"),
+ strings.Join([]string{
+ "",
+ " | test1 | ",
+ " @test | ",
+ " user | ",
+ " test1@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 1 | ",
+ "
",
+ "",
+ "",
+ " | test2 | ",
+ " @test2 | ",
+ " user | ",
+ " test2@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 0 | ",
+ "
",
+ "",
+ "",
+ " | test3 | ",
+ " @test3 | ",
+ " user | ",
+ " test3@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 0 | ",
+ "
",
+ }, "\r\n"),
+ strings.Join([]string{
+ "",
+ " post-id-1",
+ " 1970-01-01T00:00:00Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " message1",
+ "",
+ "",
+ "",
+ " post-id-1",
+ " 1970-01-01T00:00:00Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " Uploaded file test1-attachment",
+ "",
+ }, "\r\n"),
+ },
+ // eml 1
+ {
+ strings.Join([]string{
+ " ",
+ " - TeamId:team-id
",
+ " - TeamName:team-name
",
+ " - TeamDisplayName:team-display-name=",
+ "li>",
+ "
- ChannelId:channel-id
",
+ " - ChannelName:channel-name
",
+ " - ChannelDisplayName:channel-display-=",
+ "name
",
+ " - Started:1970-01-01T00:00:00Z
",
+ " - Ended:1970-01-01T00:01:40Z
",
+ " - Duration:2 minutes
",
+ "
",
+ }, "\r\n"),
+ strings.Join([]string{
+ "",
+ " | test1 | ",
+ " @test | ",
+ " user | ",
+ " test1@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 1 | ",
+ "
",
+ "",
+ "",
+ " | test2 | ",
+ " @test2 | ",
+ " user | ",
+ " test2@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 0 | ",
+ "
",
+ "",
+ "",
+ " | test3 | ",
+ " @test3 | ",
+ " user | ",
+ " test3@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 0 | ",
+ "
",
+ }, "\r\n"),
+ strings.Join([]string{
+ "",
+ " post-id-2",
+ " 1970-01-01T00:01:40Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " message2",
+ "",
+ "",
+ "",
+ " post-id-2",
+ " 1970-01-01T00:01:40Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " Uploaded file test2-attachment",
+ "",
+ }, "\r\n"),
+ },
+ },
+ expectedWarnings: 0,
+ },
+
+ {
+ name: "posts with attachments, attachment too large, remove it but only one eml needed",
+ cmhs: map[string][]*model.ChannelMemberHistoryResult{
+ "channel-id": {
+ {
+ JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
+ },
+ {
+ JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
+ },
+ {
+ JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
+ },
+ },
+ },
+ metadata: map[string]*shared.MetadataChannel{
+ "channel-id": {
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: "channel-id",
+ ChannelName: "channel-name",
+ ChannelDisplayName: "channel-display-name",
+ ChannelType: chanTypeDirect,
+ RoomId: "direct - channel-id",
+ StartTime: 1,
+ EndTime: 100,
+ },
+ },
+ startTime: 1,
+ endTime: 100_000,
+ posts: []*model.MessageExport{
+ {
+ PostId: model.NewPointer("post-id-1"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(1)),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostMessage: model.NewPointer("message1"),
+ PostProps: model.NewPointer("{}"),
+ PostType: model.NewPointer(""),
+ UserEmail: model.NewPointer("test1@test.com"),
+ UserId: model.NewPointer("test1"),
+ Username: model.NewPointer("test1"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{"test1"},
+ },
+ {
+ PostId: model.NewPointer("post-id-2"),
+ PostRootId: model.NewPointer("post-id-1"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(100_000)),
+ PostCreateAt: model.NewPointer(int64(100_000)),
+ PostMessage: model.NewPointer("message2"),
+ PostProps: model.NewPointer("{}"),
+ PostType: model.NewPointer(""),
+ UserEmail: model.NewPointer("test1@test.com"),
+ UserId: model.NewPointer("test1"),
+ Username: model.NewPointer("test1"),
+ ChannelType: &chanTypeDirect,
+ },
+ },
+ attachments: map[string][]*model.FileInfo{
+ "post-id-1": {
+ {
+ Name: "test1-attachment",
+ Id: "test1-attachment",
+ Path: "test1-attachment",
+ CreateAt: 1,
+ Size: 101,
+ },
+ },
+ },
+ attachmentsContent: map[string]string{
+ "test1-attachment": "this is the attachment content",
+ },
+ expectedAttachmentContent: [][]string{
+ {""},
+ },
+ maxEmailBytes: 100,
+ numExpectedEmls: 1,
+ expectedHeaders: [][]string{{
+ "MIME-Version: 1.0",
+ "X-Mattermost-ChannelType: direct",
+ "Content-Transfer-Encoding: 8bit",
+ "Precedence: bulk",
+ "X-GlobalRelay-MsgType: Mattermost",
+ "X-Mattermost-ChannelID: channel-id",
+ "X-Mattermost-ChannelName: channel-display-name",
+ "Auto-Submitted: auto-generated",
+ "Date: Thu, 01 Jan 1970 00:01:40 +0000",
+ "From: test1@test.com",
+ "To: test1@test.com,test2@test.com",
+ "Subject: Mattermost Compliance Export: channel-display-name",
+ }},
+
+ expectedTexts: [][]string{{
+ strings.Join([]string{
+ "* TeamId: team-id",
+ "* TeamName: team-name",
+ "* TeamDisplayName: team-display-name",
+ "* ChannelId: channel-id",
+ "* ChannelName: channel-name",
+ "* ChannelDisplayName: channel-display-name",
+ "* Started: 1970-01-01T00:00:00Z",
+ "* Ended: 1970-01-01T00:01:40Z",
+ "* Duration: 2 minutes",
+ }, "\r\n"),
+ strings.Join([]string{
+ "--------",
+ "Messages",
+ "--------",
+ "",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "message1",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "Uploaded file \"test1-attachment\" (id 'test1-attachment') was removed becaus=",
+ "e it was too large to send.",
+ "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
+ "message2",
+ }, "\r\n"),
+ }},
+
+ notExpectedTexts: [][]string{{
+ "Content-Disposition: attachment; filename=\"test1-attachment\"",
+ }},
+ expectedHTMLs: [][]string{{
+ strings.Join([]string{
+ " ",
+ " - TeamId:team-id
",
+ " - TeamName:team-name
",
+ " - TeamDisplayName:team-display-name=",
+ "li>",
+ "
- ChannelId:channel-id
",
+ " - ChannelName:channel-name
",
+ " - ChannelDisplayName:channel-display-=",
+ "name
",
+ " - Started:1970-01-01T00:00:00Z
",
+ " - Ended:1970-01-01T00:01:40Z
",
+ " - Duration:2 minutes
",
+ "
",
+ }, "\r\n"),
+ strings.Join([]string{
+ "",
+ " | test1 | ",
+ " @test | ",
+ " user | ",
+ " test1@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 2 | ",
+ "
",
+ "",
+ "",
+ " | test2 | ",
+ " @test2 | ",
+ " user | ",
+ " test2@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 0 | ",
+ "
",
+ "",
+ "",
+ " | test3 | ",
+ " @test3 | ",
+ " user | ",
+ " test3@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 0 | ",
+ "
",
+ }, "\r\n"),
+
+ strings.Join([]string{
+ "",
+ " post-id-1",
+ " 1970-01-01T00:00:00Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " message1",
+ "",
+ "",
+ "",
+ " post-id-1",
+ " 1970-01-01T00:00:00Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " Uploaded file "test1-attachment" (id =",
+ "39;test1-attachment') was removed because it was too large to send.",
+ "",
+ "",
+ "",
+ " post-id-2",
+ " 1970-01-01T00:01:40Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " message2",
+ "",
+ }, "\r\n"),
+ }},
+ expectedWarnings: 0,
+ },
+
+ {
+ name: "posts with attachments, post size and attachment too large, new channel export needed and attachment removed",
+ cmhs: map[string][]*model.ChannelMemberHistoryResult{
+ "channel-id": {
+ {
+ JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
+ },
+ {
+ JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
+ },
+ {
+ JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
+ },
+ },
+ },
+ metadata: map[string]*shared.MetadataChannel{
+ "channel-id": {
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: "channel-id",
+ ChannelName: "channel-name",
+ ChannelDisplayName: "channel-display-name",
+ ChannelType: chanTypeDirect,
+ RoomId: "direct - channel-id",
+ StartTime: 1,
+ EndTime: 100,
+ },
+ },
+ startTime: 1,
+ endTime: 100_000,
+ posts: []*model.MessageExport{
+ {
+ PostId: model.NewPointer("post-id-1"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(1)),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostMessage: model.NewPointer("message1"),
+ PostProps: model.NewPointer("{}"),
+ PostType: model.NewPointer(""),
+ UserEmail: model.NewPointer("test1@test.com"),
+ UserId: model.NewPointer("test1"),
+ Username: model.NewPointer("test1"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{"test1"},
+ },
+ {
+ PostId: model.NewPointer("post-id-2"),
+ PostRootId: model.NewPointer("post-id-1"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(100_000)),
+ PostCreateAt: model.NewPointer(int64(100_000)),
+ PostMessage: model.NewPointer("message2"),
+ PostProps: model.NewPointer("{}"),
+ PostType: model.NewPointer(""),
+ UserEmail: model.NewPointer("test1@test.com"),
+ UserId: model.NewPointer("test1"),
+ Username: model.NewPointer("test1"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{"test2"},
+ },
+ },
+ attachments: map[string][]*model.FileInfo{
+ "post-id-1": {
+ {
+ Name: "test1-attachment",
+ Id: "test1-attachment",
+ Path: "test1-attachment",
+ Size: 20, // 20 + 8 for message = message of 28 bytes. next one will be too big.
+ },
+ },
+ "post-id-2": {
+ {
+ Name: "test2-attachment",
+ Id: "test2-attachment",
+ Path: "test2-attachment",
+ Size: 31, // too big, will be deleted
+ },
+ },
+ },
+ maxEmailBytes: 30,
+ numExpectedEmls: 2,
+ attachmentsContent: map[string]string{
+ "test1-attachment": "this is the attachment content",
+ "test2-attachment": "this is the second attachment content",
+ },
+ expectedAttachmentContent: [][]string{
+ {base64.StdEncoding.EncodeToString([]byte("this is the attachment content"))},
+ {""},
+ },
+ notExpectedTexts: [][]string{
+ // eml 0
+ {
+ "Content-Disposition: attachment; filename=\"test2-attachment\"",
+ },
+ // eml 1
+ {
+ "Content-Disposition: attachment; filename=\"test1-attachment\"",
+ "Content-Disposition: attachment; filename=\"test2-attachment\"",
+ },
+ },
+
+ expectedHeaders: [][]string{
+ // eml 0
+ {
+ "MIME-Version: 1.0",
+ "X-Mattermost-ChannelType: direct",
+ "Content-Transfer-Encoding: 8bit",
+ "Precedence: bulk",
+ "X-GlobalRelay-MsgType: Mattermost",
+ "X-Mattermost-ChannelID: channel-id",
+ "X-Mattermost-ChannelName: channel-display-name",
+ "Auto-Submitted: auto-generated",
+ "Date: Thu, 01 Jan 1970 00:01:40 +0000",
+ "From: test1@test.com",
+ "To: test1@test.com,test2@test.com",
+ "Subject: Mattermost Compliance Export: channel-display-name",
+ },
+ // eml 1
+ {
+ "MIME-Version: 1.0",
+ "X-Mattermost-ChannelType: direct",
+ "Content-Transfer-Encoding: 8bit",
+ "Precedence: bulk",
+ "X-GlobalRelay-MsgType: Mattermost",
+ "X-Mattermost-ChannelID: channel-id",
+ "X-Mattermost-ChannelName: channel-display-name",
+ "Auto-Submitted: auto-generated",
+ "Date: Thu, 01 Jan 1970 00:01:40 +0000",
+ "From: test1@test.com",
+ "To: test1@test.com,test2@test.com",
+ "Subject: Mattermost Compliance Export: channel-display-name",
+ },
+ },
+
+ expectedTexts: [][]string{
+ // eml 0
+ {
+ strings.Join([]string{
+ "* TeamId: team-id",
+ "* TeamName: team-name",
+ "* TeamDisplayName: team-display-name",
+ "* ChannelId: channel-id",
+ "* ChannelName: channel-name",
+ "* ChannelDisplayName: channel-display-name",
+ "* Started: 1970-01-01T00:00:00Z",
+ "* Ended: 1970-01-01T00:01:40Z",
+ "* Duration: 2 minutes",
+ }, "\r\n"),
+ strings.Join([]string{
+ "--------",
+ "Messages",
+ "--------",
+ "",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "message1",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "Uploaded file test1-attachment",
+ }, "\r\n"),
+ strings.Join([]string{
+ "Content-Disposition: attachment; filename=\"test1-attachment\"",
+ "Content-Transfer-Encoding: base64",
+ "Content-Type: application/octet-stream; name=\"test1-attachment\"",
+ }, "\r\n"),
+ },
+ // eml 1
+ {
+ strings.Join([]string{
+ "* TeamId: team-id",
+ "* TeamName: team-name",
+ "* TeamDisplayName: team-display-name",
+ "* ChannelId: channel-id",
+ "* ChannelName: channel-name",
+ "* ChannelDisplayName: channel-display-name",
+ "* Started: 1970-01-01T00:00:00Z",
+ "* Ended: 1970-01-01T00:01:40Z",
+ "* Duration: 2 minutes",
+ }, "\r\n"),
+ strings.Join([]string{
+ "--------",
+ "Messages",
+ "--------",
+ "",
+ "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
+ "message2",
+ "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
+ "Uploaded file \"test2-attachment\" (id 'test2-attachment') was removed becaus=",
+ "e it was too large to send.",
+ }, "\r\n"),
+ },
+ },
+
+ expectedHTMLs: [][]string{
+ // eml 0
+ {
+ strings.Join([]string{
+ " ",
+ " - TeamId:team-id
",
+ " - TeamName:team-name
",
+ " - TeamDisplayName:team-display-name=",
+ "li>",
+ "
- ChannelId:channel-id
",
+ " - ChannelName:channel-name
",
+ " - ChannelDisplayName:channel-display-=",
+ "name
",
+ " - Started:1970-01-01T00:00:00Z
",
+ " - Ended:1970-01-01T00:01:40Z
",
+ " - Duration:2 minutes
",
+ "
",
+ }, "\r\n"),
+ strings.Join([]string{
+ "",
+ " | test1 | ",
+ " @test | ",
+ " user | ",
+ " test1@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 1 | ",
+ "
",
+ "",
+ "",
+ " | test2 | ",
+ " @test2 | ",
+ " user | ",
+ " test2@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 0 | ",
+ "
",
+ "",
+ "",
+ " | test3 | ",
+ " @test3 | ",
+ " user | ",
+ " test3@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 0 | ",
+ "
",
+ }, "\r\n"),
+ strings.Join([]string{
+ "",
+ " post-id-1",
+ " 1970-01-01T00:00:00Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " message1",
+ "",
+ "",
+ "",
+ " post-id-1",
+ " 1970-01-01T00:00:00Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " Uploaded file test1-attachment",
+ "",
+ }, "\r\n"),
+ },
+ // eml 1
+ {
+ strings.Join([]string{
+ " ",
+ " - TeamId:team-id
",
+ " - TeamName:team-name
",
+ " - TeamDisplayName:team-display-name=",
+ "li>",
+ "
- ChannelId:channel-id
",
+ " - ChannelName:channel-name
",
+ " - ChannelDisplayName:channel-display-=",
+ "name
",
+ " - Started:1970-01-01T00:00:00Z
",
+ " - Ended:1970-01-01T00:01:40Z
",
+ " - Duration:2 minutes
",
+ "
",
+ }, "\r\n"),
+ strings.Join([]string{
+ "",
+ " | test1 | ",
+ " @test | ",
+ " user | ",
+ " test1@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 1 | ",
+ "
",
+ "",
+ "",
+ " | test2 | ",
+ " @test2 | ",
+ " user | ",
+ " test2@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 0 | ",
+ "
",
+ "",
+ "",
+ " | test3 | ",
+ " @test3 | ",
+ " user | ",
+ " test3@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 0 | ",
+ "
",
+ }, "\r\n"),
+ strings.Join([]string{
+ "",
+ " post-id-2",
+ " 1970-01-01T00:01:40Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " message2",
+ "",
+ "",
+ "",
+ " post-id-2",
+ " 1970-01-01T00:01:40Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " Uploaded file "test2-attachment" (id =",
+ "39;test2-attachment') was removed because it was too large to send.",
+ "",
+ }, "\r\n"),
+ },
+ },
+ expectedWarnings: 0,
+ },
+
+ {
+ name: "posts with multiple attachments, size ok",
+ cmhs: map[string][]*model.ChannelMemberHistoryResult{
+ "channel-id": {
+ {
+ JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
+ },
+ {
+ JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
+ },
+ {
+ JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
+ },
+ },
+ },
+ metadata: map[string]*shared.MetadataChannel{
+ "channel-id": {
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: "channel-id",
+ ChannelName: "channel-name",
+ ChannelDisplayName: "channel-display-name",
+ ChannelType: chanTypeDirect,
+ RoomId: "direct - channel-id",
+ StartTime: 1,
+ EndTime: 100,
+ },
+ },
+ startTime: 1,
+ endTime: 100_000,
+ posts: []*model.MessageExport{
+ {
+ PostId: model.NewPointer("post-id-1"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(1)),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostMessage: model.NewPointer("message1"),
+ PostProps: model.NewPointer("{}"),
+ PostType: model.NewPointer(""),
+ UserEmail: model.NewPointer("test1@test.com"),
+ UserId: model.NewPointer("test1"),
+ Username: model.NewPointer("test1"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{"test1", "test1-2"},
+ },
+ {
+ PostId: model.NewPointer("post-id-2"),
+ PostRootId: model.NewPointer("post-id-1"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(100_000)),
+ PostCreateAt: model.NewPointer(int64(100_000)),
+ PostMessage: model.NewPointer("message2"),
+ PostProps: model.NewPointer("{}"),
+ PostType: model.NewPointer(""),
+ UserEmail: model.NewPointer("test1@test.com"),
+ UserId: model.NewPointer("test1"),
+ Username: model.NewPointer("test1"),
+ ChannelType: &chanTypeDirect,
+ },
+ },
+ attachments: map[string][]*model.FileInfo{
+ "post-id-1": {
+ {
+ Name: "test1-attachment",
+ Id: "test1-attachment",
+ Path: "test1-attachment",
+ CreateAt: 1,
+ },
+ {
+ Name: "test1-attachment-2",
+ Id: "test1-attachment-2",
+ Path: "test1-attachment-2",
+ CreateAt: 1,
+ },
+ },
+ },
+ attachmentsContent: map[string]string{
+ "test1-attachment": "this is the attachment content",
+ "test1-attachment-2": "this is the second attachment content",
+ },
+ expectedAttachmentContent: [][]string{
+ {base64.StdEncoding.EncodeToString([]byte("this is the attachment content"))},
+ {base64.StdEncoding.EncodeToString([]byte("this is the second attachment content"))},
+ },
+
+ expectedHeaders: [][]string{{
+ "MIME-Version: 1.0",
+ "X-Mattermost-ChannelType: direct",
+ "Content-Transfer-Encoding: 8bit",
+ "Precedence: bulk",
+ "X-GlobalRelay-MsgType: Mattermost",
+ "X-Mattermost-ChannelID: channel-id",
+ "X-Mattermost-ChannelName: channel-display-name",
+ "Auto-Submitted: auto-generated",
+ "Date: Thu, 01 Jan 1970 00:01:40 +0000",
+ "From: test1@test.com",
+ "To: test1@test.com,test2@test.com",
+ "Subject: Mattermost Compliance Export: channel-display-name",
+ }},
+
+ expectedTexts: [][]string{{
+ strings.Join([]string{
+ "* TeamId: team-id",
+ "* TeamName: team-name",
+ "* TeamDisplayName: team-display-name",
+ "* ChannelId: channel-id",
+ "* ChannelName: channel-name",
+ "* ChannelDisplayName: channel-display-name",
+ "* Started: 1970-01-01T00:00:00Z",
+ "* Ended: 1970-01-01T00:01:40Z",
+ "* Duration: 2 minutes",
+ }, "\r\n"),
+ strings.Join([]string{
+ "--------",
+ "Messages",
+ "--------",
+ "",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "message1",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "Uploaded file test1-attachment",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "Uploaded file test1-attachment-2",
+ "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
+ "message2",
+ }, "\r\n"),
+ strings.Join([]string{
+ "Content-Disposition: attachment; filename=\"test1-attachment\"",
+ "Content-Transfer-Encoding: base64",
+ "Content-Type: application/octet-stream; name=\"test1-attachment\"",
+ }, "\r\n"),
+ strings.Join([]string{
+ "Content-Disposition: attachment; filename=\"test1-attachment-2\"",
+ "Content-Transfer-Encoding: base64",
+ "Content-Type: application/octet-stream; name=\"test1-attachment-2\"",
+ }, "\r\n"),
+ }},
+
+ expectedHTMLs: [][]string{{
+ strings.Join([]string{
+ " ",
+ " - TeamId:team-id
",
+ " - TeamName:team-name
",
+ " - TeamDisplayName:team-display-name=",
+ "li>",
+ "
- ChannelId:channel-id
",
+ " - ChannelName:channel-name
",
+ " - ChannelDisplayName:channel-display-=",
+ "name
",
+ " - Started:1970-01-01T00:00:00Z
",
+ " - Ended:1970-01-01T00:01:40Z
",
+ " - Duration:2 minutes
",
+ "
",
+ }, "\r\n"),
+ strings.Join([]string{
+ "",
+ " | test1 | ",
+ " @test | ",
+ " user | ",
+ " test1@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 2 | ",
+ "
",
+ "",
+ "",
+ " | test2 | ",
+ " @test2 | ",
+ " user | ",
+ " test2@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 0 | ",
+ "
",
+ "",
+ "",
+ " | test3 | ",
+ " @test3 | ",
+ " user | ",
+ " test3@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 0 | ",
+ "
",
+ }, "\r\n"),
+
+ strings.Join([]string{
+ "",
+ " post-id-1",
+ " 1970-01-01T00:00:00Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " message1",
+ "",
+ "",
+ "",
+ " post-id-1",
+ " 1970-01-01T00:00:00Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " Uploaded file test1-attachment",
+ "",
+ "",
+ "",
+ " post-id-1",
+ " 1970-01-01T00:00:00Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " Uploaded file test1-attachment-2",
+ "",
+ "",
+ "",
+ " post-id-2",
+ " 1970-01-01T00:01:40Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " message2",
+ "",
+ }, "\r\n"),
+ }},
+ expectedWarnings: 0,
+ numExpectedEmls: 1,
+ },
+
+ {
+ name: "posts with deleted post and deleted attachments",
+ cmhs: map[string][]*model.ChannelMemberHistoryResult{
+ "channel-id": {
+ {
+ JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
+ },
+ {
+ JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
+ },
+ {
+ JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
+ },
+ },
+ },
+ metadata: map[string]*shared.MetadataChannel{
+ "channel-id": {
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: "channel-id",
+ ChannelName: "channel-name",
+ ChannelDisplayName: "channel-display-name",
+ ChannelType: chanTypeDirect,
+ RoomId: "direct - channel-id",
+ StartTime: 1,
+ EndTime: 100,
+ },
+ },
+ startTime: 1,
+ endTime: 200_000,
+ posts: []*model.MessageExport{
+ {
+ PostId: model.NewPointer("post-id-1"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(1)),
PostCreateAt: model.NewPointer(int64(1)),
PostMessage: model.NewPointer("message"),
PostProps: model.NewPointer("{}"),
@@ -266,7 +1743,6 @@ func TestGlobalRelayExport(t *testing.T) {
},
{
PostId: model.NewPointer("post-id-2"),
- PostOriginalId: model.NewPointer("post-original-id"),
PostRootId: model.NewPointer("post-id-1"),
TeamId: model.NewPointer("team-id"),
TeamName: model.NewPointer("team-name"),
@@ -274,7 +1750,261 @@ func TestGlobalRelayExport(t *testing.T) {
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(100000)),
+ PostCreateAt: model.NewPointer(int64(100_000)),
+ PostUpdateAt: model.NewPointer(int64(200_000)),
+ PostDeleteAt: model.NewPointer(int64(200_000)),
+ PostMessage: model.NewPointer("message"),
+ PostProps: model.NewPointer("{\"deleteBy\":\"fy8j97gwii84bk4zxprbpc9d9w\"}"),
+ PostType: model.NewPointer(""),
+ UserEmail: model.NewPointer("test1@test.com"),
+ UserId: model.NewPointer("test1"),
+ Username: model.NewPointer("test1"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{},
+ },
+ },
+ attachments: map[string][]*model.FileInfo{
+ "post-id-1": {
+ {
+ Name: "test1-attachment",
+ Id: "test1-attachment",
+ Path: "test1-attachment",
+ CreateAt: 1,
+ UpdateAt: 200_000,
+ DeleteAt: 200_000,
+ },
+ },
+ },
+ attachmentsContent: map[string]string{
+ "test1-attachment": "this is the attachment content",
+ },
+ expectedAttachmentContent: [][]string{
+ {base64.StdEncoding.EncodeToString([]byte("this is the attachment content"))},
+ },
+ expectedHeaders: [][]string{{
+ "MIME-Version: 1.0",
+ "X-Mattermost-ChannelType: direct",
+ "Content-Transfer-Encoding: 8bit",
+ "Precedence: bulk",
+ "X-GlobalRelay-MsgType: Mattermost",
+ "X-Mattermost-ChannelID: channel-id",
+ "X-Mattermost-ChannelName: channel-display-name",
+ "Auto-Submitted: auto-generated",
+ "Date: Thu, 01 Jan 1970 00:03:20 +0000",
+ "From: test1@test.com",
+ "To: test1@test.com,test2@test.com",
+ "Subject: Mattermost Compliance Export: channel-display-name",
+ }},
+
+ expectedTexts: [][]string{{
+ strings.Join([]string{
+ "* TeamId: team-id",
+ "* TeamName: team-name",
+ "* TeamDisplayName: team-display-name",
+ "* ChannelId: channel-id",
+ "* ChannelName: channel-name",
+ "* ChannelDisplayName: channel-display-name",
+ "* Started: 1970-01-01T00:00:00Z",
+ "* Ended: 1970-01-01T00:03:20Z",
+ "* Duration: 3 minutes",
+ }, "\r\n"),
+ strings.Join([]string{
+ "--------",
+ "Messages",
+ "--------",
+ "",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "message",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "Uploaded file test1-attachment",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "Deleted file test1-attachment FileDeleted 1970-01-01T00:03:20Z",
+ "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
+ "message",
+ "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
+ "delete message Deleted 1970-01-01T00:03:20Z",
+ }, "\r\n"),
+ strings.Join([]string{
+ "Content-Disposition: attachment; filename=\"test1-attachment\"",
+ "Content-Transfer-Encoding: base64",
+ "Content-Type: application/octet-stream; name=\"test1-attachment\"",
+ }, "\r\n"),
+ }},
+
+ expectedHTMLs: [][]string{{
+ strings.Join([]string{
+ " ",
+ " - TeamId:team-id
",
+ " - TeamName:team-name
",
+ " - TeamDisplayName:team-display-name=",
+ "li>",
+ "
- ChannelId:channel-id
",
+ " - ChannelName:channel-name
",
+ " - ChannelDisplayName:channel-display-=",
+ "name
",
+ " - Started:1970-01-01T00:00:00Z
",
+ " - Ended:1970-01-01T00:03:20Z
",
+ " - Duration:3 minutes
",
+ "
",
+ }, "\r\n"),
+ strings.Join([]string{
+ "",
+ " | test1 | ",
+ " @test | ",
+ " user | ",
+ " test1@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 3 | ",
+ "
",
+ "", "",
+ " | test2 | ",
+ " @test2 | ",
+ " user | ",
+ " test2@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:01:40Z | ",
+ " 2 minutes | ",
+ " 0 | ",
+ "
",
+ "", "",
+ " | test3 | ",
+ " @test3 | ",
+ " user | ",
+ " test3@test.com | ",
+ " 1970-01-01T00:00:00Z | ",
+ " 1970-01-01T00:03:20Z | ",
+ " 3 minutes | ",
+ " 0 | ",
+ "
",
+ }, "\r\n"),
+
+ strings.Join([]string{
+ "",
+ " post-id-1",
+ " 1970-01-01T00:00:00Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " message",
+ "",
+ "", "",
+ " post-id-1",
+ " 1970-01-01T00:00:00Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " Uploaded file test1-attachment",
+ "",
+ "", "",
+ " post-id-1",
+ " 1970-01-01T00:00:00Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " Deleted file test1-attachment",
+ " FileDeleted",
+ " 1970-01-01T00:03:20Z",
+ " ",
+ "",
+ "", "",
+ " post-id-2",
+ " 1970-01-01T00:01:40Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " message",
+ "",
+ "", "",
+ " post-id-2",
+ " 1970-01-01T00:01:40Z",
+ " @test1",
+ " test1",
+ " @test1",
+ " user",
+ " (test1@test.com)",
+ " delete message",
+ " Deleted",
+ " 1970-01-01T00:03:20Z",
+ " ",
+ "",
+ }, "\r\n"),
+ }},
+ expectedWarnings: 0,
+ numExpectedEmls: 1,
+ },
+
+ {
+ name: "posts with deleted attachments, no deleted post, and at different time from original post",
+ cmhs: map[string][]*model.ChannelMemberHistoryResult{
+ "channel-id": {
+ {
+ JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
+ },
+ {
+ JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
+ },
+ {
+ JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
+ },
+ },
+ },
+ metadata: map[string]*shared.MetadataChannel{
+ "channel-id": {
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: "channel-id",
+ ChannelName: "channel-name",
+ ChannelDisplayName: "channel-display-name",
+ ChannelType: chanTypeDirect,
+ RoomId: "direct - channel-id",
+ StartTime: 1,
+ EndTime: 100,
+ },
+ },
+ startTime: 1,
+ endTime: 100_000,
+ posts: []*model.MessageExport{
+ {
+ PostId: model.NewPointer("post-id-1"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(1)),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostMessage: model.NewPointer("message"),
+ PostProps: model.NewPointer("{}"),
+ PostType: model.NewPointer(""),
+ UserEmail: model.NewPointer("test1@test.com"),
+ UserId: model.NewPointer("test1"),
+ Username: model.NewPointer("test1"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{"test1"},
+ },
+ {
+ PostId: model.NewPointer("post-id-2"),
+ PostRootId: model.NewPointer("post-id-1"),
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: model.NewPointer("channel-id"),
+ ChannelName: model.NewPointer("channel-name"),
+ ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(100_000)),
+ PostCreateAt: model.NewPointer(int64(100_000)),
PostMessage: model.NewPointer("message"),
PostProps: model.NewPointer("{}"),
PostType: model.NewPointer(""),
@@ -292,10 +2022,18 @@ func TestGlobalRelayExport(t *testing.T) {
Id: "test1-attachment",
Path: "test1-attachment",
CreateAt: 1,
+ UpdateAt: 200_000,
+ DeleteAt: 200_000,
},
},
},
- expectedHeaders: []string{
+ attachmentsContent: map[string]string{
+ "test1-attachment": "this is the attachment content",
+ },
+ expectedAttachmentContent: [][]string{
+ {base64.StdEncoding.EncodeToString([]byte("this is the attachment content"))},
+ },
+ expectedHeaders: [][]string{{
"MIME-Version: 1.0",
"X-Mattermost-ChannelType: direct",
"Content-Transfer-Encoding: 8bit",
@@ -308,11 +2046,16 @@ func TestGlobalRelayExport(t *testing.T) {
"From: test1@test.com",
"To: test1@test.com,test2@test.com",
"Subject: Mattermost Compliance Export: channel-display-name",
- },
+ }},
- expectedTexts: []string{
+ expectedTexts: [][]string{{
strings.Join([]string{
- "* Channel: channel-display-name",
+ "* TeamId: team-id",
+ "* TeamName: team-name",
+ "* TeamDisplayName: team-display-name",
+ "* ChannelId: channel-id",
+ "* ChannelName: channel-name",
+ "* ChannelDisplayName: channel-display-name",
"* Started: 1970-01-01T00:00:00Z",
"* Ended: 1970-01-01T00:01:40Z",
"* Duration: 2 minutes",
@@ -322,32 +2065,41 @@ func TestGlobalRelayExport(t *testing.T) {
"Messages",
"--------",
"",
- "* 1970-01-01T00:00:00Z @test1 user (test1@test.com): message",
- "* 1970-01-01T00:00:00Z @test1 user (test1@test.com): Uploaded file test1-at=",
- "tachment",
- "* 1970-01-01T00:01:40Z @test1 user (test1@test.com): message",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "message",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "Uploaded file test1-attachment",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "Deleted file test1-attachment FileDeleted 1970-01-01T00:03:20Z",
+ "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
+ "message",
}, "\r\n"),
strings.Join([]string{
"Content-Disposition: attachment; filename=\"test1-attachment\"",
"Content-Transfer-Encoding: base64",
"Content-Type: application/octet-stream; name=\"test1-attachment\"",
}, "\r\n"),
- },
+ }},
- expectedHTMLs: []string{
+ expectedHTMLs: [][]string{{
strings.Join([]string{
" ",
- " - Channel: channel-display-name<=",
- "/li>",
- "
- Started: 1970-01-01T00:00:00Z<=",
- "/li>",
- "
- Ended: 1970-01-01T00:01:40Z",
- "
- Duration: 2 minutes
",
+ " - TeamId:team-id
",
+ " - TeamName:team-name
",
+ " - TeamDisplayName:team-display-name=",
+ "li>",
+ "
- ChannelId:channel-id
",
+ " - ChannelName:channel-name
",
+ " - ChannelDisplayName:channel-display-=",
+ "name
",
+ " - Started:1970-01-01T00:00:00Z
",
+ " - Ended:1970-01-01T00:01:40Z
",
+ " - Duration:2 minutes
",
"
",
}, "\r\n"),
strings.Join([]string{
"",
+ " | test1 | ",
" @test | ",
" user | ",
" test1@test.com | ",
@@ -358,6 +2110,7 @@ func TestGlobalRelayExport(t *testing.T) {
"
",
"",
"",
+ " | test2 | ",
" @test2 | ",
" user | ",
" test2@test.com | ",
@@ -368,6 +2121,7 @@ func TestGlobalRelayExport(t *testing.T) {
"
",
"",
"",
+ " | test3 | ",
" @test3 | ",
" user | ",
" test3@test.com | ",
@@ -380,238 +2134,88 @@ func TestGlobalRelayExport(t *testing.T) {
strings.Join([]string{
"",
+ " post-id-1",
" 1970-01-01T00:00:00Z",
" @test1",
- " ",
+ " test1",
+ " @test1",
" user",
- " (test1@test.com):",
+ " (test1@test.com)",
" message",
- " ",
"",
"",
"",
+ " post-id-1",
" 1970-01-01T00:00:00Z",
" @test1",
- " ",
+ " test1",
+ " @test1",
" user",
- " (test1@test.com):",
+ " (test1@test.com)",
" Uploaded file test1-attachment",
- " ",
"",
"",
"",
- " 1970-01-01T00:01:40Z",
- " @test1",
- " ",
- " user",
- " (test1@test.com):",
- " message",
- " ",
- "",
- }, "\r\n"),
- },
- expectedFiles: 1,
- expectedWarnings: 0,
- },
- {
- name: "posts with deleted attachments",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(400)),
- },
- {
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(80)),
- },
- {
- JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
- },
- },
- },
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id-1"),
- PostOriginalId: model.NewPointer("post-original-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"test1"},
- },
- {
- PostId: model.NewPointer("post-id-2"),
- PostOriginalId: model.NewPointer("post-original-id"),
- PostRootId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(100000)),
- PostMessage: model.NewPointer("message"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- },
- attachments: map[string][]*model.FileInfo{
- "post-id-1": {
- {
- Name: "test1-attachment",
- Id: "test1-attachment",
- Path: "test1-attachment",
- DeleteAt: 200000,
- },
- },
- },
- expectedHeaders: []string{
- "MIME-Version: 1.0",
- "X-Mattermost-ChannelType: direct",
- "Content-Transfer-Encoding: 8bit",
- "Precedence: bulk",
- "X-GlobalRelay-MsgType: Mattermost",
- "X-Mattermost-ChannelID: channel-id",
- "X-Mattermost-ChannelName: channel-display-name",
- "Auto-Submitted: auto-generated",
- "Date: Thu, 01 Jan 1970 00:01:40 +0000",
- "From: test1@test.com",
- "To: test1@test.com,test2@test.com",
- "Subject: Mattermost Compliance Export: channel-display-name",
- },
-
- expectedTexts: []string{
- strings.Join([]string{
- "* Channel: channel-display-name",
- "* Started: 1970-01-01T00:00:00Z",
- "* Ended: 1970-01-01T00:01:40Z",
- "* Duration: 2 minutes",
- }, "\r\n"),
- strings.Join([]string{
- "--------",
- "Messages",
- "--------",
- "",
- "* 1970-01-01T00:00:00Z @test1 user (test1@test.com): message",
- "* 1970-01-01T00:01:40Z @test1 user (test1@test.com): message",
- "* 1970-01-01T00:03:20Z @test1 user (test1@test.com): Deleted file test1-att=",
- "achment",
- }, "\r\n"),
- strings.Join([]string{
- "Content-Disposition: attachment; filename=\"test1-attachment\"",
- "Content-Transfer-Encoding: base64",
- "Content-Type: application/octet-stream; name=\"test1-attachment\"",
- }, "\r\n"),
- },
-
- expectedHTMLs: []string{
- strings.Join([]string{
- " ",
- " - Channel: channel-display-name<=",
- "/li>",
- "
- Started: 1970-01-01T00:00:00Z<=",
- "/li>",
- "
- Ended: 1970-01-01T00:01:40Z",
- "
- Duration: 2 minutes
",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "
",
- " | @test | ",
- " user | ",
- " test1@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 2 | ",
- "
",
- "",
- "",
- " | @test2 | ",
- " user | ",
- " test2@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- "",
- "",
- " | @test3 | ",
- " user | ",
- " test3@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- }, "\r\n"),
-
- strings.Join([]string{
- "",
+ " post-id-1",
" 1970-01-01T00:00:00Z",
" @test1",
- " ",
+ " test1",
+ " @test1",
" user",
- " (test1@test.com):",
- " message",
- " ",
+ " (test1@test.com)",
+ " Deleted file test1-attachment",
+ " FileDeleted",
+ " 1970-01-01T00:03:20Z",
+ " ",
"",
"",
"",
+ " post-id-2",
" 1970-01-01T00:01:40Z",
" @test1",
- " ",
+ " test1",
+ " @test1",
" user",
- " (test1@test.com):",
+ " (test1@test.com)",
" message",
- " ",
- "",
- "",
- "",
- " 1970-01-01T00:03:20Z",
- " @test1",
- " ",
- " user",
- " (test1@test.com):",
- " Deleted file test1-attachment",
- " ",
"",
}, "\r\n"),
- },
- expectedFiles: 1,
+ }},
expectedWarnings: 0,
+ numExpectedEmls: 1,
},
+
{
name: "posts with missing attachments",
cmhs: map[string][]*model.ChannelMemberHistoryResult{
"channel-id": {
{
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(400)),
+ JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
},
{
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(80)),
+ JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
},
{
JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
},
},
},
+ metadata: map[string]*shared.MetadataChannel{
+ "channel-id": {
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: "channel-id",
+ ChannelName: "channel-name",
+ ChannelDisplayName: "channel-display-name",
+ ChannelType: chanTypeDirect,
+ RoomId: "direct - channel-id",
+ StartTime: 1,
+ EndTime: 100,
+ },
+ },
+ startTime: 1,
+ endTime: 100_000,
posts: []*model.MessageExport{
{
PostId: model.NewPointer("post-id-1"),
@@ -622,6 +2226,7 @@ func TestGlobalRelayExport(t *testing.T) {
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(1)),
PostCreateAt: model.NewPointer(int64(1)),
PostMessage: model.NewPointer("message"),
PostProps: model.NewPointer("{}"),
@@ -642,7 +2247,8 @@ func TestGlobalRelayExport(t *testing.T) {
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(100000)),
+ PostUpdateAt: model.NewPointer(int64(100_000)),
+ PostCreateAt: model.NewPointer(int64(100_000)),
PostMessage: model.NewPointer("message"),
PostProps: model.NewPointer("{}"),
PostType: model.NewPointer(""),
@@ -663,7 +2269,7 @@ func TestGlobalRelayExport(t *testing.T) {
},
},
},
- expectedHeaders: []string{
+ expectedHeaders: [][]string{{
"MIME-Version: 1.0",
"X-Mattermost-ChannelType: direct",
"Content-Transfer-Encoding: 8bit",
@@ -676,11 +2282,16 @@ func TestGlobalRelayExport(t *testing.T) {
"From: test1@test.com",
"To: test1@test.com,test2@test.com",
"Subject: Mattermost Compliance Export: channel-display-name",
- },
+ }},
- expectedTexts: []string{
+ expectedTexts: [][]string{{
strings.Join([]string{
- "* Channel: channel-display-name",
+ "* TeamId: team-id",
+ "* TeamName: team-name",
+ "* TeamDisplayName: team-display-name",
+ "* ChannelId: channel-id",
+ "* ChannelName: channel-name",
+ "* ChannelDisplayName: channel-display-name",
"* Started: 1970-01-01T00:00:00Z",
"* Ended: 1970-01-01T00:01:40Z",
"* Duration: 2 minutes",
@@ -690,32 +2301,39 @@ func TestGlobalRelayExport(t *testing.T) {
"Messages",
"--------",
"",
- "* 1970-01-01T00:00:00Z @test1 user (test1@test.com): message",
- "* 1970-01-01T00:00:00Z @test1 user (test1@test.com): Uploaded file test1-at=",
- "tachment",
- "* 1970-01-01T00:01:40Z @test1 user (test1@test.com): message",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "message",
+ "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
+ "Uploaded file test1-attachment",
+ "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
+ "message",
}, "\r\n"),
strings.Join([]string{
"Content-Disposition: attachment; filename=\"test1-attachment\"",
"Content-Transfer-Encoding: base64",
"Content-Type: application/octet-stream; name=\"test1-attachment\"",
}, "\r\n"),
- },
+ }},
- expectedHTMLs: []string{
+ expectedHTMLs: [][]string{{
strings.Join([]string{
" ",
- " - Channel: channel-display-name<=",
- "/li>",
- "
- Started: 1970-01-01T00:00:00Z<=",
- "/li>",
- "
- Ended: 1970-01-01T00:01:40Z",
- "
- Duration: 2 minutes
",
+ " - TeamId:team-id
",
+ " - TeamName:team-name
",
+ " - TeamDisplayName:team-display-name=",
+ "li>",
+ "
- ChannelId:channel-id
",
+ " - ChannelName:channel-name
",
+ " - ChannelDisplayName:channel-display-=",
+ "name
",
+ " - Started:1970-01-01T00:00:00Z
",
+ " - Ended:1970-01-01T00:01:40Z
",
+ " - Duration:2 minutes
",
"
",
}, "\r\n"),
strings.Join([]string{
"",
+ " | test1 | ",
" @test | ",
" user | ",
" test1@test.com | ",
@@ -726,6 +2344,7 @@ func TestGlobalRelayExport(t *testing.T) {
"
",
"",
"",
+ " | test2 | ",
" @test2 | ",
" user | ",
" test2@test.com | ",
@@ -736,6 +2355,7 @@ func TestGlobalRelayExport(t *testing.T) {
"
",
"",
"",
+ " | test3 | ",
" @test3 | ",
" user | ",
" test3@test.com | ",
@@ -748,54 +2368,74 @@ func TestGlobalRelayExport(t *testing.T) {
strings.Join([]string{
"",
+ " post-id-1",
" 1970-01-01T00:00:00Z",
" @test1",
- " ",
+ " test1",
+ " @test1",
" user",
- " (test1@test.com):",
+ " (test1@test.com)",
" message",
- " ",
"",
"",
"",
+ " post-id-1",
" 1970-01-01T00:00:00Z",
" @test1",
- " ",
+ " test1",
+ " @test1",
" user",
- " (test1@test.com):",
+ " (test1@test.com)",
" Uploaded file test1-attachment",
- " ",
"",
"",
"",
+ " post-id-2",
" 1970-01-01T00:01:40Z",
" @test1",
- " ",
+ " test1",
+ " @test1",
" user",
- " (test1@test.com):",
+ " (test1@test.com)",
" message",
- " ",
"",
}, "\r\n"),
- },
- expectedFiles: 1,
+ }},
expectedWarnings: 1,
+ numExpectedEmls: 1,
},
+
{
name: "posts with override_username property",
cmhs: map[string][]*model.ChannelMemberHistoryResult{
"channel-id": {
{
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(400)),
+ JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
},
{
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(80)),
+ JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
},
{
JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
},
},
},
+ metadata: map[string]*shared.MetadataChannel{
+ "channel-id": {
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: "channel-id",
+ ChannelName: "channel-name",
+ ChannelDisplayName: "channel-display-name",
+ ChannelType: chanTypeDirect,
+ RoomId: "direct - channel-id",
+ StartTime: 1,
+ EndTime: 100,
+ },
+ },
+ startTime: 1,
+ endTime: 100_000,
posts: []*model.MessageExport{
{
PostId: model.NewPointer("post-id"),
@@ -806,6 +2446,7 @@ func TestGlobalRelayExport(t *testing.T) {
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(1)),
PostCreateAt: model.NewPointer(int64(1)),
PostMessage: model.NewPointer("message"),
PostProps: model.NewPointer("{}"),
@@ -826,7 +2467,8 @@ func TestGlobalRelayExport(t *testing.T) {
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(100000)),
+ PostUpdateAt: model.NewPointer(int64(100_000)),
+ PostCreateAt: model.NewPointer(int64(100_000)),
PostMessage: model.NewPointer("message"),
PostProps: model.NewPointer("{\"from_webhook\":\"true\",\"html\":\"Test HTML\",\"override_username\":\"test_username_override\",\"webhook_display_name\":\"Test Bot\"}"),
PostType: model.NewPointer(""),
@@ -838,7 +2480,7 @@ func TestGlobalRelayExport(t *testing.T) {
},
},
attachments: map[string][]*model.FileInfo{},
- expectedHeaders: []string{
+ expectedHeaders: [][]string{{
"MIME-Version: 1.0",
"X-Mattermost-ChannelType: direct",
"Content-Transfer-Encoding: 8bit",
@@ -851,11 +2493,15 @@ func TestGlobalRelayExport(t *testing.T) {
"From: test1@test.com",
"To: test1@test.com,test2@test.com",
"Subject: Mattermost Compliance Export: channel-display-name",
- },
-
- expectedTexts: []string{
+ }},
+ expectedTexts: [][]string{{
strings.Join([]string{
- "* Channel: channel-display-name",
+ "* TeamId: team-id",
+ "* TeamName: team-name",
+ "* TeamDisplayName: team-display-name",
+ "* ChannelId: channel-id",
+ "* ChannelName: channel-name",
+ "* ChannelDisplayName: channel-display-name",
"* Started: 1970-01-01T00:00:00Z",
"* Ended: 1970-01-01T00:01:40Z",
"* Duration: 2 minutes",
@@ -865,25 +2511,32 @@ func TestGlobalRelayExport(t *testing.T) {
"Messages",
"--------",
"",
- "* 1970-01-01T00:00:00Z @test1 user (test1@test.com): message",
- "* 1970-01-01T00:01:40Z @test1 @test_username_override user (test1@test.com): message",
+ "* post-id 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) me=",
+ "ssage",
+ "* post-id 1970-01-01T00:01:40Z @test1 test1 @test_username_override user (t=",
+ "est1@test.com) message",
}, "\r\n"),
- },
+ }},
- expectedHTMLs: []string{
+ expectedHTMLs: [][]string{{
strings.Join([]string{
" ",
- " - Channel: channel-display-name<=",
- "/li>",
- "
- Started: 1970-01-01T00:00:00Z<=",
- "/li>",
- "
- Ended: 1970-01-01T00:01:40Z",
- "
- Duration: 2 minutes
",
+ " - TeamId:team-id
",
+ " - TeamName:team-name
",
+ " - TeamDisplayName:team-display-name=",
+ "li>",
+ "
- ChannelId:channel-id
",
+ " - ChannelName:channel-name
",
+ " - ChannelDisplayName:channel-display-=",
+ "name
",
+ " - Started:1970-01-01T00:00:00Z
",
+ " - Ended:1970-01-01T00:01:40Z
",
+ " - Duration:2 minutes
",
"
",
}, "\r\n"),
strings.Join([]string{
"
",
+ " | test1 | ",
" @test | ",
" user | ",
" test1@test.com | ",
@@ -894,6 +2547,7 @@ func TestGlobalRelayExport(t *testing.T) {
"
",
"",
"",
+ " | test2 | ",
" @test2 | ",
" user | ",
" test2@test.com | ",
@@ -904,6 +2558,7 @@ func TestGlobalRelayExport(t *testing.T) {
"
",
"",
"",
+ " | test3 | ",
" @test3 | ",
" user | ",
" test3@test.com | ",
@@ -916,42 +2571,62 @@ func TestGlobalRelayExport(t *testing.T) {
strings.Join([]string{
"",
+ " post-id",
" 1970-01-01T00:00:00Z",
" @test1",
- " ",
+ " test1",
+ " @test1",
" user",
- " (test1@test.com):",
+ " (test1@test.com)",
" message",
- " ",
"",
"",
"",
+ " post-id",
" 1970-01-01T00:01:40Z",
" @test1",
+ " test1",
" @test_username_override",
" user",
- " (test1@test.com):",
+ " (test1@test.com)",
" message",
- " ",
"",
}, "\r\n"),
- },
+ }},
+ numExpectedEmls: 1,
},
+
{
name: "posts with webhook_display_name property",
cmhs: map[string][]*model.ChannelMemberHistoryResult{
"channel-id": {
{
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(400)),
+ JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
},
{
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(80)),
+ JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
},
{
JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
},
},
},
+ metadata: map[string]*shared.MetadataChannel{
+ "channel-id": {
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: "channel-id",
+ ChannelName: "channel-name",
+ ChannelDisplayName: "channel-display-name",
+ ChannelType: chanTypeDirect,
+ RoomId: "direct - channel-id",
+ StartTime: 1,
+ EndTime: 100,
+ },
+ },
+ startTime: 1,
+ endTime: 100_000,
posts: []*model.MessageExport{
{
PostId: model.NewPointer("post-id"),
@@ -962,6 +2637,7 @@ func TestGlobalRelayExport(t *testing.T) {
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(1)),
PostCreateAt: model.NewPointer(int64(1)),
PostMessage: model.NewPointer("message"),
PostProps: model.NewPointer("{}"),
@@ -982,7 +2658,8 @@ func TestGlobalRelayExport(t *testing.T) {
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(100000)),
+ PostUpdateAt: model.NewPointer(int64(100_000)),
+ PostCreateAt: model.NewPointer(int64(100_000)),
PostMessage: model.NewPointer("message"),
PostProps: model.NewPointer("{\"from_webhook\":\"true\",\"webhook_display_name\":\"Test Bot\"}"),
PostType: model.NewPointer(""),
@@ -994,7 +2671,7 @@ func TestGlobalRelayExport(t *testing.T) {
},
},
attachments: map[string][]*model.FileInfo{},
- expectedHeaders: []string{
+ expectedHeaders: [][]string{{
"MIME-Version: 1.0",
"X-Mattermost-ChannelType: direct",
"Content-Transfer-Encoding: 8bit",
@@ -1007,11 +2684,16 @@ func TestGlobalRelayExport(t *testing.T) {
"From: test1@test.com",
"To: test1@test.com,test2@test.com",
"Subject: Mattermost Compliance Export: channel-display-name",
- },
+ }},
- expectedTexts: []string{
+ expectedTexts: [][]string{{
strings.Join([]string{
- "* Channel: channel-display-name",
+ "* TeamId: team-id",
+ "* TeamName: team-name",
+ "* TeamDisplayName: team-display-name",
+ "* ChannelId: channel-id",
+ "* ChannelName: channel-name",
+ "* ChannelDisplayName: channel-display-name",
"* Started: 1970-01-01T00:00:00Z",
"* Ended: 1970-01-01T00:01:40Z",
"* Duration: 2 minutes",
@@ -1021,25 +2703,32 @@ func TestGlobalRelayExport(t *testing.T) {
"Messages",
"--------",
"",
- "* 1970-01-01T00:00:00Z @test1 user (test1@test.com): message",
- "* 1970-01-01T00:01:40Z @test1 @Test Bot user (test1@test.com): message",
+ "* post-id 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) me=",
+ "ssage",
+ "* post-id 1970-01-01T00:01:40Z @test1 test1 @Test Bot user (test1@test.com)=",
+ " message",
}, "\r\n"),
- },
+ }},
- expectedHTMLs: []string{
+ expectedHTMLs: [][]string{{
strings.Join([]string{
" ",
- " - Channel: channel-display-name<=",
- "/li>",
- "
- Started: 1970-01-01T00:00:00Z<=",
- "/li>",
- "
- Ended: 1970-01-01T00:01:40Z",
- "
- Duration: 2 minutes
",
+ " - TeamId:team-id
",
+ " - TeamName:team-name
",
+ " - TeamDisplayName:team-display-name=",
+ "li>",
+ "
- ChannelId:channel-id
",
+ " - ChannelName:channel-name
",
+ " - ChannelDisplayName:channel-display-=",
+ "name
",
+ " - Started:1970-01-01T00:00:00Z
",
+ " - Ended:1970-01-01T00:01:40Z
",
+ " - Duration:2 minutes
",
"
",
}, "\r\n"),
strings.Join([]string{
"
",
+ " | test1 | ",
" @test | ",
" user | ",
" test1@test.com | ",
@@ -1050,6 +2739,7 @@ func TestGlobalRelayExport(t *testing.T) {
"
",
"",
"",
+ " | test2 | ",
" @test2 | ",
" user | ",
" test2@test.com | ",
@@ -1060,6 +2750,7 @@ func TestGlobalRelayExport(t *testing.T) {
"
",
"",
"",
+ " | test3 | ",
" @test3 | ",
" user | ",
" test3@test.com | ",
@@ -1072,42 +2763,62 @@ func TestGlobalRelayExport(t *testing.T) {
strings.Join([]string{
"",
+ " post-id",
" 1970-01-01T00:00:00Z",
" @test1",
- " ",
+ " test1",
+ " @test1",
" user",
- " (test1@test.com):",
+ " (test1@test.com)",
" message",
- " ",
"",
"",
"",
+ " post-id",
" 1970-01-01T00:01:40Z",
" @test1",
+ " test1",
" @Test Bot",
" user",
- " (test1@test.com):",
+ " (test1@test.com)",
" message",
- " ",
"",
}, "\r\n"),
- },
+ }},
+ numExpectedEmls: 1,
},
+
{
name: "post with permalink preview",
cmhs: map[string][]*model.ChannelMemberHistoryResult{
"channel-id": {
{
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(400)),
+ JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
},
{
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(80)),
+ JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
},
{
JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
},
},
},
+ metadata: map[string]*shared.MetadataChannel{
+ "channel-id": {
+ TeamId: model.NewPointer("team-id"),
+ TeamName: model.NewPointer("team-name"),
+ TeamDisplayName: model.NewPointer("team-display-name"),
+ ChannelId: "channel-id",
+ ChannelName: "channel-name",
+ ChannelDisplayName: "channel-display-name",
+ ChannelType: chanTypeDirect,
+ RoomId: "direct - channel-id",
+ StartTime: 1,
+ EndTime: 100,
+ },
+ },
+ startTime: 1,
+ endTime: 100_000,
posts: []*model.MessageExport{
{
PostId: model.NewPointer("post-id"),
@@ -1118,6 +2829,7 @@ func TestGlobalRelayExport(t *testing.T) {
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
+ PostUpdateAt: model.NewPointer(int64(1)),
PostCreateAt: model.NewPointer(int64(1)),
PostMessage: model.NewPointer("message"),
PostProps: model.NewPointer(`{"previewed_post":"o4w39mc1ff8y5fite4b8hacy1x"}`),
@@ -1138,7 +2850,8 @@ func TestGlobalRelayExport(t *testing.T) {
ChannelId: model.NewPointer("channel-id"),
ChannelName: model.NewPointer("channel-name"),
ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(100000)),
+ PostUpdateAt: model.NewPointer(int64(100_000)),
+ PostCreateAt: model.NewPointer(int64(100_000)),
PostMessage: model.NewPointer("message"),
PostProps: model.NewPointer("{}"),
PostType: model.NewPointer(""),
@@ -1150,7 +2863,7 @@ func TestGlobalRelayExport(t *testing.T) {
},
},
attachments: map[string][]*model.FileInfo{},
- expectedHeaders: []string{
+ expectedHeaders: [][]string{{
"MIME-Version: 1.0",
"X-Mattermost-ChannelType: direct",
"Content-Transfer-Encoding: 8bit",
@@ -1163,11 +2876,16 @@ func TestGlobalRelayExport(t *testing.T) {
"From: test1@test.com",
"To: test1@test.com,test2@test.com",
"Subject: Mattermost Compliance Export: channel-display-name",
- },
+ }},
- expectedTexts: []string{
+ expectedTexts: [][]string{{
strings.Join([]string{
- "* Channel: channel-display-name",
+ "* TeamId: team-id",
+ "* TeamName: team-name",
+ "* TeamDisplayName: team-display-name",
+ "* ChannelId: channel-id",
+ "* ChannelName: channel-name",
+ "* ChannelDisplayName: channel-display-name",
"* Started: 1970-01-01T00:00:00Z",
"* Ended: 1970-01-01T00:01:40Z",
"* Duration: 2 minutes",
@@ -1177,26 +2895,32 @@ func TestGlobalRelayExport(t *testing.T) {
"Messages",
"--------",
"",
- "* 1970-01-01T00:00:00Z @test1 user (test1@test.com): message o4w39mc1ff8y5f=",
- "ite4b8hacy1x",
- "* 1970-01-01T00:01:40Z @test1 user (test1@test.com): message",
+ "* post-id 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) me=",
+ "ssage o4w39mc1ff8y5fite4b8hacy1x",
+ "* post-id 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) me=",
+ "ssage",
}, "\r\n"),
- },
+ }},
- expectedHTMLs: []string{
+ expectedHTMLs: [][]string{{
strings.Join([]string{
" ",
- " - Channel: channel-display-name<=",
- "/li>",
- "
- Started: 1970-01-01T00:00:00Z<=",
- "/li>",
- "
- Ended: 1970-01-01T00:01:40Z",
- "
- Duration: 2 minutes
",
+ " - TeamId:team-id
",
+ " - TeamName:team-name
",
+ " - TeamDisplayName:team-display-name=",
+ "li>",
+ "
- ChannelId:channel-id
",
+ " - ChannelName:channel-name
",
+ " - ChannelDisplayName:channel-display-=",
+ "name
",
+ " - Started:1970-01-01T00:00:00Z
",
+ " - Ended:1970-01-01T00:01:40Z
",
+ " - Duration:2 minutes
",
"
",
}, "\r\n"),
strings.Join([]string{
"
",
+ " | test1 | ",
" @test | ",
" user | ",
" test1@test.com | ",
@@ -1207,6 +2931,7 @@ func TestGlobalRelayExport(t *testing.T) {
"
",
"",
"",
+ " | test2 | ",
" @test2 | ",
" user | ",
" test2@test.com | ",
@@ -1217,6 +2942,7 @@ func TestGlobalRelayExport(t *testing.T) {
"
",
"",
"",
+ " | test3 | ",
" @test3 | ",
" user | ",
" test3@test.com | ",
@@ -1229,103 +2955,172 @@ func TestGlobalRelayExport(t *testing.T) {
strings.Join([]string{
"",
+ " post-id",
" 1970-01-01T00:00:00Z",
" @test1",
- " ",
+ " test1",
+ " @test1",
" user",
- " (test1@test.com):",
+ " (test1@test.com)",
" message",
" o4w39mc1ff8y5fite4b8hacy1x",
"",
"",
"",
+ " post-id",
" 1970-01-01T00:01:40Z",
" @test1",
- " ",
+ " test1",
+ " @test1",
" user",
- " (test1@test.com):",
+ " (test1@test.com)",
" message",
- " ",
"",
}, "\r\n"),
- },
- expectedFiles: 1,
+ }},
expectedWarnings: 0,
+ numExpectedEmls: 1,
},
}
- for _, tt := range csvExportTests {
+ for _, tt := range grExportTests {
t.Run(tt.name, func(t *testing.T) {
mockStore := &storetest.Store{}
defer mockStore.AssertExpectations(t)
if len(tt.attachments) > 0 {
- for post_id, attachments := range tt.attachments {
- attachments := attachments // TODO: Remove once go1.22 is used
- call := mockStore.FileInfoStore.On("GetForPost", post_id, true, true, false)
+ for postId, attachments := range tt.attachments {
+ call := mockStore.FileInfoStore.On("GetForPost", postId, true, true, false)
call.Run(func(args mock.Arguments) {
call.Return(attachments, nil)
})
- if tt.expectedWarnings == 0 {
- _, err = fileBackend.WriteFile(bytes.NewReader([]byte{}), attachments[0].Path)
- require.NoError(t, err)
+ for _, attachment := range attachments {
+ if tt.expectedWarnings == 0 {
+ content, ok := tt.attachmentsContent[attachment.Id]
+ require.True(t, ok, "attachment not found for id: %s", attachment.Id)
+ _, err = fileBackend.WriteFile(strings.NewReader(content), attachment.Path)
+ require.NoError(t, err)
- t.Cleanup(func() {
- err = fileBackend.RemoveFile(attachments[0].Path)
- assert.NoError(t, err)
+ t.Cleanup(func() {
+ err = fileBackend.RemoveFile(attachment.Path)
+ assert.NoError(t, err)
+ })
+ }
+ }
+ }
+ }
+
+ if tt.maxEmailBytes > 0 {
+ origMaxEmailBytes := MaxEmailBytes
+ MaxEmailBytes = tt.maxEmailBytes
+ t.Cleanup(func() {
+ MaxEmailBytes = origMaxEmailBytes
+ })
+ }
+
+ exportFileName := path.Join("export", "jobName", "jobName-batch001.zip")
+ results, err := GlobalRelayExport(rctx, shared.ExportParams{
+ ExportType: model.ComplianceExportTypeGlobalrelayZip,
+ ChannelMetadata: tt.metadata,
+ Posts: tt.posts,
+ ChannelMemberHistories: tt.cmhs,
+ BatchPath: exportFileName,
+ BatchStartTime: tt.startTime,
+ BatchEndTime: tt.endTime,
+ Db: shared.NewMessageExportStore(mockStore),
+ FileAttachmentBackend: fileBackend,
+ ExportBackend: exportBackend,
+ Templates: templatesContainer,
+ })
+ assert.NoError(t, err)
+ assert.Equal(t, tt.expectedWarnings, results.NumWarnings)
+
+ // channel-name - (channel-id) - 0.eml
+ if !tt.empty {
+ if tt.numExpectedEmls == 0 {
+ require.True(t, false, "need numExpectedEmls to be at least 1")
+ }
+ openZipAndAssertNumEmls(t, exportBackend, exportFileName, tt.numExpectedEmls)
+
+ for batchNum := 0; batchNum < tt.numExpectedEmls; batchNum++ {
+ metadata := tt.metadata["channel-id"]
+ emlName := fmt.Sprintf("%s - (%s) - %d.eml", metadata.ChannelName,
+ metadata.ChannelId, batchNum)
+ eml := openZipAndReadFileStartingWith(t, exportBackend, exportFileName, emlName)
+
+ // Note: for debugging, better keep this in case we need it again.
+ //t.Logf("<><> batchNum %d actual: \n%s", batchNum, eml)
+
+ t.Run("headers", func(t *testing.T) {
+ for _, expectedHeader := range tt.expectedHeaders[batchNum] {
+ assert.Contains(t, eml, expectedHeader, "batchNum %d, expected: %s", batchNum, expectedHeader)
+ }
+ })
+
+ t.Run("text-version", func(t *testing.T) {
+ for _, expectedText := range tt.expectedTexts[batchNum] {
+ assert.Contains(t, eml, expectedText, "batchNum %d, expected: %s", batchNum, expectedText)
+ }
+ if len(tt.notExpectedTexts) > 0 {
+ for _, notExpectedText := range tt.notExpectedTexts[batchNum] {
+ assert.NotContains(t, eml, notExpectedText, "batchNum %d, expected: %s", batchNum, notExpectedText)
+ }
+ }
+ })
+
+ t.Run("html-version", func(t *testing.T) {
+ for _, expectedHTML := range tt.expectedHTMLs[batchNum] {
+ assert.Contains(t, eml, expectedHTML, "batchNum %d, \nexpected:\n %s\n\nactual:\n %s\n", batchNum, expectedHTML, eml)
+ }
+ })
+
+ assert.Len(t, tt.expectedAttachmentContent, len(tt.attachmentsContent), "every attachmentsContent should have an expectedAttachmentContent")
+
+ if len(tt.expectedAttachmentContent) > 0 {
+ t.Run("file encoding", func(t *testing.T) {
+ for _, expected := range tt.expectedAttachmentContent[batchNum] {
+ assert.Contains(t, eml, expected, "batchNum %d, \nexpected:\n %s\n\nactual:\n %s\n", batchNum, expected, eml)
+ }
})
}
}
}
-
- if len(tt.cmhs) > 0 {
- for channelId, cmhs := range tt.cmhs {
- mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", int64(1), int64(100000), channelId).Return(cmhs, nil)
- }
- }
-
- dest, err := os.CreateTemp("", "")
- assert.NoError(t, err)
- defer os.Remove(dest.Name())
-
- _, warningCount, appErr := GlobalRelayExport(rctx, tt.posts, mockStore, fileBackend, dest, templatesContainer)
- assert.Nil(t, appErr)
- assert.Equal(t, int64(tt.expectedWarnings), warningCount)
-
- _, err = dest.Seek(0, 0)
- assert.NoError(t, err)
-
- destInfo, err := dest.Stat()
- assert.NoError(t, err)
-
- zipFile, err := zip.NewReader(dest, destInfo.Size())
- assert.NoError(t, err)
-
- if tt.expectedFiles > 0 {
- firstFile, err := zipFile.File[0].Open()
- assert.NoError(t, err)
-
- data, err := io.ReadAll(firstFile)
- assert.NoError(t, err)
-
- t.Run("headers", func(t *testing.T) {
- for _, expectedHeader := range tt.expectedHeaders {
- assert.Contains(t, string(data), expectedHeader)
- }
- })
-
- t.Run("text-version", func(t *testing.T) {
- for _, expectedText := range tt.expectedTexts {
- assert.Contains(t, string(data), expectedText)
- }
- })
-
- t.Run("html-version", func(t *testing.T) {
- for _, expectedHTML := range tt.expectedHTMLs {
- assert.Contains(t, string(data), expectedHTML)
- }
- })
- }
})
}
}
+
+func openZipAndReadFileStartingWith(t *testing.T, backend filestore.FileBackend, path string, startsWith string) string {
+ zipBytes, err := backend.ReadFile(path)
+ require.NoError(t, err)
+
+ zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+ require.NoError(t, err)
+
+ var names []string
+ for _, f := range zipReader.File {
+ if strings.HasPrefix(f.Name, startsWith) {
+ file, err := f.Open()
+ require.NoError(t, err)
+ contents, err := io.ReadAll(file)
+ require.NoError(t, err)
+ err = file.Close()
+ require.NoError(t, err)
+
+ return string(contents)
+ }
+ names = append(names, f.Name)
+ }
+
+ require.True(t, false, "called openZipAndReadFileStartingWith but didn't file file starting with: %s. Found: %v", startsWith, names)
+ return ""
+}
+
+func openZipAndAssertNumEmls(t *testing.T, backend filestore.FileBackend, path string, numEmls int) {
+ zipBytes, err := backend.ReadFile(path)
+ require.NoError(t, err)
+
+ zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+ require.NoError(t, err)
+
+ require.Len(t, zipReader.File, numEmls, "numEmls wrong")
+}
diff --git a/server/enterprise/message_export/global_relay_export/test_helpers.go b/server/enterprise/message_export/global_relay_export/test_helpers.go
new file mode 100644
index 0000000000..28bc8aaf97
--- /dev/null
+++ b/server/enterprise/message_export/global_relay_export/test_helpers.go
@@ -0,0 +1,29 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package global_relay_export
+
+import (
+ "net/mail"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func AssertHeaderContains(t *testing.T, msg string, expected map[string]string) {
+ t.Helper()
+ m, err := mail.ReadMessage(strings.NewReader(msg))
+ require.NoError(t, err)
+
+ for k, v := range expected {
+ assert.Equal(t, v, m.Header.Get(k))
+ }
+}
+
+func CleanTestOutput(msg string) string {
+ msg = strings.Replace(msg, "=\r\n", "", -1)
+ msg = strings.Replace(msg, "\r\n", "\n", -1)
+ return msg
+}
diff --git a/server/enterprise/message_export/global_relay_export/to_html.go b/server/enterprise/message_export/global_relay_export/to_html.go
index c8ad4eb8d8..4b59992fe5 100644
--- a/server/enterprise/message_export/global_relay_export/to_html.go
+++ b/server/enterprise/message_export/global_relay_export/to_html.go
@@ -17,9 +17,15 @@ import (
"github.com/mattermost/mattermost/server/v8/platform/shared/templates"
)
+func TimestampConvert(timestampMS int64) string {
+ return time.Unix(timestampMS/1000, 0).UTC().Format(time.RFC3339)
+
+ // for testing: (keep in case we need to be specific -- helps when you have joins and leaves within millis of each other)
+ //return fmt.Sprintf("%d", timestampMS)
+}
+
func channelExportToHTML(rctx request.CTX, channelExport *ChannelExport, t *templates.Container) (string, error) {
durationMilliseconds := channelExport.EndTime - channelExport.StartTime
- // TODO CHECK IF WE NEED THE MILISECONS HERE OR WE CAN ROUND IT DIRECTLY HERE
duration := time.Duration(durationMilliseconds) * time.Millisecond
var participantRowsBuffer bytes.Buffer
@@ -35,7 +41,9 @@ func channelExportToHTML(rctx request.CTX, channelExport *ChannelExport, t *temp
var messagesBuffer bytes.Buffer
sort.Slice(channelExport.Messages, func(i, j int) bool {
if channelExport.Messages[i].SentTime == channelExport.Messages[j].SentTime {
- return !strings.HasPrefix(channelExport.Messages[i].Message, "Uploaded file") && !strings.HasPrefix(channelExport.Messages[i].Message, "Deleted file")
+ return !strings.HasPrefix(channelExport.Messages[i].Message, "Uploaded file") &&
+ !strings.HasPrefix(channelExport.Messages[i].Message, "Deleted file") &&
+ channelExport.Messages[i].UpdateType == ""
}
return channelExport.Messages[i].SentTime < channelExport.Messages[j].SentTime
})
@@ -50,13 +58,18 @@ func channelExportToHTML(rctx request.CTX, channelExport *ChannelExport, t *temp
data := templates.Data{
Props: map[string]any{
- "ChannelName": channelExport.ChannelName,
- "Started": time.Unix(channelExport.StartTime/1000, 0).UTC().Format(time.RFC3339),
- "Ended": time.Unix(channelExport.EndTime/1000, 0).UTC().Format(time.RFC3339),
- "Duration": durafmt.Parse(duration.Round(time.Minute)).String(),
- "ParticipantRows": template.HTML(participantRowsBuffer.String()),
- "Messages": template.HTML(messagesBuffer.String()),
- "ExportDate": time.Unix(channelExport.ExportedOn/1000, 0).UTC().Format(time.RFC3339),
+ "TeamId": channelExport.TeamId,
+ "TeamName": channelExport.TeamName,
+ "TeamDisplayName": channelExport.TeamDisplayName,
+ "ChannelId": channelExport.ChannelId,
+ "ChannelName": channelExport.ChannelName,
+ "ChannelDisplayName": channelExport.ChannelDisplayName,
+ "Started": TimestampConvert(channelExport.StartTime),
+ "Ended": TimestampConvert(channelExport.EndTime),
+ "Duration": durafmt.Parse(duration.Round(time.Minute)).String(),
+ "ParticipantRows": template.HTML(participantRowsBuffer.String()),
+ "Messages": template.HTML(messagesBuffer.String()),
+ "ExportDate": TimestampConvert(channelExport.ExportedOn),
},
}
@@ -65,16 +78,16 @@ func channelExportToHTML(rctx request.CTX, channelExport *ChannelExport, t *temp
func participantToHTML(participant *ParticipantRow, t *templates.Container) (string, error) {
durationMilliseconds := participant.LeaveTime - participant.JoinTime
- // TODO CHECK IF WE NEED THE MILISECONS HERE OR WE CAN ROUND IT DIRECTLY HERE
duration := time.Duration(durationMilliseconds) * time.Millisecond
data := templates.Data{
Props: map[string]any{
+ "UserId": participant.UserId,
"Username": participant.Username,
"UserType": participant.UserType,
- "Email": participant.Email,
- "Joined": time.Unix(participant.JoinTime/1000, 0).UTC().Format(time.RFC3339),
- "Left": time.Unix(participant.LeaveTime/1000, 0).UTC().Format(time.RFC3339),
+ "Email": participant.UserEmail,
+ "Joined": TimestampConvert(participant.JoinTime),
+ "Left": TimestampConvert(participant.LeaveTime),
"Duration": durafmt.Parse(duration.Round(time.Minute)).String(),
"NumMessages": participant.MessagesSent,
},
@@ -90,14 +103,18 @@ func messageToHTML(message *Message, t *templates.Container) (string, error) {
}
data := templates.Data{
Props: map[string]any{
- "SentTime": time.Unix(message.SentTime/1000, 0).UTC().Format(time.RFC3339),
- "Username": message.SenderUsername,
- "PostUsername": postUsername,
- "UserType": message.SenderUserType,
- "PostType": message.PostType,
- "Email": message.SenderEmail,
- "Message": message.Message,
- "PreviewsPost": message.PreviewsPost,
+ "PostId": message.Id,
+ "SentTime": TimestampConvert(message.SentTime),
+ "UserId": message.SenderId,
+ "Username": message.SenderUsername,
+ "PostUsername": postUsername,
+ "UserType": message.SenderUserType,
+ "Email": message.SenderEmail,
+ "Message": message.Message,
+ "PreviewsPost": message.PreviewsPost,
+ "UpdateTime": TimestampConvert(message.UpdateAt),
+ "UpdateType": message.UpdateType,
+ "EditedNewMsgId": message.EditedNewMsgId,
},
}
diff --git a/server/enterprise/message_export/message_export.go b/server/enterprise/message_export/message_export.go
index 50643019c9..b124d86320 100644
--- a/server/enterprise/message_export/message_export.go
+++ b/server/enterprise/message_export/message_export.go
@@ -6,34 +6,22 @@ package message_export
import (
"encoding/json"
"errors"
- "fmt"
- "math"
- "net/http"
- "os"
- "path"
- "time"
-
"strconv"
+ "time"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/app"
- "github.com/mattermost/mattermost/server/v8/channels/store"
- "github.com/mattermost/mattermost/server/v8/channels/utils/fileutils"
"github.com/mattermost/mattermost/server/v8/einterfaces"
ejobs "github.com/mattermost/mattermost/server/v8/einterfaces/jobs"
- "github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
- "github.com/mattermost/mattermost/server/v8/platform/shared/templates"
-
"github.com/mattermost/mattermost/server/v8/enterprise/message_export/actiance_export"
"github.com/mattermost/mattermost/server/v8/enterprise/message_export/csv_export"
"github.com/mattermost/mattermost/server/v8/enterprise/message_export/global_relay_export"
+ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
)
-const (
- GlobalRelayExportFilename = "global-relay.zip"
-)
+const GlobalRelayExportFilename = "global-relay.zip"
type MessageExportInterfaceImpl struct {
Server *app.Server
@@ -56,7 +44,7 @@ func (m *MessageExportInterfaceImpl) StartSynchronizeJob(rctx request.CTX, expor
// if a valid export time was specified, put it in the job data
jobData := make(map[string]string)
if exportFromTimestamp >= 0 {
- jobData[JobDataBatchStartTimestamp] = strconv.FormatInt(exportFromTimestamp, 10)
+ jobData[shared.JobDataBatchStartTime] = strconv.FormatInt(exportFromTimestamp, 10)
}
// passing nil for job data will cause the worker to inherit start time from previously successful job
@@ -85,32 +73,137 @@ func (m *MessageExportInterfaceImpl) StartSynchronizeJob(rctx request.CTX, expor
return job, nil
}
-func (m *MessageExportInterfaceImpl) RunExport(rctx request.CTX, exportType string, since int64, limit int) (warningCount int64, appErr *model.AppError) {
- if limit < 0 {
- limit = math.MaxInt64
- }
- postsToExport, _, err := m.Server.Store().Compliance().MessageExport(rctx, model.MessageExportCursor{LastPostUpdateAt: since}, limit)
+func RunBatch(rctx request.CTX, data shared.JobData, params shared.BackendParams) (shared.RunExportResults, shared.JobData, error) {
+ start := time.Now()
+ var err error
+ var res shared.RunExportResults
+ data, err = GetDataForBatch(rctx, data, params)
if err != nil {
- return warningCount, model.NewAppError("RunExport", "ent.message_export.run_export.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
- }
- rctx.Logger().Debug("Found posts to export", mlog.Int("number_of_posts", len(postsToExport)))
-
- fileBackend := m.Server.FileBackend()
- templatesDir, ok := fileutils.FindDir("templates")
- if !ok {
- return warningCount, model.NewAppError("RunExport", "ent.compliance.run_export.template_watcher.appError", nil, "", http.StatusAccepted)
+ return res, data, err
}
- t, err2 := templates.New(templatesDir)
- if err2 != nil {
- return warningCount, model.NewAppError("RunExport", "ent.compliance.run_export.template_watcher.appError", nil, "", http.StatusAccepted).Wrap(err2)
+ if data.Finished {
+ return res, data, nil
}
- exportDirectory := getOutputDirectoryPath(since, model.GetMillis())
- return runExportByType(rctx, exportType, postsToExport, exportDirectory, m.Server.Store(), fileBackend, fileBackend, t, m.Server.Config())
+ // Now write the data to the export type.
+ res, err = RunExportByType(rctx, DataToExportParams(data), params)
+ if err != nil {
+ return res, data, err
+ }
+
+ data.ProcessingPostsMs = append(data.ProcessingPostsMs, res.ProcessingPostsMs)
+ data.ProcessingXmlMs = append(data.ProcessingXmlMs, res.ProcessingXmlMs)
+ data.TransferringFilesMs = append(data.TransferringFilesMs, res.TransferringFilesMs)
+ data.TransferringZipMs = append(data.TransferringZipMs, res.TransferringZipMs)
+ data.TotalBatchMs = append(data.TotalBatchMs, time.Since(start).Milliseconds())
+ data.WarningCount += res.NumWarnings
+ data.BatchStartTime = data.BatchEndTime
+
+ return res, data, err
}
-func runExportByType(rctx request.CTX, exportType string, postsToExport []*model.MessageExport, exportDirectory string, db store.Store, exportBackend filestore.FileBackend, fileAttachmentBackend filestore.FileBackend, htmlTemplates *templates.Container, config *model.Config) (warningCount int64, appErr *model.AppError) {
+// GetDataForBatch gets the posts for this batch and updates JobData with the current state.
+func GetDataForBatch(rctx request.CTX, data shared.JobData, params shared.BackendParams) (shared.JobData, error) {
+ start := time.Now()
+ var err error
+ // Using BatchSize+1 is a trick to test whether or not we've reached the final batch.
+ data.PostsToExport, data.Cursor, err = params.Store.Compliance().MessageExport(rctx, data.Cursor, data.BatchSize+1)
+ if err != nil {
+ return data, err
+ }
+ data.MessageExportMs = append(data.MessageExportMs, time.Since(start).Milliseconds())
+
+ if len(data.PostsToExport) == data.BatchSize+1 {
+ // We still have posts after this current batch.
+ // Remove the last post, we have to leave it for the next batch.
+ lastPostIdx := len(data.PostsToExport) - 1
+ data.PostsToExport = data.PostsToExport[:lastPostIdx]
+ lastPostIdx = len(data.PostsToExport) - 1
+ data.Cursor.LastPostUpdateAt = *data.PostsToExport[lastPostIdx].PostUpdateAt
+ data.Cursor.LastPostId = *data.PostsToExport[lastPostIdx].PostId
+ data.BatchEndTime = data.Cursor.LastPostUpdateAt
+ } else {
+ // We've reached the final batch; we need to include all join/leave events that occur after the lastpost.
+ // This will let us also pick up the joins/leaves that occur after lastPostUpdateAt but before JobEndTime.
+ data.BatchEndTime = data.JobEndTime
+ }
+
+ if len(data.PostsToExport) == 0 {
+ data.Finished = true
+ return data, nil
+ }
+
+ rctx.Logger().Debug("Found posts to export", mlog.Int("num_posts", len(data.PostsToExport)))
+ data.MessagesExported += len(data.PostsToExport)
+ data.BatchNumber++
+ data.BatchPath = shared.GetBatchPath(data.ExportDir, data.BatchStartTime, data.BatchEndTime, data.BatchNumber)
+
+ return data, nil
+}
+
+type ExportParams struct {
+ ExportType string
+ ChannelMetadata map[string]*shared.MetadataChannel
+ ChannelMemberHistories map[string][]*model.ChannelMemberHistoryResult
+ PostsToExport []*model.MessageExport
+ JobStartTime int64
+ BatchPath string
+ BatchStartTime int64
+ BatchEndTime int64
+}
+
+func DataToExportParams(data shared.JobData) ExportParams {
+ return ExportParams{
+ ExportType: data.ExportType,
+ ChannelMetadata: data.ChannelMetadata,
+ ChannelMemberHistories: data.ChannelMemberHistories,
+ PostsToExport: data.PostsToExport,
+ JobStartTime: data.JobStartTime,
+ BatchPath: data.BatchPath,
+ BatchStartTime: data.BatchStartTime,
+ BatchEndTime: data.BatchEndTime,
+ }
+}
+
+func RunExportByType(rctx request.CTX, p ExportParams, b shared.BackendParams) (results shared.RunExportResults, err error) {
+ preparePosts(rctx, p.PostsToExport)
+
+ exportParams := shared.ExportParams{
+ ExportType: p.ExportType,
+ ChannelMetadata: p.ChannelMetadata,
+ Posts: p.PostsToExport,
+ ChannelMemberHistories: p.ChannelMemberHistories,
+ JobStartTime: p.JobStartTime,
+ BatchPath: p.BatchPath,
+ BatchStartTime: p.BatchStartTime,
+ BatchEndTime: p.BatchEndTime,
+ Config: b.Config,
+ Db: b.Store,
+ FileAttachmentBackend: b.FileAttachmentBackend,
+ ExportBackend: b.ExportBackend,
+ Templates: b.HtmlTemplates,
+ }
+
+ switch p.ExportType {
+ case model.ComplianceExportTypeCsv:
+ rctx.Logger().Debug("Exporting CSV")
+ return csv_export.CsvExport(rctx, exportParams)
+
+ case model.ComplianceExportTypeActiance:
+ rctx.Logger().Debug("Exporting Actiance")
+ return actiance_export.ActianceExport(rctx, exportParams)
+
+ case model.ComplianceExportTypeGlobalrelay, model.ComplianceExportTypeGlobalrelayZip:
+ rctx.Logger().Debug("Exporting GlobalRelay")
+ return global_relay_export.GlobalRelayExport(rctx, exportParams)
+
+ default:
+ return results, errors.New("Unknown output format: " + p.ExportType)
+ }
+}
+
+func preparePosts(rctx request.CTX, postsToExport []*model.MessageExport) {
// go through all the posts and if the post's props contain 'from_bot' - override the IsBot field, since it's possible that the sender is not a user, but was a Bot and vise-versa
for _, post := range postsToExport {
if post.PostProps != nil {
@@ -178,57 +271,4 @@ func runExportByType(rctx request.CTX, exportType string, postsToExport []*model
post.PostCreateAt = new(int64)
}
}
-
- switch exportType {
- case model.ComplianceExportTypeCsv:
- rctx.Logger().Debug("Exporting CSV")
- return csv_export.CsvExport(rctx, postsToExport, db, exportBackend, fileAttachmentBackend, exportDirectory)
-
- case model.ComplianceExportTypeActiance:
- rctx.Logger().Debug("Exporting Actiance")
- return actiance_export.ActianceExport(rctx, postsToExport, db, exportBackend, fileAttachmentBackend, exportDirectory)
-
- case model.ComplianceExportTypeGlobalrelay, model.ComplianceExportTypeGlobalrelayZip:
- rctx.Logger().Debug("Exporting GlobalRelay")
- f, err := os.CreateTemp("", "")
- if err != nil {
- return warningCount, model.NewAppError("RunExport", "ent.compliance.global_relay.open_temporary_file.appError", nil, "", http.StatusAccepted).Wrap(err)
- }
- defer f.Close()
- defer os.Remove(f.Name())
-
- attachmentsRemovedPostIDs, warnings, appErr := global_relay_export.GlobalRelayExport(rctx, postsToExport, db, fileAttachmentBackend, f, htmlTemplates)
- if appErr != nil {
- return warningCount, appErr
- }
- warningCount = warnings
- _, err = f.Seek(0, 0)
- if err != nil {
- return warningCount, model.NewAppError("RunExport", "ent.compliance.global_relay.rewind_temporary_file.appError", nil, "", http.StatusAccepted).Wrap(err)
- }
-
- if exportType == model.ComplianceExportTypeGlobalrelayZip {
- // Try to disable the write timeout for the potentially big export file.
- _, nErr := filestore.TryWriteFileContext(rctx.Context(), exportBackend, f, path.Join(exportDirectory, GlobalRelayExportFilename))
- if nErr != nil {
- return warningCount, model.NewAppError("runExportByType", "ent.compliance.global_relay.write_file.appError", nil, "", http.StatusInternalServerError).Wrap(nErr)
- }
- } else {
- appErr = global_relay_export.Deliver(f, config)
- if appErr != nil {
- return warningCount, appErr
- }
- }
-
- if len(attachmentsRemovedPostIDs) > 0 {
- rctx.Logger().Debug("Global Relay Attachments Removed because they were too large to send to Global Relay", mlog.Array("attachment_ids", attachmentsRemovedPostIDs))
- description := fmt.Sprintf("Attachments to post IDs %v were removed because they were too large to send to Global Relay.", attachmentsRemovedPostIDs)
- appErr = model.NewAppError("RunExport", "ent.compliance.global_relay.attachments_removed.appError", map[string]any{"Description": description}, description, http.StatusAccepted)
- return warningCount, appErr
- }
- default:
- err := errors.New("Unknown output format " + exportType)
- return warningCount, model.NewAppError("RunExport", "ent.compliance.bad_export_type.appError", map[string]any{"ExportType": exportType}, "", http.StatusBadRequest).Wrap(err)
- }
- return warningCount, nil
}
diff --git a/server/enterprise/message_export/message_export_test.go b/server/enterprise/message_export/message_export_test.go
index ad41c2c79c..f5f2eeb036 100644
--- a/server/enterprise/message_export/message_export_test.go
+++ b/server/enterprise/message_export/message_export_test.go
@@ -4,43 +4,224 @@
package message_export
import (
- "context"
+ "archive/zip"
+ "bytes"
+ _ "embed"
+ "encoding/xml"
+ "fmt"
+ "io"
"os"
+ "path"
"strconv"
+ "strings"
"testing"
"time"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+ "github.com/stretchr/testify/require"
+
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
- "github.com/mattermost/mattermost/server/v8/channels/api4"
- "github.com/mattermost/mattermost/server/v8/channels/jobs"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
+ st "github.com/mattermost/mattermost/server/v8/channels/store/storetest"
+ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/actiance_export"
+ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/global_relay_export"
+ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
)
+//go:embed testdata/actianceXMLHeader.tmpl
+var actianceXMLHeader string
+
+//go:embed testdata/actianceE2E1Batch1ch2.tmpl
+var actianceE2E1Batch1ch2tmpl string
+
+//go:embed testdata/actianceE2E1Batch1ch3.tmpl
+var actianceE2E1Batch1ch3tmpl string
+
+//go:embed testdata/actianceE2E1Batch2.tmpl
+var actianceE2E1Batch2tmpl string
+
+//go:embed testdata/actianceE2E1Batch3ch2.tmpl
+var actianceE2E1Batch3ch2tmpl string
+
+//go:embed testdata/actianceE2E1Batch3ch4.tmpl
+var actianceE2E1Batch3ch4tmpl string
+
+//go:embed testdata/actianceE2E2.tmpl
+var actianceE2E2 string
+
+//go:embed testdata/grE2E1Batch1Summary.tmpl
+var grE2E1Batch1Summary string
+
+//go:embed testdata/grE2E1Batch1.tmpl
+var grE2E1Batch1 string
+
+//go:embed testdata/grE2E1Batch1SummaryCh3.tmpl
+var grE2E1Batch1SummaryCh3 string
+
+//go:embed testdata/grE2E1Batch1Ch3.tmpl
+var grE2E1Batch1Ch3 string
+
+//go:embed testdata/grE2E1Batch2Summary.tmpl
+var grE2E1Batch2Summary string
+
+//go:embed testdata/grE2E1Batch2.tmpl
+var grE2E1Batch2 string
+
+//go:embed testdata/grE2E1Batch3Summary.tmpl
+var grE2E1Batch3Summary string
+
+//go:embed testdata/grE2E1Batch3.tmpl
+var grE2E1Batch3 string
+
+//go:embed testdata/grE2E1Batch3SummaryCh4.tmpl
+var grE2E1Batch3SummaryCh4 string
+
+//go:embed testdata/grE2E1Batch3Ch4.tmpl
+var grE2E1Batch3Ch4 string
+
+//go:embed testdata/csvE2E1Batch1.tmpl
+var csvE2E1Batch1 string
+
+//go:embed testdata/csvE2E1Batch2.tmpl
+var csvE2E1Batch2 string
+
+//go:embed testdata/csvE2E1Batch3.tmpl
+var csvE2E1Batch3 string
+
+//go:embed testdata/grE2E2Batch1Summary.tmpl
+var grE2E2Batch1Summary string
+
+//go:embed testdata/grE2E2Batch1.tmpl
+var grE2E2Batch1 string
+
+//go:embed testdata/csvE2E2Batch1.tmpl
+var csvE2E2Batch1 string
+
+//go:embed testdata/grE2E3Batch1SummaryPerm1.tmpl
+var grE2E3Batch1SummaryPerm1 string
+
+//go:embed testdata/grE2E3Batch1SummaryPerm2.tmpl
+var grE2E3Batch1SummaryPerm2 string
+
+//go:embed testdata/grE2E3Batch1SummaryPerm3.tmpl
+var grE2E3Batch1SummaryPerm3 string
+
+//go:embed testdata/grE2E3Batch1SummaryPerm4.tmpl
+var grE2E3Batch1SummaryPerm4 string
+
+//go:embed testdata/grE2E3Batch1Perm1.tmpl
+var grE2E3Batch1Perm1 string
+
+//go:embed testdata/grE2E3Batch1Perm2.tmpl
+var grE2E3Batch1Perm2 string
+
+//go:embed testdata/grE2E3Batch1Perm3.tmpl
+var grE2E3Batch1Perm3 string
+
+//go:embed testdata/grE2E3Batch1Perm4.tmpl
+var grE2E3Batch1Perm4 string
+
+//go:embed testdata/grE2E3Batch2SummaryPerm1.tmpl
+var grE2E3Batch2SummaryPerm1 string
+
+//go:embed testdata/grE2E3Batch2SummaryPerm2.tmpl
+var grE2E3Batch2SummaryPerm2 string
+
+//go:embed testdata/grE2E3Batch2Perm1.tmpl
+var grE2E3Batch2Perm1 string
+
+//go:embed testdata/grE2E3Batch2Perm2.tmpl
+var grE2E3Batch2Perm2 string
+
+//go:embed testdata/grE2E4Summary.tmpl
+var grE2E4Summary string
+
+//go:embed testdata/csvE2E3Batch1Perm1.tmpl
+var csvE2E3Batch1Perm1 string
+
+//go:embed testdata/csvE2E3Batch1Perm2.tmpl
+var csvE2E3Batch1Perm2 string
+
+//go:embed testdata/csvE2E3Batch1Perm3.tmpl
+var csvE2E3Batch1Perm3 string
+
+//go:embed testdata/csvE2E3Batch1Perm4.tmpl
+var csvE2E3Batch1Perm4 string
+
+//go:embed testdata/csvE2E3Batch2Perm1.tmpl
+var csvE2E3Batch2Perm1 string
+
+//go:embed testdata/csvE2E3Batch2Perm2.tmpl
+var csvE2E3Batch2Perm2 string
+
+//go:embed testdata/csvE2E4Batch1.tmpl
+var csvE2E4Batch1 string
+
+func conv(dateTime int64) string {
+ return global_relay_export.TimestampConvert(dateTime)
+}
+
+type MyReporter struct {
+ mock.Mock
+}
+
+func (mr *MyReporter) ReportProgressMessage(message string) {
+ mr.Called(message)
+}
+
func TestRunExportByType(t *testing.T) {
- tempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() {
- err = os.RemoveAll(tempDir)
+ t.Run("no dedicated export filestore", func(t *testing.T) {
+ exportTempDir, err := os.MkdirTemp("", "")
+ require.NoError(t, err)
+
+ fileBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
+ DriverName: model.ImageDriverLocal,
+ Directory: exportTempDir,
+ })
assert.NoError(t, err)
+
+ testRunExportByType(t, fileBackend, exportTempDir, fileBackend, exportTempDir)
})
- config := filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: tempDir,
- }
+ t.Run("using dedicated export filestore", func(t *testing.T) {
+ exportTempDir, err := os.MkdirTemp("", "")
+ require.NoError(t, err)
- fileBackend, err := filestore.NewFileBackend(config)
- require.NoError(t, err)
+ exportBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
+ DriverName: model.ImageDriverLocal,
+ Directory: exportTempDir,
+ })
+ assert.NoError(t, err)
+ attachmentTempDir, err := os.MkdirTemp("", "")
+ require.NoError(t, err)
+
+ attachmentBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
+ DriverName: model.ImageDriverLocal,
+ Directory: attachmentTempDir,
+ })
+ require.NoError(t, err)
+
+ testRunExportByType(t, exportBackend, exportTempDir, attachmentBackend, attachmentTempDir)
+ })
+}
+
+func testRunExportByType(t *testing.T, exportBackend filestore.FileBackend, exportDir string, attachmentBackend filestore.FileBackend, attachmentDir string) {
rctx := request.TestContext(t)
chanTypeDirect := model.ChannelTypeDirect
+
t.Run("missing user info", func(t *testing.T) {
+ t.Cleanup(func() {
+ err := os.RemoveAll(exportDir)
+ assert.NoError(t, err)
+ err = os.RemoveAll(attachmentDir)
+ assert.NoError(t, err)
+ })
+
posts := []*model.MessageExport{
{
PostId: model.NewPointer("post-id"),
@@ -54,6 +235,9 @@ func TestRunExportByType(t *testing.T) {
PostCreateAt: model.NewPointer(int64(1)),
PostUpdateAt: model.NewPointer(int64(1)),
PostMessage: model.NewPointer("message"),
+ UserEmail: model.NewPointer("test@example.com"),
+ Username: model.NewPointer("Mr. Test"),
+ UserId: model.NewPointer(st.NewTestID()),
ChannelType: &chanTypeDirect,
PostFileIds: []string{},
},
@@ -61,66 +245,119 @@ func TestRunExportByType(t *testing.T) {
mockStore := &storetest.Store{}
defer mockStore.AssertExpectations(t)
- mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", int64(1), int64(1), "channel-id").Return([]*model.ChannelMemberHistoryResult{}, nil)
+ mockStore.ChannelMemberHistoryStore.On("GetChannelsWithActivityDuring", int64(1), int64(1)).
+ Return([]string{"channel-id"}, nil)
+ mockStore.ChannelStore.On("GetMany", []string{"channel-id"}, true).
+ Return(model.ChannelList{{
+ Id: "channel-id",
+ DisplayName: "channel-display-name",
+ Name: "channel-name",
+ Type: chanTypeDirect,
+ }}, nil)
+ mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", int64(1), int64(1), []string{"channel-id"}).Return([]*model.ChannelMemberHistoryResult{}, nil)
- warnings, err := runExportByType(rctx, model.ComplianceExportTypeActiance, posts, tempDir, mockStore, fileBackend, fileBackend, nil, nil)
- require.Nil(t, err)
- require.Zero(t, warnings)
+ myMockReporter := MyReporter{}
+ defer myMockReporter.AssertExpectations(t)
+ myMockReporter.On("ReportProgressMessage", "Exporting channel information for 1 channels.")
+ myMockReporter.On("ReportProgressMessage", "Calculating channel activity: 0/1 channels completed.")
+
+ channelMetadata, channelMemberHistories, err := shared.CalculateChannelExports(rctx,
+ shared.ChannelExportsParams{
+ Store: shared.NewMessageExportStore(mockStore),
+ ExportPeriodStartTime: 1,
+ ExportPeriodEndTime: 1,
+ ChannelBatchSize: 100,
+ ChannelHistoryBatchSize: 100,
+ ReportProgressMessage: myMockReporter.ReportProgressMessage,
+ })
+ assert.NoError(t, err)
+
+ res, err := RunExportByType(rctx, ExportParams{
+ ExportType: model.ComplianceExportTypeActiance,
+ ChannelMetadata: channelMetadata,
+ ChannelMemberHistories: channelMemberHistories,
+ PostsToExport: posts,
+ BatchPath: "testZipName",
+ BatchStartTime: 1,
+ BatchEndTime: 1,
+ }, shared.BackendParams{
+ Store: shared.NewMessageExportStore(mockStore),
+ FileAttachmentBackend: attachmentBackend,
+ ExportBackend: exportBackend,
+ HtmlTemplates: nil,
+ Config: nil,
+ })
+ require.NoError(t, err)
+ require.Zero(t, res.NumWarnings)
})
}
-func runJobForTest(t *testing.T, th *api4.TestHelper) *model.Job {
- job, _, err := th.SystemAdminClient.CreateJob(context.Background(), &model.Job{Type: "message_export"})
- require.NoError(t, err)
- // poll until completion
- doneChan := make(chan bool)
- go func() {
- defer close(doneChan)
- for {
- jobs, _, err := th.SystemAdminClient.GetJobsByType(context.Background(), "message_export", 0, 1)
- require.NoError(t, err)
- require.Len(t, jobs, 1)
- require.Equal(t, job.Id, jobs[0].Id)
- job = jobs[0]
- if job.Status != "pending" && job.Status != "in_progress" {
- break
- }
- time.Sleep(1 * time.Second)
- }
- require.Equal(t, "success", job.Status)
- }()
- select {
- case <-doneChan:
- case <-time.After(10 * time.Second):
- require.True(t, false, "job is taking too long")
- }
- return job
+func TestRunExportJobE2EByType(t *testing.T) {
+ t.Run("no dedicated export filestore", func(t *testing.T) {
+ exportTempDir, err := os.MkdirTemp("", "")
+ require.NoError(t, err)
+
+ fileBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
+ DriverName: model.ImageDriverLocal,
+ Directory: exportTempDir,
+ })
+ assert.NoError(t, err)
+
+ testRunExportJobE2E(t, fileBackend, exportTempDir, fileBackend, exportTempDir)
+ })
+
+ t.Run("using dedicated export filestore", func(t *testing.T) {
+ exportTempDir, err := os.MkdirTemp("", "")
+ require.NoError(t, err)
+
+ exportBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
+ DriverName: model.ImageDriverLocal,
+ Directory: exportTempDir,
+ })
+ assert.NoError(t, err)
+
+ attachmentTempDir, err := os.MkdirTemp("", "")
+ require.NoError(t, err)
+
+ attachmentBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
+ DriverName: model.ImageDriverLocal,
+ Directory: attachmentTempDir,
+ })
+ require.NoError(t, err)
+
+ testRunExportJobE2E(t, exportBackend, exportTempDir, attachmentBackend, attachmentTempDir)
+ })
}
-func TestRunExportJob(t *testing.T) {
+func testRunExportJobE2E(t *testing.T, exportBackend filestore.FileBackend, exportDir string,
+ attachmentBackend filestore.FileBackend, attachmentDir string) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
- jobs.DefaultWatcherPollingInterval = 100
- th := api4.SetupEnterprise(t).InitBasic()
- th.App.Srv().SetLicense(model.NewTestLicense("message_export"))
- defer th.TearDown()
- messageExportImpl := MessageExportJobInterfaceImpl{th.App.Srv()}
- th.App.Srv().Jobs.RegisterJobType(model.JobTypeMessageExport, messageExportImpl.MakeWorker(), messageExportImpl.MakeScheduler())
-
- err := th.App.Srv().Jobs.StartWorkers()
- require.NoError(t, err)
-
- err = th.App.Srv().Jobs.StartSchedulers()
- require.NoError(t, err)
-
- th.App.UpdateConfig(func(cfg *model.Config) {
- *cfg.MessageExportSettings.EnableExport = true
- })
-
t.Run("conflicting timestamps", func(t *testing.T) {
- time.Sleep(100 * time.Millisecond)
+ th := setup(t)
+ defer th.TearDown()
+ defer func() {
+ err := os.RemoveAll(exportDir)
+ assert.NoError(t, err)
+ err = os.RemoveAll(attachmentDir)
+ assert.NoError(t, err)
+ }()
+
+ th.App.UpdateConfig(func(cfg *model.Config) {
+ *cfg.MessageExportSettings.EnableExport = true
+ *cfg.FileSettings.DriverName = model.ImageDriverLocal
+ *cfg.FileSettings.Directory = attachmentDir
+
+ if exportDir != attachmentDir {
+ *cfg.FileSettings.DedicatedExportStore = true
+ *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
+ *cfg.FileSettings.ExportDirectory = exportDir
+ }
+ })
+
+ time.Sleep(1 * time.Millisecond)
now := model.GetMillis()
th.App.UpdateConfig(func(cfg *model.Config) {
*cfg.MessageExportSettings.ExportFromTimestamp = now - 1
@@ -128,18 +365,2248 @@ func TestRunExportJob(t *testing.T) {
})
for i := 0; i < 3; i++ {
- _, err := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ _, err2 := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
ChannelId: th.BasicChannel.Id,
- UserId: model.NewId(),
- Message: "zz" + model.NewId() + "b",
+ UserId: st.NewTestID(),
+ Message: st.NewTestID(),
CreateAt: now,
})
- require.NoError(t, err)
+ require.NoError(t, err2)
}
- job := runJobForTest(t, th)
- numExported, err := strconv.ParseInt(job.Data["messages_exported"], 0, 64)
+ job := runJobForTest(t, th, nil)
+
+ warnings, err := strconv.Atoi(job.Data[shared.JobDataWarningCount])
+ require.NoError(t, err)
+ require.Equal(t, 0, warnings)
+
+ numExported, err := strconv.ParseInt(job.Data[shared.JobDataMessagesExported], 0, 64)
require.NoError(t, err)
require.Equal(t, int64(3), numExported)
})
+
+ t.Run("actiance -- multiple batches, 1 zip per batch, output to a single directory", func(t *testing.T) {
+ th := setup(t)
+ defer th.TearDown()
+ defer func() {
+ err := os.RemoveAll(exportDir)
+ assert.NoError(t, err)
+ err = os.RemoveAll(attachmentDir)
+ assert.NoError(t, err)
+ }()
+
+ res := generateActianceBatchTest1(t, th, attachmentDir, exportDir, attachmentBackend)
+
+ files, err := exportBackend.ListDirectory(res.jobExportDir)
+ require.NoError(t, err)
+ require.ElementsMatch(t, res.batches, files)
+
+ fileContents := openZipAndReadFile(t, exportBackend, res.batches[0], res.attachments[0].Path)
+
+ require.EqualValuesf(t, res.contents[0], fileContents, "file contents not equal")
+ })
+
+ t.Run("actiance -- multiple batches, using UntilUpdateAt", func(t *testing.T) {
+ th := setup(t)
+ defer th.TearDown()
+ defer func() {
+ err := os.RemoveAll(exportDir)
+ assert.NoError(t, err)
+ err = os.RemoveAll(attachmentDir)
+ assert.NoError(t, err)
+ }()
+
+ res := generateActianceBatchTest2(t, th, attachmentDir, exportDir)
+
+ files, err := exportBackend.ListDirectory(res.jobExportDir)
+ require.NoError(t, err)
+ require.ElementsMatch(t, res.batches, files)
+ })
+
+ t.Run("actiance e2e 1", func(t *testing.T) {
+ tests := []struct {
+ name string
+ testStopping bool
+ }{
+ {
+ name: "full tests, no stopping",
+ testStopping: false,
+ },
+ {
+ name: "full tests, stopped and resumed",
+ // This uses the same output as the previous e2e test, but tests that the job can be stopped and
+ // resumed with no change to the directory, files, file contents, or job.Data that shouldn't change.
+ // We want to be confident that jobs can resume without data missing or added from the original run.
+ testStopping: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ th := setup(t)
+ defer th.TearDown()
+ defer func() {
+ err := os.RemoveAll(exportDir)
+ assert.NoError(t, err)
+ err = os.RemoveAll(attachmentDir)
+ assert.NoError(t, err)
+ }()
+
+ ret := generateE2ETestType1Results(t, th, model.ComplianceExportTypeActiance, attachmentDir,
+ exportDir, attachmentBackend, exportBackend, tt.testStopping)
+ channel2 := ret.channels[0]
+ channel3 := ret.channels[1]
+ channel4 := ret.channels[2]
+ start := ret.start
+ jl := ret.joinLeaves
+ posts := ret.posts
+ createUpdateTimes := ret.createUpdateTimes
+ attachments := ret.attachments
+ contents := ret.contents
+ jobEndTime := ret.jobEndTime
+ batches := ret.batches
+
+ // Expected data:
+ // - batch1 has two channels, and we're not sure which will come first. What a pain.
+
+ // actiance exports, for batch 1:
+ // for ch2:
+ // 3 participants entered
+ // message 0
+ // file for message 0 (start and ended)
+ // message 1
+ // file for message 1 (start and ended)
+ // message 2
+ // file for message 2 (start and ended)
+ // 3 participants left
+ // for ch3:
+ // 1 participant entered
+ // 1 participant left
+
+ // actiance exports, for batch 2:
+ // for ch2:
+ // 2 participants entered
+ // message 3
+ // file for message 3 (start and ended)
+ // message 4
+ // file for message 4 (start and ended)
+ // message 5
+ // file for message 5 (start and ended)
+ // 2 participants left
+
+ // actiance exports, for batch 3:
+ // for ch2:
+ // 3 participants entered
+ // message 6
+ // file for message 6 (start and ended)
+ // message 7
+ // file for message 7 (start and ended)
+ // message 8
+ // file for message 8 (start and ended)
+ // 3 participants left
+ // for ch4:
+ // 1 participant entered
+ // 1 participant left
+
+ batch1ch2 := fmt.Sprintf(actianceE2E1Batch1ch2tmpl, channel2.Id, start, jl[0].join, jl[1].join, jl[2].join,
+ posts[0].Id, createUpdateTimes[0], createUpdateTimes[0], createUpdateTimes[0],
+ posts[1].Id, createUpdateTimes[1], createUpdateTimes[1], createUpdateTimes[1],
+ posts[2].Id, createUpdateTimes[2], createUpdateTimes[2], createUpdateTimes[2],
+ jl[1].leave, jl[2].leave, createUpdateTimes[2], createUpdateTimes[2])
+
+ batch1ch3 := fmt.Sprintf(actianceE2E1Batch1ch3tmpl, channel3.Id, start, jl[6].join, jl[6].leave, createUpdateTimes[2])
+
+ xmlHeader := strings.TrimSpace(actianceXMLHeader)
+ batch1Possibility1 := fmt.Sprintf(xmlHeader, batch1ch2, batch1ch3)
+ batch1Possibility2 := fmt.Sprintf(xmlHeader, batch1ch3, batch1ch2)
+
+ batch2xml := fmt.Sprintf(actianceE2E1Batch2tmpl, channel2.Id, createUpdateTimes[2], jl[0].join, jl[3].join,
+ posts[3].Id, createUpdateTimes[3], createUpdateTimes[3], createUpdateTimes[3],
+ posts[4].Id, createUpdateTimes[4], createUpdateTimes[4], createUpdateTimes[4],
+ posts[5].Id, createUpdateTimes[5], createUpdateTimes[5], createUpdateTimes[5],
+ jl[3].leave, createUpdateTimes[5], createUpdateTimes[5])
+ batch2xml = strings.TrimSpace(batch2xml)
+
+ // Batch3 has two channels, and we're not sure which will come first. What a pain.
+ batch3ch2 := fmt.Sprintf(actianceE2E1Batch3ch2tmpl, channel2.Id, createUpdateTimes[5], jl[0].join, jl[4].join, jl[5].join,
+ posts[6].Id, createUpdateTimes[6], createUpdateTimes[6], createUpdateTimes[6],
+ posts[7].Id, createUpdateTimes[7], createUpdateTimes[7], createUpdateTimes[7],
+ posts[8].Id, createUpdateTimes[8], createUpdateTimes[8], createUpdateTimes[8],
+ jl[4].leave, jl[5].leave, jobEndTime, jobEndTime)
+
+ batch3ch4 := fmt.Sprintf(actianceE2E1Batch3ch4tmpl, channel4.Id, createUpdateTimes[5], jl[7].join, jobEndTime, jobEndTime)
+
+ batch3Possibility1 := fmt.Sprintf(xmlHeader, batch3ch2, batch3ch4)
+
+ batch3Possibility2 := fmt.Sprintf(xmlHeader, batch3ch4, batch3ch2)
+
+ for b, batchName := range batches {
+ xmlContents := openZipAndReadFile(t, exportBackend, batchName, "actiance_export.xml")
+
+ // this is so clunky, sorry. but it's simple.
+ if b == 0 {
+ if xmlContents != batch1Possibility1 && xmlContents != batch1Possibility2 {
+ // to make some output
+ assert.Equal(t, batch1Possibility1, xmlContents, "batch 1 possibility 1")
+ assert.Equal(t, batch1Possibility2, xmlContents, "batch 1 possibility 2")
+ }
+ }
+
+ if b == 1 {
+ require.Equal(t, batch2xml, xmlContents, "batch 2")
+ }
+
+ if b == 2 {
+ if xmlContents != batch3Possibility1 && xmlContents != batch3Possibility2 {
+ // to make some output
+ assert.Equal(t, batch3Possibility1, xmlContents, "batch 3 possibility 1")
+ assert.Equal(t, batch3Possibility2, xmlContents, "batch 3 possibility 2")
+ }
+ }
+
+ zipBytes, err := exportBackend.ReadFile(batchName)
+ require.NoError(t, err)
+ zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+ require.NoError(t, err)
+
+ for i := 0; i < 3; i++ {
+ num := b*3 + i
+ attachmentInZip, err := zipReader.Open(attachments[num].Path)
+ require.NoError(t, err)
+ attachmentInZipContents, err := io.ReadAll(attachmentInZip)
+ require.NoError(t, err)
+ err = attachmentInZip.Close()
+ require.NoError(t, err)
+ require.EqualValuesf(t, contents[num], string(attachmentInZipContents), "file contents not equal")
+ }
+ }
+ })
+ }
+ })
+
+ t.Run("GlobalRelay e2e 1", func(t *testing.T) {
+ tests := []struct {
+ name string
+ testStopping bool
+ }{
+ {
+ name: "full tests, no stopping",
+ testStopping: false,
+ },
+ {
+ name: "full tests, stopped and resumed",
+ // This uses the same output as the previous e2e test, but tests that the job can be stopped and
+ // resumed with no change to the directory, files, file contents, or job.Data that shouldn't change.
+ // We want to be confident that jobs can resume without data missing or added from the original run.
+ testStopping: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ th := setup(t)
+ defer th.TearDown()
+ defer func() {
+ err := os.RemoveAll(exportDir)
+ assert.NoError(t, err)
+ err = os.RemoveAll(attachmentDir)
+ assert.NoError(t, err)
+ }()
+
+ ret := generateE2ETestType1Results(t, th, model.ComplianceExportTypeGlobalrelayZip, attachmentDir,
+ exportDir, attachmentBackend, exportBackend, tt.testStopping)
+ teams := ret.teams
+ channel2 := ret.channels[0]
+ channel3 := ret.channels[1]
+ channel4 := ret.channels[2]
+ users := ret.users
+ posts := ret.posts
+ batchTimes := ret.batchTimes
+ jobStartTime := ret.start
+ jl := ret.joinLeaves
+ batches := ret.batches
+ cu := ret.createUpdateTimes
+
+ // aligned with actiance exports, for batch 1:
+ // ** except that there is no closed-out leaves at batch end
+ // for ch2:
+ // 3 participants entered (1, 2, 3)
+ // message 0
+ // file for message 0 (start and ended)
+ // message 1
+ // file for message 1 (start and ended)
+ // message 2
+ // file for message 2 (start and ended)
+ // 1 participants left
+ // for ch3:
+ // 1 participant entered
+ // 1 participant left
+
+ // for batch 2:
+ // for ch2:
+ // 2 participants entered
+ // message 3
+ // file for message 3 (start and ended)
+ // message 4
+ // file for message 4 (start and ended)
+ // message 5
+ // file for message 5 (start and ended)
+ // 1 participant left
+
+ // for batch 3:
+ // for ch2:
+ // 3 participants entered
+ // message 6
+ // file for message 6 (start and ended)
+ // message 7
+ // file for message 7 (start and ended)
+ // message 8
+ // file for message 8 (start and ended)
+ // 2 participants left
+ // for ch4:
+ // 1 participant entered
+
+ for batchNum, batchName := range batches {
+ data1 := openZipAndReadFileStartingWith(t, exportBackend, batchName, channel2.Name)
+ // clean some bad csrf if present
+ msg1 := global_relay_export.CleanTestOutput(data1)
+
+ batchStartTime := batchTimes[batchNum].start
+ batchEndTime := batchTimes[batchNum].end
+ expectedBatchExportCh2 := []string{
+ // batch 1 Summary
+ fmt.Sprintf(grE2E1Batch1Summary,
+ // 1 2 3 4 5
+ channel2.DisplayName, conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 6 7 8 9
+ conv(jl[1].join), conv(jl[1].leave), conv(jl[2].join), conv(jl[2].leave),
+ // 10 11 12
+ conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt),
+ // 13 14 15 16
+ channel2.Id, channel2.TeamId, teams[0].Name, teams[0].DisplayName,
+ // 17 18 19 20 21 22
+ users[0].Id, users[1].Id, users[2].Id, posts[0].Id, posts[1].Id, posts[2].Id),
+
+ // batch 1
+ fmt.Sprintf(grE2E1Batch1,
+ // 1 2 3 4 5
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime), conv(jl[1].join),
+ // 6 7 8 9 10
+ conv(jl[1].leave), conv(jl[2].join), conv(jl[2].leave), conv(cu[0]), conv(cu[1]),
+ // 11 12 13 14 15
+ conv(cu[2]), conv(jobStartTime), teams[0].Id, teams[0].Name, teams[0].DisplayName,
+ // 16 17 18 19 20 21
+ users[0].Id, users[1].Id, users[2].Id, posts[0].Id, posts[1].Id, posts[2].Id,
+ // 22
+ channel2.Id),
+
+ // batch 2 Summary
+ fmt.Sprintf(grE2E1Batch2Summary,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 6 7 8
+ conv(jl[3].join), conv(jl[3].leave), conv(posts[3].CreateAt), conv(posts[4].CreateAt),
+ // 9
+ conv(posts[5].CreateAt),
+ // 10 11 12 13
+ channel2.Id, channel2.TeamId, teams[0].Name, teams[0].DisplayName,
+ // 14 15 16 17 18
+ users[0].Id, users[3].Id, posts[3].Id, posts[4].Id, posts[5].Id,
+ ),
+
+ // batch 2
+ fmt.Sprintf(grE2E1Batch2,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 6 7 8 9
+ conv(jl[3].join), conv(jl[3].leave), conv(cu[3]), conv(cu[4]), conv(cu[5]),
+ // 10 11 12 13
+ conv(jobStartTime), teams[0].Id, teams[0].Name, teams[0].DisplayName,
+ // 14 15 16 17 18 19
+ users[0].Id, users[1].Id, users[3].Id, posts[3].Id, posts[4].Id, posts[5].Id,
+ // 20
+ channel2.Id),
+
+ // batch 3 Summary
+ fmt.Sprintf(grE2E1Batch3Summary,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 6 7 8
+ conv(jl[4].join), conv(jl[4].leave), conv(jl[5].join), conv(jl[5].leave),
+ // 9 10 11
+ conv(posts[6].CreateAt), conv(posts[7].CreateAt), conv(posts[8].CreateAt),
+ // 12 13 14 15
+ channel2.Id, channel2.TeamId, teams[0].Name, teams[0].DisplayName,
+ // 16 17 18 19 20 21
+ users[0].Id, users[1].Id, users[2].Id, posts[6].Id, posts[7].Id, posts[8].Id,
+ // 22 23
+ users[4].Id, users[5].Id),
+
+ // batch 3
+ fmt.Sprintf(grE2E1Batch3,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 6 7 8
+ conv(jl[4].join), conv(jl[4].leave), conv(jl[5].join), conv(jl[5].leave),
+ // 9 10 11 12
+ conv(cu[6]), conv(cu[7]), conv(cu[8]), conv(jobStartTime),
+ // 13 14 15
+ teams[0].Id, teams[0].Name, teams[0].DisplayName,
+ // 16 17 18 19 20 21
+ users[0].Id, users[4].Id, users[5].Id, posts[6].Id, posts[7].Id, posts[8].Id,
+ // 22
+ channel2.Id),
+ }
+
+ if batchNum == 0 {
+ global_relay_export.AssertHeaderContains(t, msg1, map[string]string{
+ "Subject": "Mattermost Compliance Export: the Channel Two",
+ "From": "user1@email",
+ "X-Mattermost-ChannelName": "the Channel Two",
+ "To": "user1@email,user2@email,user3@email",
+ "X-Mattermost-ChannelID": channel2.Id,
+ "X-Mattermost-ChannelType": "private",
+ })
+ assert.Contains(t, msg1, expectedBatchExportCh2[0], "batch 1 Ch2 summary")
+ assert.Contains(t, msg1, expectedBatchExportCh2[1], "batch 1 Ch2")
+
+ // now read second channel's export
+ data2 := openZipAndReadFileStartingWith(t, exportBackend, batchName, channel3.Name)
+ // clean some bad csrf if present
+ msg2 := global_relay_export.CleanTestOutput(data2)
+
+ expectedBatchExportCh3 := []string{
+ // batch 1 Summary
+ fmt.Sprintf(grE2E1Batch1SummaryCh3,
+ // 1 2 3 4
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channel3.Id,
+ // 5 6 7 8 9
+ conv(batchStartTime), conv(batchEndTime), users[6].Id, conv(jl[6].join), conv(jl[6].leave)),
+
+ // batch 1
+ fmt.Sprintf(grE2E1Batch1Ch3,
+ // 1 2 3 4
+ teams[0].Id, channel3.Id, conv(batchStartTime), conv(batchEndTime),
+ // 5 6 7 8
+ users[6].Id, conv(jl[6].join), conv(jl[6].leave), conv(jobStartTime)),
+ }
+
+ // Note: for debugging, better keep this in case we need it again.
+ //t.Logf("<><>batch1 Ch3 actual\n\n%s\n\n<><>batch1 Ch3 Summary:\n\n%s\n\n",
+ // msg2, expectedBatchExportCh3[0])
+
+ // Channel 3
+ global_relay_export.AssertHeaderContains(t, msg2, map[string]string{
+ "Subject": "Mattermost Compliance Export: the Channel Three",
+ "From": "user7@email",
+ "X-Mattermost-ChannelName": "the Channel Three",
+ "To": "user7@email",
+ "X-Mattermost-ChannelID": channel3.Id,
+ "X-Mattermost-ChannelType": "public",
+ })
+ assert.Contains(t, msg2, expectedBatchExportCh3[0], "batch 1 Ch3 summary")
+ assert.Contains(t, msg2, expectedBatchExportCh3[1], "batch 1 Ch3")
+ }
+
+ if batchNum == 1 {
+ global_relay_export.AssertHeaderContains(t, msg1, map[string]string{
+ "Subject": "Mattermost Compliance Export: the Channel Two",
+ "From": "user1@email",
+ "X-Mattermost-ChannelName": "the Channel Two",
+ "To": "user1@email,user4@email",
+ "X-Mattermost-ChannelID": channel2.Id,
+ "X-Mattermost-ChannelType": "private",
+ })
+ assert.Contains(t, msg1, expectedBatchExportCh2[2], "batch 2 ch2 summary")
+ assert.Contains(t, msg1, expectedBatchExportCh2[3], "batch 2 ch2")
+ }
+
+ if batchNum == 2 {
+ global_relay_export.AssertHeaderContains(t, msg1, map[string]string{
+ "Subject": "Mattermost Compliance Export: the Channel Two",
+ "From": "user1@email",
+ "X-Mattermost-ChannelName": "the Channel Two",
+ "To": "user1@email,user5@email,user6@email",
+ "X-Mattermost-ChannelID": channel2.Id,
+ "X-Mattermost-ChannelType": "private",
+ })
+ assert.Contains(t, msg1, expectedBatchExportCh2[4], "batch 3 ch2 summary")
+ assert.Contains(t, msg1, expectedBatchExportCh2[5], "batch 3 ch2")
+
+ // now read second channel's export
+ data2 := openZipAndReadFileStartingWith(t, exportBackend, batchName, channel4.Name)
+ // clean some bad csrf if present
+ msg2 := global_relay_export.CleanTestOutput(data2)
+
+ expectedBatchExportCh4 := []string{
+ // batch 1 Summary
+ fmt.Sprintf(grE2E1Batch3SummaryCh4,
+ // 1 2 3 4
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channel4.Id,
+ // 5 6 7 8
+ conv(batchStartTime), conv(batchEndTime), users[7].Id, conv(jl[7].join)),
+
+ // batch 1
+ fmt.Sprintf(grE2E1Batch3Ch4,
+ // 1 2 3 4
+ teams[0].Id, channel4.Id, conv(batchStartTime), conv(batchEndTime),
+ // 5 6 7 8
+ users[7].Id, conv(jl[7].join), conv(jl[7].leave), conv(jobStartTime)),
+ }
+
+ // Channel 3
+ global_relay_export.AssertHeaderContains(t, msg2, map[string]string{
+ "Subject": "Mattermost Compliance Export: the Channel Four",
+ "From": "user8@email",
+ "X-Mattermost-ChannelName": "the Channel Four",
+ "To": "user8@email",
+ "X-Mattermost-ChannelID": channel4.Id,
+ "X-Mattermost-ChannelType": "public",
+ })
+ assert.Contains(t, msg2, expectedBatchExportCh4[0], "batch 3 Ch4 summary")
+ assert.Contains(t, msg2, expectedBatchExportCh4[1], "batch 3 Ch4")
+ }
+ }
+ })
+ }
+ })
+
+ t.Run("CSV e2e 1", func(t *testing.T) {
+ tests := []struct {
+ name string
+ testStopping bool
+ }{
+ {
+ name: "full tests, no stopping",
+ testStopping: false,
+ },
+ {
+ name: "full tests, stopped and resumed",
+ // This uses the same output as the previous e2e test, but tests that the job can be stopped and
+ // resumed with no change to the directory, files, file contents, or job.Data that shouldn't change.
+ // We want to be confident that jobs can resume without data missing or added from the original run.
+ testStopping: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ th := setup(t)
+ defer th.TearDown()
+ defer func() {
+ err := os.RemoveAll(exportDir)
+ assert.NoError(t, err)
+ err = os.RemoveAll(attachmentDir)
+ assert.NoError(t, err)
+ }()
+
+ ret := generateE2ETestType1Results(t, th, model.ComplianceExportTypeCsv, attachmentDir,
+ exportDir, attachmentBackend, exportBackend, tt.testStopping)
+ jl := ret.joinLeaves
+ cu := ret.createUpdateTimes
+ posts := ret.posts
+ files := ret.attachments
+ users := ret.users
+ channels := ret.channels
+ teams := ret.teams
+ batches := ret.batches
+
+ // aligned with actiance exports, for batch 1:
+ // ** except that there is no closed-out leaves at batch end
+ // for ch2:
+ // 3 participants entered
+ // message 0
+ // file for message 0 (start and ended)
+ // message 1
+ // file for message 1 (start and ended)
+ // message 2
+ // file for message 2 (start and ended)
+ // 1 participants left
+ // for ch3:
+ // 1 participant entered
+ // 1 participant left
+
+ // for batch 2:
+ // for ch2:
+ // 2 participants entered
+ // message 3
+ // file for message 3 (start and ended)
+ // message 4
+ // file for message 4 (start and ended)
+ // message 5
+ // file for message 5 (start and ended)
+ // 1 participant left
+
+ // for batch 3:
+ // for ch2:
+ // 3 participants entered
+ // message 6
+ // file for message 6 (start and ended)
+ // message 7
+ // file for message 7 (start and ended)
+ // message 8
+ // file for message 8 (start and ended)
+ // 2 participants left
+ // for ch4:
+ // 1 participant entered
+
+ expectedBatchExport := []string{
+ // batch 1
+ fmt.Sprintf(csvE2E1Batch1,
+ // 1 2 3 4 5
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, channels[1].Id,
+ // 6 7 8 9
+ users[0].Id, users[1].Id, users[2].Id, users[6].Id,
+ // 10 11 12 13 14 15
+ posts[0].Id, posts[1].Id, posts[2].Id, files[0].Id, files[1].Id, files[2].Id,
+ // 16 17 18 19 20 21 22
+ jl[0].join, jl[1].join, jl[1].leave, cu[0], jl[2].join, cu[1], jl[2].leave,
+ // 23 24 25
+ jl[6].join, jl[6].leave, cu[2]),
+
+ // batch 2
+ fmt.Sprintf(csvE2E1Batch2,
+ // 1 2 3 4
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id,
+ // 5 6 7 8 9
+ users[0].Id, users[3].Id, posts[3].Id, posts[4].Id, posts[5].Id,
+ // 10 11 12
+ files[3].Id, files[4].Id, files[5].Id,
+ // 13 14 15 16 17 18
+ jl[0].join, jl[3].join, jl[3].leave, cu[3], cu[4], cu[5]),
+
+ // batch 3
+ fmt.Sprintf(csvE2E1Batch3,
+ // 1 2 3 4 5
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, channels[2].Id,
+ // 6 7 8 9
+ users[0].Id, users[4].Id, users[5].Id, users[7].Id,
+ // 10 11 12
+ posts[6].Id, posts[7].Id, posts[8].Id,
+ // 13 14 15
+ files[6].Id, files[7].Id, files[8].Id,
+ // 16 17 18 19 20 21
+ jl[0].join, jl[4].join, jl[4].leave, cu[6], cu[7], cu[8],
+ // 22 23 24
+ jl[5].join, jl[5].leave, jl[7].join),
+ }
+
+ for batchNum, batchName := range batches {
+ export := openZipAndReadFileNum(t, exportBackend, batchName, 0)
+
+ exportLines := strings.Split(export, "\n")
+ expectedLines := strings.Split(expectedBatchExport[batchNum], "\n")
+
+ // the export is not always sorted when there are > 1 channels, so do this:
+ assert.Len(t, exportLines, len(expectedLines))
+ for _, l := range expectedLines {
+ assert.Contains(t, exportLines, l, "batch %d, batchName: %s, \nExpected:\n\n%s\n\nGot:\n\n%v\n\n", batchNum+1, batchName, l, exportLines)
+ }
+ }
+ })
+ }
+ })
+
+ t.Run("actiance e2e 2 - post from user not in channel", func(t *testing.T) {
+ th := setup(t)
+ defer th.TearDown()
+ defer func() {
+ err := os.RemoveAll(exportDir)
+ assert.NoError(t, err)
+ err = os.RemoveAll(attachmentDir)
+ assert.NoError(t, err)
+ }()
+
+ ret := generateE2ETestType2Results(t, th, model.ComplianceExportTypeActiance, attachmentDir,
+ exportDir, attachmentBackend, exportBackend)
+ channel2 := ret.channels[0]
+ start := ret.start
+ jl := ret.joinLeaves
+ posts := ret.posts
+ createUpdateTimes := ret.createUpdateTimes
+ jobEndTime := ret.jobEndTime
+ batches := ret.batches
+
+ // actiance export:
+ // 2 participants entered
+ // message 1
+ // message 2
+ // 2 participants left
+
+ // Expected data:
+ batch1xml := fmt.Sprintf(strings.TrimSpace(actianceE2E2), channel2.Id, start, jl[0].join, start,
+ posts[0].Id, createUpdateTimes[0],
+ posts[1].Id, createUpdateTimes[1],
+ jobEndTime, jobEndTime, jobEndTime)
+
+ xmlContents := openZipAndReadFile(t, exportBackend, batches[0], "actiance_export.xml")
+
+ require.Equal(t, batch1xml, xmlContents, "batch 1")
+ })
+
+ t.Run("GlobalRelay e2e 2 - post from user not in channel", func(t *testing.T) {
+ th := setup(t)
+ defer th.TearDown()
+ defer func() {
+ err := os.RemoveAll(exportDir)
+ assert.NoError(t, err)
+ err = os.RemoveAll(attachmentDir)
+ assert.NoError(t, err)
+ }()
+
+ ret := generateE2ETestType2Results(t, th, model.ComplianceExportTypeGlobalrelayZip, attachmentDir,
+ exportDir, attachmentBackend, exportBackend)
+ posts := ret.posts
+ batchTimes := ret.batchTimes
+ jobStartTime := ret.start
+ jl := ret.joinLeaves
+ batches := ret.batches
+ cu := ret.createUpdateTimes
+ users := ret.users
+ channels := ret.channels
+ teams := ret.teams
+
+ // to align with actiance export:
+ // 2 participants entered
+ // message 1
+ // message 2
+ // 2 participants left
+
+ batchStartTime := batchTimes[0].start
+ batchEndTime := batchTimes[0].end
+
+ expectedBatchExport := []string{
+ // batch 1 Summary
+ fmt.Sprintf(grE2E2Batch1Summary,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 6 7 8
+ conv(batchStartTime), conv(batchEndTime), conv(posts[0].CreateAt), conv(posts[1].CreateAt),
+ // 9 10 11 12 13 14
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id, users[1].Id,
+ // 15 16
+ posts[0].Id, posts[1].Id,
+ ),
+
+ // batch 1
+ fmt.Sprintf(grE2E2Batch1,
+ // 1 2 3 4 5
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime), conv(batchStartTime),
+ // 6 7 8 9
+ conv(batchEndTime), conv(cu[0]), conv(cu[1]), conv(jobStartTime),
+ // 10 11 12 13 14 15
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id, users[1].Id,
+ // 16 17
+ posts[0].Id, posts[1].Id,
+ ),
+ }
+
+ data := openZipAndReadFileNum(t, exportBackend, batches[0], 0)
+ // clean some bad csrf if present
+ msg := global_relay_export.CleanTestOutput(data)
+
+ // For debugging, better keep it in case we need it again.
+ //t.Logf("<><>actual\n\n%s\n\n<><>batch1 Summary:\n\n%s\n\n<><>batch1:\n%s\n", msg, expectedBatchExport[0], expectedBatchExport[1])
+
+ assert.Contains(t, msg, expectedBatchExport[0], "batch1 summary")
+ assert.Contains(t, msg, expectedBatchExport[1], "batch1")
+ })
+
+ t.Run("CSV e2e 2 - post from user not in channel", func(t *testing.T) {
+ th := setup(t)
+ defer th.TearDown()
+ defer func() {
+ err := os.RemoveAll(exportDir)
+ assert.NoError(t, err)
+ err = os.RemoveAll(attachmentDir)
+ assert.NoError(t, err)
+ }()
+
+ ret := generateE2ETestType2Results(t, th, model.ComplianceExportTypeCsv, attachmentDir,
+ exportDir, attachmentBackend, exportBackend)
+ posts := ret.posts
+ jl := ret.joinLeaves
+ batches := ret.batches
+ batchTimes := ret.batchTimes
+ cu := ret.createUpdateTimes
+ users := ret.users
+ channels := ret.channels
+ teams := ret.teams
+ batchStartTime := batchTimes[0].start
+
+ // to align with actiance export: (remember no close-out message on batch end)o
+ // 2 participants entered
+ // message 1
+ // message 2
+
+ expectedExport := fmt.Sprintf(csvE2E2Batch1,
+ // 1 2 3 4 5
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
+ // 6 7 8 9 10 11 12
+ users[1].Id, posts[0].Id, posts[1].Id, jl[0].join, batchStartTime, cu[0], cu[1])
+
+ export := openZipAndReadFileNum(t, exportBackend, batches[0], 0)
+ assert.Equal(t, expectedExport, export)
+ })
+
+ t.Run("actiance e2e 3 - test create, update, delete xml fields", func(t *testing.T) {
+ th := setup(t)
+ defer th.TearDown()
+
+ ret, type3Ret := generateE2ETestType3Results(t, th, model.ComplianceExportTypeActiance, attachmentDir, exportDir, attachmentBackend, exportBackend)
+ batches := ret.batches
+ posts := ret.posts
+ users := ret.users
+ attachments := ret.attachments
+ contents := ret.contents
+
+ // actiance export:
+ // message 0
+ // message 1
+ // message 1 deleted
+ // message 2 updated (reaction): post2 createdAt, updatedPost2 updateAt
+ // message 3 created
+ // file 3 upload start and stopped
+ // message 3 deleted
+ // file 3 deleted
+ // message 4 -- same update at as below
+ // edited message 4 -- same update at as above
+ // message 6 -- same update at as below
+ // edited message 6 -- same update at as above
+
+ // 2 participants left
+
+ for b, batchName := range batches {
+ xmlContents := openZipAndReadFile(t, exportBackend, batchName, "actiance_export.xml")
+
+ // Because 7 and 8 fall on the boundary, they could be in either batch, so save them here
+ // and then test for one or the other in each batch.
+ // The new post that got modified by message8
+ message7 := &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[6].Id,
+ UserEmail: users[0].Email,
+ UserType: "user",
+ CreateAt: posts[6].CreateAt,
+ Message: posts[6].Message, // the original message
+ UpdateAt: posts[6].UpdateAt, // the edit update at
+ UpdatedType: shared.EditedOriginalMsg,
+ EditedNewMsgId: posts[7].Id,
+ }
+ // The old post which has been edited
+ message8 := &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[7].Id,
+ UserEmail: users[0].Email,
+ UserType: "user",
+ CreateAt: posts[7].CreateAt,
+ Message: posts[7].Message, // edited message
+ UpdateAt: posts[7].UpdateAt,
+ UpdatedType: shared.EditedNewMsg,
+ }
+
+ if b == 0 {
+ // Note: for debugging, better keep this in case we need it again.
+ //t.Logf("<><> xml contents: \n\n%s\n\n", xmlContents)
+ exportedChannels := actiance_export.GetChannelExports(t, strings.NewReader(xmlContents))
+ assert.Len(t, exportedChannels, 1)
+ messages := exportedChannels[0].Messages
+ require.Len(t, messages, 10) // batch size 7 + deleted msg1, deleted ms3, 1 deleted file msg
+
+ // message 3's deleted file was uploaded in this batch period
+ fileTransferStarted := exportedChannels[0].FileStarts
+ require.Len(t, fileTransferStarted, 1)
+ fileTransferStopped := exportedChannels[0].FileStops
+ require.Len(t, fileTransferStopped, 1)
+
+ // 0 - post create
+ assert.Equal(t, &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[0].Id,
+ UserEmail: users[0].Email,
+ UserType: "user",
+ CreateAt: posts[0].CreateAt,
+ Message: posts[0].Message,
+ }, messages[0])
+
+ // 1 - post created
+ assert.Equal(t, &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[1].Id,
+ UserEmail: users[0].Email,
+ UserType: "user",
+ CreateAt: posts[1].CreateAt,
+ Message: posts[1].Message,
+ }, messages[1])
+
+ // 1 - post deleted
+ assert.Equal(t, &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[1].Id,
+ UserEmail: users[0].Email,
+ UserType: "user",
+ CreateAt: posts[1].CreateAt,
+ Message: "delete " + posts[1].Message,
+ UpdateAt: type3Ret.message1DeleteAt,
+ UpdatedType: shared.Deleted,
+ }, messages[2])
+
+ // 2 - post updated not edited (e.g., reaction)
+ assert.Equal(t, &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[2].Id,
+ UserEmail: users[0].Email,
+ UserType: "user",
+ CreateAt: posts[2].CreateAt,
+ Message: posts[2].Message,
+ UpdateAt: type3Ret.updatedPost2.UpdateAt,
+ UpdatedType: shared.UpdatedNoMsgChange,
+ }, messages[3])
+
+ // 3 - post created
+ assert.Equal(t, &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[3].Id,
+ UserEmail: users[0].Email,
+ UserType: "user",
+ CreateAt: posts[3].CreateAt,
+ Message: posts[3].Message,
+ }, messages[4])
+
+ // 3 - post deleted with a deleted file
+ assert.Equal(t, &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[3].Id,
+ UserEmail: users[0].Email,
+ UserType: "user",
+ CreateAt: posts[3].CreateAt,
+ Message: "delete " + posts[3].Message,
+ UpdateAt: type3Ret.message3AndFileInfoDeleteAt,
+ UpdatedType: shared.Deleted,
+ }, messages[5])
+
+ // file deleted message
+ assert.Equal(t, &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: messages[4].MessageId, // cheating bc we don't have this messageId
+ UserEmail: users[0].Email,
+ UserType: "user",
+ CreateAt: posts[3].CreateAt,
+ Message: "delete " + attachments[0].Path,
+ UpdateAt: type3Ret.message3AndFileInfoDeleteAt,
+ UpdatedType: shared.FileDeleted,
+ }, messages[6])
+
+ // the next messages 5, 6 can be in any order because all have equal `updateAt`s
+ // 4 - original post
+ equalUpdateAts := []*actiance_export.PostExport{
+ {
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[4].Id,
+ UserEmail: users[0].Email,
+ UserType: "user",
+ CreateAt: posts[4].CreateAt,
+ Message: posts[4].Message,
+ UpdateAt: posts[4].UpdateAt, // will be the edit update at
+ UpdatedType: shared.EditedOriginalMsg,
+ EditedNewMsgId: posts[5].Id,
+ },
+ {
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[5].Id,
+ UserEmail: users[0].Email,
+ UserType: "user",
+ CreateAt: posts[5].CreateAt,
+ Message: posts[5].Message,
+ UpdateAt: posts[5].UpdateAt,
+ UpdatedType: shared.EditedNewMsg,
+ },
+ }
+ require.ElementsMatch(t, equalUpdateAts, []*actiance_export.PostExport{
+ messages[7], messages[8]})
+ require.ElementsMatch(t, []string{posts[4].Id, posts[5].Id}, []string{messages[7].MessageId, messages[8].MessageId})
+
+ // the last message is one of the two edited or original
+ if messages[9].MessageId == message7.MessageId {
+ assert.Equal(t, message7, messages[9])
+ } else {
+ assert.Equal(t, message8, messages[9])
+ }
+
+ // only batch one has files, but they're deleted
+ zipBytes, err := exportBackend.ReadFile(batchName)
+ require.NoError(t, err)
+ zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+ require.NoError(t, err)
+
+ attachmentInZip, err := zipReader.Open(attachments[0].Path)
+ require.NoError(t, err)
+ attachmentInZipContents, err := io.ReadAll(attachmentInZip)
+ require.NoError(t, err)
+ err = attachmentInZip.Close()
+ require.NoError(t, err)
+ assert.EqualValuesf(t, contents[0], string(attachmentInZipContents), "file contents not equal")
+ }
+
+ if b == 1 {
+ exportedChannels := actiance_export.GetChannelExports(t, strings.NewReader(xmlContents))
+ assert.Len(t, exportedChannels, 1)
+ messages := exportedChannels[0].Messages
+ require.Len(t, messages, 1)
+
+ // check for either message 7 or message8
+ if messages[0].MessageId == message7.MessageId {
+ assert.Equal(t, message7, messages[0])
+ } else {
+ assert.Equal(t, message8, messages[0])
+ }
+ }
+ }
+ })
+
+ t.Run("GlobalRelay e2e 3 - test create, update, delete fields", func(t *testing.T) {
+ th := setup(t)
+ defer th.TearDown()
+
+ ret, type3Ret := generateE2ETestType3Results(t, th, model.ComplianceExportTypeGlobalrelayZip, attachmentDir,
+ exportDir, attachmentBackend, exportBackend)
+ jl := ret.joinLeaves
+ posts := ret.posts
+ batchTimes := ret.batchTimes
+ jobStartTime := ret.start
+ batches := ret.batches
+ users := ret.users
+ channels := ret.channels
+ teams := ret.teams
+
+ // to align with actiance export:
+ // message 0
+ // message 1
+ // message 1 deleted
+ // message 2 updated (reaction): post2 createdAt, updatedPost2 updateAt
+ // message 3 created
+ // file 3 upload start and stopped
+ // message 3 deleted
+ // file 3 deleted
+ // message 4 -- same update at as below
+ // edited message 4 -- same update at as above
+ // message 6 -- same update at as below
+ // edited message 6 -- same update at as above
+
+ // 2 participants left
+
+ batchStartTime := batchTimes[0].start
+ batchEndTime := batchTimes[0].end
+
+ // The comments on 10, 11, 12 show the permutation -- this is needed because 10 & 11 have same UpdateAt,
+ // and 12 & 1 (in batch 2) have same UpdateAt
+ expectedBatch1Summaries := []string{
+ // batch 1 Summary -- Permutation 1
+ fmt.Sprintf(grE2E3Batch1SummaryPerm1,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 6 7 8
+ conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
+ // 9 10 original 11 edited
+ conv(type3Ret.message3AndFileInfoDeleteAt), conv(posts[4].CreateAt), conv(posts[5].CreateAt),
+ // 12 original
+ conv(posts[6].CreateAt),
+ // 13 14 15 16 17
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
+ // 18 19 20 21 22 23 24 25
+ posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id, posts[6].Id, posts[7].Id,
+ // 26 27 28 message 4 orig/edited
+ conv(type3Ret.message1DeleteAt), conv(type3Ret.updatedPost2.UpdateAt), conv(posts[4].UpdateAt),
+ // 29 editedOriginal
+ conv(posts[6].UpdateAt)),
+
+ // batch 1 Summary -- Permutation 2
+ fmt.Sprintf(grE2E3Batch1SummaryPerm2,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 6 7 8
+ conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
+ // 9 10 original 11 edited
+ conv(type3Ret.message3AndFileInfoDeleteAt), conv(posts[4].CreateAt), conv(posts[5].CreateAt),
+ // 12 original
+ conv(posts[6].CreateAt),
+ // 13 14 15 16 17
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
+ // 18 19 20 21 22 23 24 25
+ posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id, posts[6].Id, posts[7].Id,
+ // 26 27 28 message 4 orig/edited
+ conv(type3Ret.message1DeleteAt), conv(type3Ret.updatedPost2.UpdateAt), conv(posts[4].UpdateAt),
+ // 29 editedOriginal
+ conv(posts[6].UpdateAt)),
+
+ // batch 1 summary - Permutation 3
+ fmt.Sprintf(grE2E3Batch1SummaryPerm3,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 6 7 8
+ conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
+ // 9 10 original 11 edited
+ conv(type3Ret.message3AndFileInfoDeleteAt), conv(posts[4].CreateAt), conv(posts[5].CreateAt),
+ // 12 original
+ conv(posts[6].CreateAt),
+ // 13 14 15 16 17
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
+ // 18 19 20 21 22 23 24 25
+ posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id, posts[6].Id, posts[7].Id,
+ // 26 27 28 message 4 orig/edited
+ conv(type3Ret.message1DeleteAt), conv(type3Ret.updatedPost2.UpdateAt), conv(posts[4].UpdateAt),
+ // 29 editedOriginal
+ conv(posts[6].UpdateAt)),
+
+ // batch 1 Summary -- Permutation 4
+ fmt.Sprintf(grE2E3Batch1SummaryPerm4,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 6 7 8
+ conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
+ // 9 10 original 11 edited
+ conv(type3Ret.message3AndFileInfoDeleteAt), conv(posts[4].CreateAt), conv(posts[5].CreateAt),
+ // 12 original
+ conv(posts[6].CreateAt),
+ // 13 14 15 16 17
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
+ // 18 19 20 21 22 23 24 25
+ posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id, posts[6].Id, posts[7].Id,
+ // 26 27 28 message 4 orig/edited
+ conv(type3Ret.message1DeleteAt), conv(type3Ret.updatedPost2.UpdateAt), conv(posts[4].UpdateAt),
+ // 29 editedOriginal
+ conv(posts[6].UpdateAt)),
+ }
+
+ // The comments on 10, 11, 12 show the permutation -- this is needed because 10 & 11 have same UpdateAt,
+ // and 12 & 1 (in batch 2) have same UpdateAt
+ expectedBatch1 := []string{
+ // batch 1 -- Permutation 1
+ fmt.Sprintf(grE2E3Batch1Perm1,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 6 7 8
+ conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
+ // 9 10 original 11 edited
+ conv(type3Ret.message3AndFileInfoDeleteAt), conv(posts[4].CreateAt), conv(posts[5].CreateAt),
+ // 12 original
+ conv(posts[6].CreateAt),
+ // 13 14 15 16 17
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
+ // 18 19 20 21 22 23 24 25
+ posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id, posts[6].Id, posts[7].Id,
+ // 26 27 28 message 4 orig/edited
+ conv(type3Ret.message1DeleteAt), conv(type3Ret.updatedPost2.UpdateAt), conv(posts[4].UpdateAt),
+ // 29 editedOriginal
+ conv(posts[6].UpdateAt),
+ ),
+
+ // batch 1 -- Permutation 2
+ fmt.Sprintf(grE2E3Batch1Perm2,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 6 7 8
+ conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
+ // 9 10 edited 11 original
+ conv(type3Ret.message3AndFileInfoDeleteAt), conv(posts[5].CreateAt), conv(posts[4].CreateAt),
+ // 12 original
+ conv(posts[6].CreateAt),
+ // 13 14 15 16 17
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
+ // 18 19 20 21 22 23 24 25
+ posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id, posts[6].Id, posts[7].Id,
+ // 26 27 28 message 4 orig/edited
+ conv(type3Ret.message1DeleteAt), conv(type3Ret.updatedPost2.UpdateAt), conv(posts[4].UpdateAt),
+ // 29 editedOriginal
+ conv(posts[6].UpdateAt),
+ ),
+
+ // batch 1 - Permutation 3
+ fmt.Sprintf(grE2E3Batch1Perm3,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 6 7 8
+ conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
+ // 9 10 original 11 edited
+ conv(type3Ret.message3AndFileInfoDeleteAt), conv(posts[4].CreateAt), conv(posts[5].CreateAt),
+ // 12 edited
+ conv(posts[7].CreateAt),
+ // 13 14 15 16 17
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
+ // 18 19 20 21 22 23 24 25
+ posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id, posts[6].Id, posts[7].Id,
+ // 26 27 28 message 4 orig/edited
+ conv(type3Ret.message1DeleteAt), conv(type3Ret.updatedPost2.UpdateAt), conv(posts[4].UpdateAt),
+ // 29 editedOriginal
+ conv(posts[6].UpdateAt),
+ ),
+
+ // batch 1 -- Permutation 4
+ fmt.Sprintf(grE2E3Batch1Perm4,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 6 7 8
+ conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
+ // 9 10 edited 11 original
+ conv(type3Ret.message3AndFileInfoDeleteAt), conv(posts[5].CreateAt), conv(posts[4].CreateAt),
+ // 12 edited
+ conv(posts[7].CreateAt),
+ // 13 14 15 16 17
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
+ // 18 19 20 21 22 23 24 25
+ posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id, posts[6].Id, posts[7].Id,
+ // 26 27 28 message 4 orig/edited
+ conv(type3Ret.message1DeleteAt), conv(type3Ret.updatedPost2.UpdateAt), conv(posts[4].UpdateAt),
+ // 29 editedOriginal
+ conv(posts[6].UpdateAt),
+ ),
+ }
+
+ batchStartTime = batchTimes[1].start
+ batchEndTime = batchTimes[1].end
+
+ expectedBatch2Summaries := []string{
+ // batch 2 Summary -- Permutation 1
+ fmt.Sprintf(grE2E3Batch2SummaryPerm1,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 edited
+ conv(posts[7].CreateAt),
+ // 6 7 8 9 10 11
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id, posts[0].Id,
+ // 12 13 14
+ conv(posts[6].UpdateAt), posts[6].Id, posts[7].Id,
+ ),
+
+ // batch 2 Summary -- Permutation 2
+ fmt.Sprintf(grE2E3Batch2SummaryPerm2,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 original
+ conv(posts[6].CreateAt),
+ // 6 7 8 9 10 11
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id, posts[7].Id,
+ // 12 13 14
+ conv(posts[6].UpdateAt), posts[6].Id, posts[7].Id),
+ }
+
+ expectedBatch2 := []string{
+ // batch 2 Summary -- Permutation 1
+ fmt.Sprintf(grE2E3Batch2Perm1,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 edited
+ conv(posts[7].CreateAt),
+ // 6 7 8 9 10 11
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id, posts[7].Id,
+ // 12 13 14
+ conv(posts[6].UpdateAt), posts[6].Id, posts[7].Id,
+ // 15
+ conv(jobStartTime)),
+
+ // batch 2 Summary -- Permutation 2
+ fmt.Sprintf(grE2E3Batch2Perm2,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 original
+ conv(posts[6].CreateAt),
+ // 6 7 8 9 10 11
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id, posts[7].Id,
+ // 12 13 14
+ conv(posts[6].UpdateAt), posts[6].Id, posts[7].Id,
+ // 15
+ conv(jobStartTime)),
+ }
+
+ data := openZipAndReadFileNum(t, exportBackend, batches[0], 0)
+ // clean some bad csrf if present
+ msg := global_relay_export.CleanTestOutput(data)
+
+ // Note: for debugging, better keep this in case we need it again.
+ //t.Logf("<><>batch1 actual\n\n%s\n\n<><>batch1 Summary Perm1:\n\n%s\n\n<><>batch1 Summary Perm2:\n\n%s\n\n<><>batch1 Summary Perm3:\n\n%s\n\n<><>batch1 Summary Perm4:\n\n%s\n\n",
+ // msg, expectedBatch1Summaries[0], expectedBatch1Summaries[1], expectedBatch1Summaries[2], expectedBatch1Summaries[3])
+ //t.Logf("<><>batch1 actual\n\n%s\n\n<><>batch1 Perm1:\n\n%s\n\n<><>batch1 Perm2:\n\n%s\n\n<><>batch1 Perm3:\n\n%s\n\n<><>batch1 Perm4:\n\n%s\n\n",
+ // msg, expectedBatch1[0], expectedBatch1[1], expectedBatch1[2], expectedBatch1[3])
+
+ matched := dataContainsOneOfExpected(msg, expectedBatch1Summaries)
+
+ assert.True(t, matched, "batch 1 summary didn't match one of the expected permutations")
+
+ matched = dataContainsOneOfExpected(msg, expectedBatch1)
+ assert.True(t, matched, "batch 1 body didn't match one of the expected permutations")
+
+ data = openZipAndReadFileNum(t, exportBackend, batches[1], 0)
+ // clean some bad csrf if present
+ msg = global_relay_export.CleanTestOutput(data)
+
+ // Note: for debugging, better keep this in case we need it again.
+ //t.Logf("<><>batch2 actual\n\n%s\n\n<><>batch2 Summary Perm1:\n\n%s\n\n<><>batch2 Summary Perm2:\n\n%s\n\n",
+ // msg, expectedBatch2Summaries[0], expectedBatch2Summaries[1])
+ //t.Logf("<><>batch2 actual\n\n%s\n\n<><>batch2 Perm1:\n\n%s\n\n<><>batch2 Perm2:\n\n%s\n\n",
+ // msg, expectedBatch2[0], expectedBatch2[1])
+
+ matched = dataContainsOneOfExpected(msg, expectedBatch2Summaries)
+ assert.True(t, matched, "batch 2 summary didn't match one of the expected permutations")
+
+ matched = dataContainsOneOfExpected(msg, expectedBatch2)
+ assert.True(t, matched, "batch 2 body didn't match one of the expected permutations")
+ })
+
+ t.Run("CSV e2e 3 - test create, update, delete fields", func(t *testing.T) {
+ th := setup(t)
+ defer th.TearDown()
+
+ ret, ret3 := generateE2ETestType3Results(t, th, model.ComplianceExportTypeCsv, attachmentDir,
+ exportDir, attachmentBackend, exportBackend)
+ jl := ret.joinLeaves
+ posts := ret.posts
+ //cu := ret.createUpdateTimes
+ //jobStartTime := ret.start
+ batches := ret.batches
+ users := ret.users
+ channels := ret.channels
+ teams := ret.teams
+ attachments := ret.attachments
+
+ // aligned with actiance export:
+ // message 0
+ // message 1
+ // message 1 deleted
+ // message 2 updated (reaction): post2 createdAt, updatedPost2 updateAt
+ // message 3 created
+ // file 3 upload start and stopped
+ // message 3 deleted
+ // file 3 deleted
+ // message 4 -- same update at as below
+ // edited message 4 -- same update at as above
+ // message 6 -- same update at as below
+ // edited message 6 -- same update at as above
+
+ // NOTE: the comments describe the order of the last three messages (21 22 23),
+ // eg in perm1: message 4, edited message 4, and message 6
+ expectedExports := []string{
+ // original, edited, "original" msg 6
+ fmt.Sprintf(csvE2E3Batch1Perm1,
+ // 1 2 3 4 5
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
+ // 6 7 8 9 10 11 edited
+ posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id,
+ // 12 13
+ posts[6].Id, attachments[0].Id,
+ // 14 15 16 17 18
+ jl[0].join, posts[0].CreateAt, posts[1].CreateAt, ret3.message1DeleteAt, posts[2].CreateAt,
+ // 19 20 21 22 23
+ posts[3].CreateAt, ret3.message3AndFileInfoDeleteAt, posts[4].CreateAt, posts[4].UpdateAt, posts[6].CreateAt,
+ // 24 editedBy for message 6, 25 26
+ posts[7].Id, ret3.updatedPost2.UpdateAt, posts[6].UpdateAt),
+
+ // edited, original, original
+ fmt.Sprintf(csvE2E3Batch1Perm2,
+ // 1 2 3 4 5
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
+ // 6 7 8 9 10 11 edited
+ posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id,
+ // 12 13
+ posts[6].Id, attachments[0].Id,
+ // 14 15 16 17 18
+ jl[0].join, posts[0].CreateAt, posts[1].CreateAt, ret3.message1DeleteAt, posts[2].CreateAt,
+ // 19 20 21 22 23
+ posts[3].CreateAt, ret3.message3AndFileInfoDeleteAt, posts[4].CreateAt, posts[4].UpdateAt, posts[6].CreateAt,
+ // 24 editedBy for message 6, 25 26
+ posts[7].Id, ret3.updatedPost2.UpdateAt, posts[6].UpdateAt),
+
+ // original, edited, edited
+ fmt.Sprintf(csvE2E3Batch1Perm3,
+ // 1 2 3 4 5
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
+ // 6 7 8 9 10 11 edited
+ posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id,
+ // 12 id of edited msg6 13
+ posts[7].Id, attachments[0].Id,
+ // 14 15 16 17 18
+ jl[0].join, posts[0].CreateAt, posts[1].CreateAt, ret3.message1DeleteAt, posts[2].CreateAt,
+ // 19 20 21 22 23
+ posts[3].CreateAt, ret3.message3AndFileInfoDeleteAt, posts[4].CreateAt, posts[4].UpdateAt, posts[6].CreateAt,
+ // 24 25 26
+ ret3.updatedPost2.UpdateAt, posts[6].CreateAt, posts[6].UpdateAt),
+
+ // edited, original, edited
+ fmt.Sprintf(csvE2E3Batch1Perm4,
+ // 1 2 3 4 5
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
+ // 6 7 8 9 10 11 edited
+ posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id,
+ // 12 id of edited msg6 13
+ posts[7].Id, attachments[0].Id,
+ // 14 15 16 17 18
+ jl[0].join, posts[0].CreateAt, posts[1].CreateAt, ret3.message1DeleteAt, posts[2].CreateAt,
+ // 19 20 21 22
+ posts[3].CreateAt, ret3.message3AndFileInfoDeleteAt, posts[4].CreateAt, posts[4].UpdateAt,
+ // 23 24 25 26
+ posts[6].CreateAt, ret3.updatedPost2.UpdateAt, posts[6].CreateAt, posts[6].UpdateAt),
+ }
+
+ export := openZipAndReadFileNum(t, exportBackend, batches[0], 0)
+
+ matched := false
+ for _, perm := range expectedExports {
+ if export == perm {
+ matched = true
+ break
+ }
+ }
+ assert.True(t, matched, "batch 1 didn't match one of the expected permutations")
+
+ expectedExports = []string{
+ // original message 6 (which says "message 6" in the message, and has new id -- which is post 6's id)
+ fmt.Sprintf(csvE2E3Batch2Perm1,
+ // 1 2 3 4 5
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
+ // 6
+ posts[6].Id,
+ // 7 this is the editedBy for message 6 -- editedBy is confusing (cause it's the original id), but it is what it is
+ posts[7].Id,
+ // 8 9 10
+ posts[6].CreateAt, jl[0].join, posts[6].UpdateAt),
+
+ // edited message 6 (which says "edited message 6" in the message, and has original id)
+ fmt.Sprintf(csvE2E3Batch2Perm2,
+ // 1 2 3 4 5
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
+ // 6 7 8 9
+ posts[7].Id, posts[6].CreateAt, jl[0].join, posts[6].UpdateAt),
+ }
+
+ export = openZipAndReadFileNum(t, exportBackend, batches[1], 0)
+
+ matched = false
+ for _, perm := range expectedExports {
+ if export == perm {
+ matched = true
+ break
+ }
+ }
+ assert.True(t, matched, "batch 2 didn't match one of the expected permutations")
+ })
+
+ t.Run("actiance e2e 4 - test edits with multiple simultaneous updates", func(t *testing.T) {
+ th := setup(t)
+ defer th.TearDown()
+
+ ret := generateE2ETestType4Results(t, th, model.ComplianceExportTypeActiance, attachmentDir, exportDir, attachmentBackend, exportBackend)
+ batch001 := ret.batches[0]
+ posts := ret.posts
+ users := ret.users
+
+ xmlContents := openZipAndReadFile(t, exportBackend, batch001, "actiance_export.xml")
+
+ exportedChannels := actiance_export.GetChannelExports(t, strings.NewReader(xmlContents))
+ assert.Len(t, exportedChannels, 1)
+ messages := exportedChannels[0].Messages
+ require.Len(t, messages, 5)
+
+ // the messages can be in any order because all have equal `updateAt`s
+ equalUpdateAts := []*actiance_export.PostExport{
+ {
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[0].Id,
+ UserEmail: users[0].Email,
+ UserType: "user",
+ CreateAt: posts[0].CreateAt,
+ Message: posts[0].Message,
+ UpdateAt: posts[1].UpdateAt, // the edit update at
+ UpdatedType: shared.EditedOriginalMsg,
+ EditedNewMsgId: posts[1].Id,
+ },
+ {
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[1].Id,
+ UserEmail: users[0].Email,
+ UserType: "user",
+ CreateAt: posts[1].CreateAt,
+ Message: posts[1].Message,
+ UpdateAt: posts[1].UpdateAt,
+ UpdatedType: shared.EditedNewMsg,
+ },
+ {
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[2].Id,
+ UserEmail: users[0].Email,
+ UserType: "user",
+ CreateAt: posts[2].CreateAt,
+ Message: posts[2].Message,
+ },
+ {
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[3].Id,
+ UserEmail: users[0].Email,
+ UserType: "user",
+ CreateAt: posts[3].CreateAt,
+ Message: posts[3].Message,
+ },
+ {
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[4].Id,
+ UserEmail: users[0].Email,
+ UserType: "user",
+ CreateAt: posts[4].CreateAt,
+ Message: posts[4].Message,
+ },
+ }
+ require.ElementsMatch(t, equalUpdateAts, []*actiance_export.PostExport{
+ messages[0], messages[1], messages[2], messages[3], messages[4]})
+ })
+
+ t.Run("GlobalRelay e2e 4 - test edits with multiple simultaneous updates", func(t *testing.T) {
+ th := setup(t)
+ defer th.TearDown()
+
+ ret := generateE2ETestType4Results(t, th, model.ComplianceExportTypeGlobalrelayZip, attachmentDir,
+ exportDir, attachmentBackend, exportBackend)
+ jl := ret.joinLeaves
+ posts := ret.posts
+ batchTimes := ret.batchTimes
+ //jobStartTime := ret.start
+ batches := ret.batches
+ users := ret.users
+ channels := ret.channels
+ teams := ret.teams
+
+ batchStartTime := batchTimes[0].start
+ batchEndTime := batchTimes[0].end
+
+ // summaryHeader
+ allExpected := []string{fmt.Sprintf(grE2E4Summary,
+ // 1 2 3 4
+ conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
+ // 5 6 7 8 9
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
+ // 10 11 12 13
+ conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
+ // 14 15 16 17 18 19
+ conv(posts[4].CreateAt), posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id,
+ )}
+
+ // summary body
+ allExpected = append(allExpected,
+ fmt.Sprintf("%[1]s %[2]s @user1 %[3]s @user1 user (user1@email) edited message 0 EditedNewMsg %[4]s",
+ posts[1].Id, conv(posts[0].CreateAt), users[0].Id, conv(posts[0].UpdateAt)),
+ fmt.Sprintf("* %[1]s %[2]s @user1 %[3]s @user1 user (user1@email) message 0 EditedOriginalMsg %[4]s %[5]s",
+ posts[0].Id, conv(posts[1].CreateAt), users[0].Id, conv(posts[0].UpdateAt), posts[1].Id),
+ fmt.Sprintf("* %[1]s %[2]s @user1 %[3]s @user1 user (user1@email) message 2",
+ posts[2].Id, conv(posts[2].CreateAt), users[0].Id),
+ fmt.Sprintf("* %[1]s %[2]s @user1 %[3]s @user1 user (user1@email) message 3",
+ posts[3].Id, conv(posts[3].CreateAt), users[0].Id),
+ fmt.Sprintf("* %[1]s %[2]s @user1 %[3]s @user1 user (user1@email) message 4",
+ posts[4].Id, conv(posts[4].CreateAt), users[0].Id),
+ )
+
+ // first two are channel and participants, rest are messages
+ allExpected = append(allExpected,
+ fmt.Sprintf("\n
\n - TeamId:%[1]s
\n - TeamName:%[2]s
\n - TeamDisplayName:%[3]s
\n - ChannelId:%[4]s
\n - ChannelName:channel_two_name
\n - ChannelDisplayName:the Channel Two
\n - Started:%[5]s
\n - Ended:%[6]s
\n - Duration:0 seconds
\n
\n
\n",
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, conv(batchStartTime), conv(batchEndTime)),
+ fmt.Sprintf("%[1]s | \n @user1 | \n user | \n user1@email | \n %[2]s | \n %[3]s | \n 0 seconds | \n 5 | ",
+ users[0].Id, conv(jl[0].join), conv(batchEndTime)),
+ fmt.Sprintf("%[1]s\n %[2]s\n @user1\n %[3]s\n @user1\n user\n (user1@email)\n edited message 0\n EditedNewMsg\n %[4]s\n ",
+ posts[1].Id, conv(posts[1].CreateAt), users[0].Id, conv(posts[1].UpdateAt)),
+ fmt.Sprintf("%[1]s\n %[2]s\n @user1\n %[3]s\n @user1\n user\n (user1@email)\n message 0\n EditedOriginalMsg\n %[4]s\n %[5]s",
+ posts[0].Id, conv(posts[0].CreateAt), users[0].Id, conv(posts[0].UpdateAt), posts[1].Id),
+ fmt.Sprintf("%[1]s\n %[2]s\n @user1\n %[3]s\n @user1\n user\n (user1@email)\n message 2",
+ posts[2].Id, conv(posts[2].CreateAt), users[0].Id, conv(posts[2].UpdateAt)),
+ fmt.Sprintf("%[1]s\n %[2]s\n @user1\n %[3]s\n @user1\n user\n (user1@email)\n message 3",
+ posts[3].Id, conv(posts[3].CreateAt), users[0].Id, conv(posts[3].UpdateAt)),
+ fmt.Sprintf("%[1]s\n %[2]s\n @user1\n %[3]s\n @user1\n user\n (user1@email)\n message 4",
+ posts[4].Id, conv(posts[4].CreateAt), users[0].Id, conv(posts[4].UpdateAt)),
+ )
+
+ data := openZipAndReadFileNum(t, exportBackend, batches[0], 0)
+ // clean some bad csrf if present
+ msg := global_relay_export.CleanTestOutput(data)
+
+ for _, expected := range allExpected {
+ assert.Contains(t, msg, expected, "expected exported msg to contain: \n%s\n\nExported msg:\n%s\n", expected, msg)
+ if !strings.Contains(msg, expected) {
+ break
+ }
+ }
+ })
+
+ t.Run("CSV e2e 4 - test edits with multiple simultaneous updates", func(t *testing.T) {
+ th := setup(t)
+ defer th.TearDown()
+
+ ret := generateE2ETestType4Results(t, th, model.ComplianceExportTypeCsv, attachmentDir,
+ exportDir, attachmentBackend, exportBackend)
+ jl := ret.joinLeaves
+ posts := ret.posts
+ batches := ret.batches
+ users := ret.users
+ channels := ret.channels
+ teams := ret.teams
+
+ // fill out the lines using the template (easier to read than if it were an inline string)
+ tmpl := fmt.Sprintf(csvE2E4Batch1,
+ // 1 2 3 4 5
+ teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
+ // 6 7 8 9 10
+ posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id,
+ // 11 12 13 14
+ jl[0].join, posts[0].CreateAt, posts[2].CreateAt, posts[1].UpdateAt)
+
+ allExpected := strings.Split(tmpl, "\n")
+
+ export := openZipAndReadFileNum(t, exportBackend, batches[0], 0)
+
+ assert.Len(t, strings.Split(export, "\n"), len(allExpected)+1) // +1 for header line
+
+ for _, expected := range allExpected {
+ assert.Contains(t, export, expected, "expected export to contain: \n%s\n\nExport:\n%s\n", expected, export)
+ if !strings.Contains(export, expected) {
+ break
+ }
+ }
+ })
+
+ t.Run("actiance e2e 5 - test delete and update semantics", func(t *testing.T) {
+ th := setup(t)
+ defer th.TearDown()
+
+ rets, ret5s := generateE2ETestType5Results(t, th, model.ComplianceExportTypeActiance, attachmentDir, exportDir, attachmentBackend, exportBackend)
+ posts := rets[0].posts
+ message0DeleteAt := ret5s[0].message0DeleteAt
+ zipBytes := ret5s[0].zipBytes[0]
+
+ //
+ // Job 1
+ //
+ xmlContents := readFileFromZip(t, zipBytes, "actiance_export.xml")
+
+ exportedChannels := actiance_export.GetChannelExports(t, strings.NewReader(xmlContents))
+ assert.Len(t, exportedChannels, 1)
+ messages := exportedChannels[0].Messages
+ require.Len(t, messages, 2) // 1 posted, 1 deleted
+
+ // post created
+ require.Equal(t, &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[0].Id,
+ UserEmail: th.BasicUser.Email,
+ UserType: "user",
+ CreateAt: posts[0].CreateAt,
+ Message: posts[0].Message,
+ }, messages[0])
+
+ // 1 - post deleted
+ require.Equal(t, &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[0].Id,
+ UserEmail: th.BasicUser.Email,
+ UserType: "user",
+ CreateAt: posts[0].CreateAt,
+ Message: "delete " + posts[0].Message,
+ UpdateAt: message0DeleteAt,
+ UpdatedType: shared.Deleted,
+ }, messages[1])
+
+ //
+ // Job 2
+ //
+ posts = rets[1].posts
+ message0DeleteAt = ret5s[1].message0DeleteAt
+ zipBytes = ret5s[1].zipBytes[0]
+ zipBytes2 := ret5s[1].zipBytes[1]
+
+ xmlContents = readFileFromZip(t, zipBytes, "actiance_export.xml")
+
+ exportedChannels = actiance_export.GetChannelExports(t, strings.NewReader(xmlContents))
+ assert.Len(t, exportedChannels, 1)
+ messages = exportedChannels[0].Messages
+ require.Len(t, messages, 1) // 1 posted
+
+ // post created
+ require.Equal(t, &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[1].Id,
+ UserEmail: th.BasicUser.Email,
+ UserType: "user",
+ CreateAt: posts[1].CreateAt,
+ Message: posts[1].Message,
+ }, messages[0])
+
+ xmlContents = readFileFromZip(t, zipBytes2, "actiance_export.xml")
+
+ exportedChannels = actiance_export.GetChannelExports(t, strings.NewReader(xmlContents))
+ assert.Len(t, exportedChannels, 1)
+ messages = exportedChannels[0].Messages
+ require.Len(t, messages, 2) // message0's create and delete messages
+
+ // post created
+ require.Equal(t, &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[0].Id,
+ UserEmail: th.BasicUser.Email,
+ UserType: "user",
+ CreateAt: posts[0].CreateAt,
+ Message: posts[0].Message,
+ }, messages[0])
+
+ // 1 - post deleted
+ require.Equal(t, &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[0].Id,
+ UserEmail: th.BasicUser.Email,
+ UserType: "user",
+ CreateAt: posts[0].CreateAt,
+ Message: "delete " + posts[0].Message,
+ UpdateAt: message0DeleteAt,
+ UpdatedType: shared.Deleted,
+ }, messages[1])
+
+ //
+ // Job 3
+ //
+ posts = rets[2].posts
+ zipBytes = ret5s[2].zipBytes[0]
+
+ xmlContents = readFileFromZip(t, zipBytes, "actiance_export.xml")
+
+ exportedChannels = actiance_export.GetChannelExports(t, strings.NewReader(xmlContents))
+ assert.Len(t, exportedChannels, 1)
+ messages = exportedChannels[0].Messages
+ require.Len(t, messages, 2) // 2 posted
+
+ // post created
+ require.Equal(t, &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[0].Id,
+ UserEmail: th.BasicUser.Email,
+ UserType: "user",
+ CreateAt: posts[0].CreateAt,
+ Message: posts[0].Message,
+ }, messages[0])
+
+ // post created
+ require.Equal(t, &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[1].Id,
+ UserEmail: th.BasicUser.Email,
+ UserType: "user",
+ CreateAt: posts[1].CreateAt,
+ Message: posts[1].Message,
+ }, messages[1])
+
+ //
+ // Job 4
+ //
+ posts = rets[3].posts
+ updatedPost1 := ret5s[3].updatedPost1
+ message0DeleteAt = ret5s[3].message0DeleteAt
+ zipBytes = ret5s[3].zipBytes[0]
+
+ xmlContents = readFileFromZip(t, zipBytes, "actiance_export.xml")
+
+ exportedChannels = actiance_export.GetChannelExports(t, strings.NewReader(xmlContents))
+ assert.Len(t, exportedChannels, 1)
+ messages = exportedChannels[0].Messages
+ require.Len(t, messages, 3) //filler post, deleted post, and updated posts ONLY
+
+ // post created
+ require.Equal(t, &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[2].Id,
+ UserEmail: th.BasicUser.Email,
+ UserType: "user",
+ CreateAt: posts[2].CreateAt,
+ Message: posts[2].Message,
+ }, messages[0])
+
+ // post deleted ONLY (not its created post, because that was in the previous job)
+ require.Equal(t, &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[0].Id,
+ UserEmail: th.BasicUser.Email,
+ UserType: "user",
+ CreateAt: posts[0].CreateAt,
+ Message: "delete " + posts[0].Message,
+ UpdateAt: message0DeleteAt,
+ UpdatedType: shared.Deleted,
+ }, messages[1])
+
+ // post updated ONLY (not its created post, because that was in the previous job)
+ require.Equal(t, &actiance_export.PostExport{
+ XMLName: xml.Name{Local: "Message"},
+ MessageId: posts[1].Id,
+ UserEmail: th.BasicUser.Email,
+ UserType: "user",
+ CreateAt: posts[1].CreateAt,
+ Message: posts[1].Message,
+ UpdateAt: updatedPost1.UpdateAt,
+ UpdatedType: shared.UpdatedNoMsgChange,
+ }, messages[2])
+ })
+
+ t.Run("GlobalRelay e2e 5 - test delete and update semantics", func(t *testing.T) {
+ // regMsg has strings in pos: 1: post_id, 2: sent_time, 3: username, 4: userId, 5: email, 6: message
+ regMsgTmpl := "\n %[1]s\n %[2]s\n @%[3]s\n %[4]s\n @%[3]s\n user\n (%[5]s)\n %[6]s\n"
+ // updatedMsgTmpl has strings in pos: 1: post_id, 2: sent_time, 3: username, 4: userId, 5: email, 6: message, 7: update_type, 8: update_time, 9: edited_new_msg_id
+ updatedMsgTmpl := "\n %[1]s\n %[2]s\n @%[3]s\n %[4]s\n @%[3]s\n user\n (%[5]s)\n %[6]s\n %[7]s\n %[8]s\n %[9]s\n\n"
+
+ assertContainsAllMsgs := func(msg string, allExpected []string, tag string) {
+ for _, expected := range allExpected {
+ assert.Contains(t, msg, expected, "%s, expected exported msg to contain: \n%s\n\nExported msg:\n%s\n", tag, expected, msg)
+ if !strings.Contains(msg, expected) {
+ break
+ }
+ }
+
+ numMessages := 0
+ for _, l := range strings.Split(msg, "\n") {
+ if strings.Contains(l, "") {
+ numMessages += 1
+ }
+ }
+
+ assert.Equal(t, len(allExpected), numMessages, tag)
+ }
+
+ th := setup(t)
+ defer th.TearDown()
+
+ rets, ret5s := generateE2ETestType5Results(t, th, model.ComplianceExportTypeGlobalrelayZip, attachmentDir, exportDir, attachmentBackend, exportBackend)
+ posts := rets[0].posts
+ zipBytes := ret5s[0].zipBytes[0]
+
+ //
+ // Job 1
+ //
+ data := readFilenumFromZip(t, zipBytes, 0)
+ // clean some bad csrf if present
+ msg := global_relay_export.CleanTestOutput(data)
+ message0DeleteAt := ret5s[0].message0DeleteAt
+
+ // post created
+ allExpected := []string{
+ // 1 2 3 4 5 6
+ fmt.Sprintf(regMsgTmpl, posts[0].Id, conv(posts[0].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, posts[0].Message),
+ // 1 2 3 4 5 6 7 8 9
+ fmt.Sprintf(updatedMsgTmpl, posts[0].Id, conv(posts[0].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, "delete "+posts[0].Message, shared.Deleted, conv(message0DeleteAt), ""),
+ }
+ assertContainsAllMsgs(msg, allExpected, "job 1")
+
+ //
+ // Job 2
+ //
+ posts = rets[1].posts
+ message0DeleteAt = ret5s[1].message0DeleteAt
+ zipBytes = ret5s[1].zipBytes[0]
+ zipBytes2 := ret5s[1].zipBytes[1]
+
+ data = readFilenumFromZip(t, zipBytes, 0)
+ // clean some bad csrf if present
+ msg = global_relay_export.CleanTestOutput(data)
+
+ // post created
+ allExpected = []string{
+ // 1 2 3 4 5 6
+ fmt.Sprintf(regMsgTmpl, posts[1].Id, conv(posts[1].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, posts[1].Message),
+ }
+ assertContainsAllMsgs(msg, allExpected, "job 2a")
+
+ data = readFilenumFromZip(t, zipBytes2, 0)
+ // clean some bad csrf if present
+ msg = global_relay_export.CleanTestOutput(data)
+
+ allExpected = []string{
+ // 1 2 3 4 5 6
+ fmt.Sprintf(regMsgTmpl, posts[0].Id, conv(posts[0].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, posts[0].Message),
+ // 1 2 3 4 5 6 7 8 9
+ fmt.Sprintf(updatedMsgTmpl, posts[0].Id, conv(posts[0].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, "delete "+posts[0].Message, shared.Deleted, conv(message0DeleteAt), ""),
+ }
+ assertContainsAllMsgs(msg, allExpected, "job 2b")
+
+ //
+ // Job 3
+ //
+ posts = rets[2].posts
+ zipBytes = ret5s[2].zipBytes[0]
+
+ data = readFilenumFromZip(t, zipBytes, 0)
+ // clean some bad csrf if present
+ msg = global_relay_export.CleanTestOutput(data)
+
+ // post created
+ allExpected = []string{
+ // 1 2 3 4 5 6
+ fmt.Sprintf(regMsgTmpl, posts[0].Id, conv(posts[0].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, posts[0].Message),
+ // 1 2 3 4 5 6
+ fmt.Sprintf(regMsgTmpl, posts[1].Id, conv(posts[1].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, posts[1].Message),
+ }
+ assertContainsAllMsgs(msg, allExpected, "job 3")
+
+ //
+ // Job 4
+ //
+ posts = rets[3].posts
+ updatedPost1 := ret5s[3].updatedPost1
+ message0DeleteAt = ret5s[3].message0DeleteAt
+ zipBytes = ret5s[3].zipBytes[0]
+
+ data = readFilenumFromZip(t, zipBytes, 0)
+ // clean some bad csrf if present
+ msg = global_relay_export.CleanTestOutput(data)
+
+ allExpected = []string{
+ // post created
+ // 1 2 3 4 5 6
+ fmt.Sprintf(regMsgTmpl, posts[2].Id, conv(posts[2].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, posts[2].Message),
+
+ // post deleted ONLY (not its created post, because that was in the previous job)
+ // 1 2 3 4 5 6 7 8 9
+ fmt.Sprintf(updatedMsgTmpl, posts[0].Id, conv(posts[0].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, "delete "+posts[0].Message, shared.Deleted, conv(message0DeleteAt), ""),
+
+ // post updated ONLY (not its created post, because that was in the previous job)
+ // 1 2 3 4 5 6 7 8 9
+ fmt.Sprintf(updatedMsgTmpl, posts[1].Id, conv(posts[1].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, posts[1].Message, shared.UpdatedNoMsgChange, conv(updatedPost1.UpdateAt), ""),
+ }
+ assertContainsAllMsgs(msg, allExpected, "job 4")
+ })
+
+ t.Run("CSV e2e 5 - test delete and update semantics", func(t *testing.T) {
+ type msgDetailsToCheck struct {
+ createAt int64
+ updateAt int64
+ updatedType shared.PostUpdatedType
+ postId string
+ message string
+ }
+
+ assertContainsAllMsgs := func(msg string, allExpected []msgDetailsToCheck, tag string) {
+ allLines := strings.Split(strings.Trim(msg, " \n"), "\n")
+ allLines = allLines[1:] // remove header
+ var msgLines []string
+ for _, l := range allLines {
+ if !strings.Contains(l, "previously-joined") {
+ msgLines = append(msgLines, l)
+ }
+ }
+
+ assert.Equal(t, len(allExpected), len(msgLines), tag)
+
+ for _, expected := range allExpected {
+ found := false
+ for _, msg := range msgLines {
+ if strings.HasPrefix(msg, fmt.Sprintf("%d,%d,%s,",
+ expected.createAt, expected.updateAt, expected.updatedType)) &&
+ strings.Contains(msg, expected.postId+",,,"+expected.message) {
+ found = true
+ break
+ }
+ }
+ assert.True(t, found, "%s actual msg did not contain expected msg. msg: \n%s\nexpected msg details: %v\n", tag, msg, expected)
+ }
+ }
+
+ th := setup(t)
+ defer th.TearDown()
+
+ rets, ret5s := generateE2ETestType5Results(t, th, model.ComplianceExportTypeCsv, attachmentDir, exportDir, attachmentBackend, exportBackend)
+ posts := rets[0].posts
+ message0DeleteAt := ret5s[0].message0DeleteAt
+ zipBytes := ret5s[0].zipBytes[0]
+
+ // NOTE: we know csv outputs correctly from above, so just test that the right post IDs are being exported
+
+ //
+ // Job 1
+ //
+ export := readFilenumFromZip(t, zipBytes, 0)
+
+ // post created, post deleted
+ assertContainsAllMsgs(export, []msgDetailsToCheck{
+ {
+ createAt: posts[0].CreateAt,
+ updateAt: message0DeleteAt,
+ postId: posts[0].Id,
+ message: posts[0].Message,
+ },
+ {
+ createAt: posts[0].CreateAt,
+ updateAt: message0DeleteAt,
+ updatedType: shared.Deleted,
+ postId: posts[0].Id,
+ message: "delete " + posts[0].Message,
+ },
+ }, "Job 1")
+
+ //
+ // Job 2
+ //
+ posts = rets[1].posts
+ message0DeleteAt = ret5s[1].message0DeleteAt
+ zipBytes = ret5s[1].zipBytes[0]
+ zipBytes2 := ret5s[1].zipBytes[1]
+
+ export = readFilenumFromZip(t, zipBytes, 0)
+
+ // post created
+ assertContainsAllMsgs(export, []msgDetailsToCheck{
+ {
+ createAt: posts[1].CreateAt,
+ updateAt: posts[1].UpdateAt,
+ postId: posts[1].Id,
+ message: posts[1].Message,
+ },
+ }, "Job 2 batch 1")
+
+ export = readFilenumFromZip(t, zipBytes2, 0)
+
+ // post created
+ assertContainsAllMsgs(export, []msgDetailsToCheck{
+ {
+ createAt: posts[0].CreateAt,
+ updateAt: message0DeleteAt,
+ postId: posts[0].Id,
+ message: posts[0].Message,
+ },
+ {
+ createAt: posts[0].CreateAt,
+ updateAt: message0DeleteAt,
+ updatedType: shared.Deleted,
+ postId: posts[0].Id,
+ message: "delete " + posts[0].Message,
+ },
+ }, "Job 2 batch 2")
+
+ // Job 3
+ //
+ posts = rets[2].posts
+ zipBytes = ret5s[2].zipBytes[0]
+
+ export = readFilenumFromZip(t, zipBytes, 0)
+
+ // 2 posts created
+ assertContainsAllMsgs(export, []msgDetailsToCheck{
+ {
+ createAt: posts[0].CreateAt,
+ updateAt: posts[0].UpdateAt,
+ postId: posts[0].Id,
+ message: posts[0].Message,
+ },
+ {
+ createAt: posts[1].CreateAt,
+ updateAt: posts[1].UpdateAt,
+ postId: posts[1].Id,
+ message: posts[1].Message,
+ },
+ }, "Job 3")
+
+ //
+ // Job 4
+ //
+ posts = rets[3].posts
+ updatedPost1 := ret5s[3].updatedPost1
+ message0DeleteAt = ret5s[3].message0DeleteAt
+ zipBytes = ret5s[3].zipBytes[0]
+
+ export = readFilenumFromZip(t, zipBytes, 0)
+
+ // post created
+ assertContainsAllMsgs(export, []msgDetailsToCheck{
+ {
+ createAt: posts[2].CreateAt,
+ updateAt: posts[2].UpdateAt,
+ postId: posts[2].Id,
+ message: posts[2].Message,
+ },
+ // post deleted ONLY (not its created post, because that was in the previous job)
+ {
+ createAt: posts[0].CreateAt,
+ updateAt: message0DeleteAt,
+ updatedType: shared.Deleted,
+ postId: posts[0].Id,
+ message: "delete " + posts[0].Message,
+ },
+ // post updated ONLY (not its created post, because that was in the previous job)
+ {
+ createAt: posts[1].CreateAt,
+ updateAt: updatedPost1.UpdateAt,
+ updatedType: shared.UpdatedNoMsgChange,
+ postId: posts[1].Id,
+ message: posts[1].Message,
+ },
+ }, "Job 4")
+ })
+
+ t.Run("csv -- multiple batches, 1 zip per batch, output to a single directory", func(t *testing.T) {
+ th := setup(t)
+ defer th.TearDown()
+ defer func() {
+ err := os.RemoveAll(exportDir)
+ assert.NoError(t, err)
+ err = os.RemoveAll(attachmentDir)
+ assert.NoError(t, err)
+ }()
+
+ time.Sleep(1 * time.Millisecond)
+ now := model.GetMillis()
+
+ jobStart := now - 1
+ th.App.UpdateConfig(func(cfg *model.Config) {
+ *cfg.MessageExportSettings.EnableExport = true
+ *cfg.MessageExportSettings.ExportFromTimestamp = jobStart
+ *cfg.MessageExportSettings.BatchSize = 5
+ *cfg.MessageExportSettings.ExportFormat = model.ComplianceExportTypeCsv
+ *cfg.FileSettings.DriverName = model.ImageDriverLocal
+ *cfg.FileSettings.Directory = attachmentDir
+
+ if exportDir != attachmentDir {
+ *cfg.FileSettings.DedicatedExportStore = true
+ *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
+ *cfg.FileSettings.ExportDirectory = exportDir
+ }
+ })
+
+ attachmentContent := "Hello there"
+ attachmentPath001 := "path/to/attachments/one.txt"
+ _, _ = attachmentBackend.WriteFile(bytes.NewBufferString(attachmentContent), attachmentPath001)
+ post, err := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: th.BasicChannel.Id,
+ UserId: st.NewTestID(),
+ Message: st.NewTestID(),
+ CreateAt: now,
+ UpdateAt: now,
+ FileIds: []string{"test1"},
+ })
+ require.NoError(t, err)
+
+ attachment, err := th.App.Srv().Store().FileInfo().Save(th.Context, &model.FileInfo{
+ Id: st.NewTestID(),
+ CreatorId: post.UserId,
+ PostId: post.Id,
+ CreateAt: now,
+ UpdateAt: now,
+ Path: attachmentPath001,
+ })
+ require.NoError(t, err)
+
+ for i := 0; i < 10; i++ {
+ _, e := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: th.BasicChannel.Id,
+ UserId: st.NewTestID(),
+ Message: st.NewTestID(),
+ CreateAt: now + int64(i),
+ UpdateAt: now + int64(i),
+ })
+ require.NoError(t, e)
+ }
+
+ job := runJobForTest(t, th, nil)
+
+ warnings, err := strconv.Atoi(job.Data[shared.JobDataWarningCount])
+ require.NoError(t, err)
+ require.Equal(t, 0, warnings)
+
+ numExported, err := strconv.ParseInt(job.Data[shared.JobDataMessagesExported], 0, 64)
+ require.NoError(t, err)
+ require.Equal(t, int64(11), numExported)
+ jobEnd, err := strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 0, 64)
+ require.NoError(t, err)
+
+ jobExportDir := job.Data[shared.JobDataExportDir]
+ batch001 := shared.GetBatchPath(jobExportDir, jobStart, now+3, 1)
+ batch002 := shared.GetBatchPath(jobExportDir, now+3, now+8, 2)
+ batch003 := shared.GetBatchPath(jobExportDir, now+8, jobEnd, 3)
+ files, err := exportBackend.ListDirectory(jobExportDir)
+ require.NoError(t, err)
+ require.ElementsMatch(t, []string{batch001, batch002, batch003}, files)
+
+ zipBytes, err := exportBackend.ReadFile(batch001)
+ require.NoError(t, err)
+
+ zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+ require.NoError(t, err)
+
+ csvZipFilePath := path.Join("files", post.Id, fmt.Sprintf("%s-%s", attachment.Id, path.Base(attachment.Path)))
+
+ attachmentInZip, err := zipReader.Open(csvZipFilePath)
+ require.NoError(t, err)
+ attachmentInZipContents, err := io.ReadAll(attachmentInZip)
+ require.NoError(t, err)
+ err = attachmentInZip.Close()
+ require.NoError(t, err)
+
+ require.EqualValuesf(t, attachmentContent, string(attachmentInZipContents), "file contents not equal")
+ })
+}
+
+func openZipAndReadFile(t *testing.T, backend filestore.FileBackend, path string, filename string) string {
+ zipBytes, err := backend.ReadFile(path)
+ require.NoError(t, err)
+ return readFileFromZip(t, zipBytes, filename)
+}
+
+func readFileFromZip(t *testing.T, zipBytes []byte, filename string) string {
+ zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+ require.NoError(t, err)
+
+ file, err := zipReader.Open(filename)
+ require.NoError(t, err)
+ contents, err := io.ReadAll(file)
+ require.NoError(t, err)
+ err = file.Close()
+ require.NoError(t, err)
+
+ return string(contents)
+}
+
+func openZipAndReadFileNum(t *testing.T, backend filestore.FileBackend, path string, fileNum int) string {
+ zipBytes, err := backend.ReadFile(path)
+ require.NoError(t, err)
+ return readFilenumFromZip(t, zipBytes, fileNum)
+}
+
+func openZipAndReadFileStartingWith(t *testing.T, backend filestore.FileBackend, path string, startsWith string) string {
+ zipBytes, err := backend.ReadFile(path)
+ require.NoError(t, err)
+
+ zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+ require.NoError(t, err)
+
+ var names []string
+ for _, f := range zipReader.File {
+ if strings.HasPrefix(f.Name, startsWith) {
+ file, err := f.Open()
+ require.NoError(t, err)
+ contents, err := io.ReadAll(file)
+ require.NoError(t, err)
+ err = file.Close()
+ require.NoError(t, err)
+
+ return string(contents)
+ }
+ names = append(names, f.Name)
+ }
+
+ require.True(t, false, "called openZipAndReadFileStartingWith but didn't file file starting with: %s. Found: %v", startsWith, names)
+ return ""
+}
+
+func readFilenumFromZip(t *testing.T, zipBytes []byte, fileNum int) string {
+ zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+ require.NoError(t, err)
+
+ file, err := zipReader.File[fileNum].Open()
+ require.NoError(t, err)
+ contents, err := io.ReadAll(file)
+ require.NoError(t, err)
+ err = file.Close()
+ require.NoError(t, err)
+
+ return string(contents)
+}
+
+func dataContainsOneOfExpected(data string, expected []string) bool {
+ for _, perm := range expected {
+ if strings.Contains(data, perm) {
+ return true
+ }
+ }
+ return false
}
diff --git a/server/enterprise/message_export/message_export_test_e2e_generators.go b/server/enterprise/message_export/message_export_test_e2e_generators.go
new file mode 100644
index 0000000000..19703006a9
--- /dev/null
+++ b/server/enterprise/message_export/message_export_test_e2e_generators.go
@@ -0,0 +1,1551 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package message_export
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "os"
+ "strconv"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/mattermost/mattermost/server/public/model"
+ "github.com/mattermost/mattermost/server/v8/channels/api4"
+ "github.com/mattermost/mattermost/server/v8/channels/jobs"
+ st "github.com/mattermost/mattermost/server/v8/channels/store/storetest"
+ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
+ "github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+)
+
+type MessageExport struct {
+ TeamId string
+ TeamName string
+ TeamDisplayName string
+
+ ChannelId string
+ ChannelName string
+ ChannelDisplayName string
+ ChannelType model.ChannelType
+
+ UserId string
+ UserEmail string
+ Username string
+ IsBot bool
+
+ PostId string
+ PostCreateAt int64
+ PostUpdateAt int64
+ PostDeleteAt int64
+ PostEditAt int64
+ PostMessage string
+ PostType string
+ PostRootId string
+ PostProps string
+ PostOriginalId string
+ PostFileIds []string
+}
+
+func removePointers(p *model.MessageExport) MessageExport {
+ return MessageExport{
+ TeamId: model.SafeDereference(p.TeamId),
+ TeamName: model.SafeDereference(p.TeamName),
+ TeamDisplayName: model.SafeDereference(p.TeamDisplayName),
+ ChannelId: model.SafeDereference(p.ChannelId),
+ ChannelName: model.SafeDereference(p.ChannelName),
+ ChannelDisplayName: model.SafeDereference(p.ChannelDisplayName),
+ ChannelType: model.SafeDereference(p.ChannelType),
+ UserId: model.SafeDereference(p.UserId),
+ UserEmail: model.SafeDereference(p.UserEmail),
+ Username: model.SafeDereference(p.Username),
+ IsBot: p.IsBot,
+ PostId: model.SafeDereference(p.PostId),
+ PostCreateAt: model.SafeDereference(p.PostCreateAt),
+ PostUpdateAt: model.SafeDereference(p.PostUpdateAt),
+ PostDeleteAt: model.SafeDereference(p.PostDeleteAt),
+ PostEditAt: model.SafeDereference(p.PostEditAt),
+ PostMessage: model.SafeDereference(p.PostMessage),
+ PostType: model.SafeDereference(p.PostType),
+ PostRootId: model.SafeDereference(p.PostRootId),
+ PostProps: model.SafeDereference(p.PostProps),
+ PostOriginalId: model.SafeDereference(p.PostOriginalId),
+ PostFileIds: p.PostFileIds,
+ }
+}
+
+// assertNumPostsToExport checks both the MessageExport and the AnalyticsPostCount -- they were sometimes giving
+// different numbers and this helps debug.
+func assertNumPostsToExport(t *testing.T, th *api4.TestHelper, num int, since, until int64) {
+ exports, _, err := th.App.Srv().Store().Compliance().MessageExport(th.Context, model.MessageExportCursor{
+ LastPostUpdateAt: since, UntilUpdateAt: until,
+ }, 100)
+ assert.NoError(t, err)
+ assert.Len(t, exports, num)
+ assert.Lenf(t, exports, num, "MessageExport posts found, since %d, th.BasicChannel.Id: %s\n", since, th.BasicChannel.Id)
+ if len(exports) != num {
+ for _, p := range exports {
+ t.Logf("%#+v\n", removePointers(p))
+ }
+ }
+
+ count, err := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{ExcludeSystemPosts: true, UsersPostsOnly: true, SincePostID: "", SinceUpdateAt: since})
+ require.NoError(t, err)
+ assert.Equal(t, num, int(count))
+}
+
+func assertNumExported(t *testing.T, expectedNum int, data map[string]string) {
+ numExported, err := strconv.Atoi(data[shared.JobDataMessagesExported])
+ require.NoError(t, err)
+ assert.Equalf(t, numExported, expectedNum, "\njobData: %v\n", data)
+ require.Equal(t, expectedNum, numExported)
+}
+
+func getMostRecentJobWithId(t *testing.T, th *api4.TestHelper, id string) *model.Job {
+ list, _, err := th.SystemAdminClient.GetJobsByType(context.Background(), "message_export", 0, 1)
+ require.NoError(t, err)
+ require.Len(t, list, 1)
+ require.Equal(t, id, list[0].Id)
+ return list[0]
+}
+
+func checkJobForStatus(t *testing.T, th *api4.TestHelper, id string, status string) {
+ doneChan := make(chan bool)
+ var job *model.Job
+ go func() {
+ defer close(doneChan)
+ for {
+ job = getMostRecentJobWithId(t, th, id)
+ if job.Status == status {
+ break
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+ require.Equal(t, status, job.Status)
+ }()
+ select {
+ case <-doneChan:
+ case <-time.After(15 * time.Second):
+ require.True(t, false, "expected job's status to be %s, got %s", status, job.Status)
+ }
+}
+
+func runJobForTest(t *testing.T, th *api4.TestHelper, jobData map[string]string) *model.Job {
+ job, _, err := th.SystemAdminClient.CreateJob(context.Background(),
+ &model.Job{Type: "message_export", Data: jobData})
+ require.NoError(t, err)
+ // poll until completion
+ checkJobForStatus(t, th, job.Id, "success")
+ job = getMostRecentJobWithId(t, th, job.Id)
+ return job
+}
+
+func setup(t *testing.T) *api4.TestHelper {
+ jobs.DefaultWatcherPollingInterval = 100
+ th := api4.SetupEnterprise(t).InitBasic()
+ th.App.Srv().SetLicense(model.NewTestLicense("message_export"))
+ messageExportImpl := MessageExportJobInterfaceImpl{th.App.Srv()}
+ th.App.Srv().Jobs.RegisterJobType(model.JobTypeMessageExport, messageExportImpl.MakeWorker(), messageExportImpl.MakeScheduler())
+
+ err := th.App.Srv().Jobs.StartWorkers()
+ require.NoError(t, err)
+
+ err = th.App.Srv().Jobs.StartSchedulers()
+ require.NoError(t, err)
+
+ return th
+}
+
+// jobDataInvariantsShouldBeEqual tests that the parts of the job.Data that shouldn't change, don't change.
+func jobDataInvariantsShouldBeEqual(t *testing.T, expected map[string]string, received map[string]string) {
+ assert.Equal(t, expected[shared.JobDataExportType], received[shared.JobDataExportType])
+ assert.Equal(t, expected[shared.JobDataBatchSize], received[shared.JobDataBatchSize])
+ assert.Equal(t, expected[shared.JobDataChannelBatchSize], received[shared.JobDataChannelBatchSize])
+ assert.Equal(t, expected[shared.JobDataChannelHistoryBatchSize], received[shared.JobDataChannelHistoryBatchSize])
+ assert.Equal(t, expected[shared.JobDataExportDir], received[shared.JobDataExportDir])
+ assert.Equal(t, expected[shared.JobDataJobEndTime], received[shared.JobDataJobEndTime])
+ assert.Equal(t, expected[shared.JobDataJobStartTime], received[shared.JobDataJobStartTime])
+}
+
+type joinLeave struct {
+ join int64
+ leave int64
+}
+type batchStartEndTimes struct {
+ start int64
+ end int64
+}
+
+type JobResults struct {
+ start int64
+ joinLeaves []joinLeave
+ users []*model.User
+ createUpdateTimes []int64
+ attachments []*model.FileInfo
+ contents []string
+ jobEndTime int64
+ batches []string
+ posts []*model.Post
+ channels []*model.Channel
+ teams []*model.Team
+ batchTimes []batchStartEndTimes
+ jobExportDir string
+}
+
+func generateActianceBatchTest1(t *testing.T, th *api4.TestHelper, attachmentDir, exportDir string,
+ attachmentBackend filestore.FileBackend) JobResults {
+ now := model.GetMillis()
+ jobStart := now - 1
+
+ th.App.UpdateConfig(func(cfg *model.Config) {
+ *cfg.MessageExportSettings.EnableExport = true
+ *cfg.MessageExportSettings.ExportFromTimestamp = jobStart
+ *cfg.MessageExportSettings.BatchSize = 5
+ *cfg.MessageExportSettings.ExportFormat = model.ComplianceExportTypeActiance
+ *cfg.FileSettings.DriverName = model.ImageDriverLocal
+ *cfg.FileSettings.Directory = attachmentDir
+
+ if exportDir != attachmentDir {
+ *cfg.FileSettings.DedicatedExportStore = true
+ *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
+ *cfg.FileSettings.ExportDirectory = exportDir
+ }
+ })
+
+ contents := []string{"Hello there"}
+ attachmentPath001 := "path/to/attachments/one.txt"
+ _, _ = attachmentBackend.WriteFile(bytes.NewBufferString(contents[0]), attachmentPath001)
+ post, err := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: th.BasicChannel.Id,
+ UserId: st.NewTestID(),
+ Message: st.NewTestID(),
+ CreateAt: now,
+ UpdateAt: now,
+ FileIds: []string{"test1"},
+ })
+ require.NoError(t, err)
+
+ attachment, err := th.App.Srv().Store().FileInfo().Save(th.Context, &model.FileInfo{
+ Id: st.NewTestID(),
+ CreatorId: post.UserId,
+ PostId: post.Id,
+ CreateAt: now,
+ UpdateAt: now,
+ Path: attachmentPath001,
+ })
+ require.NoError(t, err)
+ attachments := []*model.FileInfo{attachment}
+
+ for i := 0; i < 10; i++ {
+ _, e := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: th.BasicChannel.Id,
+ UserId: st.NewTestID(),
+ Message: st.NewTestID(),
+ CreateAt: now + int64(i),
+ UpdateAt: now + int64(i),
+ })
+ require.NoError(t, e)
+ }
+
+ until := model.GetMillis()
+ assertNumPostsToExport(t, th, 11, jobStart, until)
+
+ job := runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
+
+ warnings, err := strconv.Atoi(job.Data[shared.JobDataWarningCount])
+ require.NoError(t, err)
+ require.Equal(t, 0, warnings)
+
+ numExported, err := strconv.ParseInt(job.Data[shared.JobDataMessagesExported], 0, 64)
+ require.NoError(t, err)
+ require.Equal(t, int64(11), numExported)
+
+ jobEnd, err := strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 0, 64)
+ require.NoError(t, err)
+ jobExportDir := job.Data[shared.JobDataExportDir]
+ batch001 := shared.GetBatchPath(jobExportDir, jobStart, now+3, 1)
+ batch002 := shared.GetBatchPath(jobExportDir, now+3, now+8, 2)
+ batch003 := shared.GetBatchPath(jobExportDir, now+8, jobEnd, 3)
+ batches := []string{batch001, batch002, batch003}
+
+ return JobResults{
+ attachments: attachments,
+ contents: contents,
+ batches: batches,
+ jobExportDir: jobExportDir,
+ }
+}
+
+func generateActianceBatchTest2(t *testing.T, th *api4.TestHelper, attachmentDir, exportDir string) JobResults {
+ now := model.GetMillis()
+
+ th.App.UpdateConfig(func(cfg *model.Config) {
+ *cfg.MessageExportSettings.EnableExport = true
+ *cfg.MessageExportSettings.BatchSize = 3
+ *cfg.MessageExportSettings.ExportFormat = model.ComplianceExportTypeActiance
+ *cfg.FileSettings.DriverName = model.ImageDriverLocal
+ *cfg.FileSettings.Directory = attachmentDir
+
+ if exportDir != attachmentDir {
+ *cfg.FileSettings.DedicatedExportStore = true
+ *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
+ *cfg.FileSettings.ExportDirectory = exportDir
+ }
+ })
+
+ for i := 0; i < 10; i++ {
+ _, e := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: th.BasicChannel.Id,
+ UserId: st.NewTestID(),
+ Message: st.NewTestID(),
+ CreateAt: now + int64(i),
+ UpdateAt: now + int64(i),
+ })
+ require.NoError(t, e)
+ }
+
+ // start at the 2nd post and get till the 7th post (inclusive) = 6 posts
+ job := runJobForTest(t, th, map[string]string{
+ shared.JobDataBatchStartTime: strconv.Itoa(int(now) + 1),
+ shared.JobDataJobEndTime: strconv.Itoa(int(now) + 6),
+ })
+ numExported, err := strconv.ParseInt(job.Data[shared.JobDataMessagesExported], 0, 64)
+ require.NoError(t, err)
+ numExpected, err := strconv.ParseInt(job.Data[shared.JobDataTotalPostsExpected], 0, 64)
+ require.NoError(t, err)
+ // test that we only exported 6 (because the JobDataJobEndTime was translated to the cursor's UntilUpdateAt)
+ require.Equal(t, 6, int(numExported))
+ // test that we were reporting that correctly in the UI
+ require.Equal(t, 6, int(numExpected))
+
+ jobEnd, err := strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 0, 64)
+ require.NoError(t, err)
+ require.Equal(t, now+6, jobEnd)
+ jobExportDir := job.Data[shared.JobDataExportDir]
+ batch001 := shared.GetBatchPath(jobExportDir, now+1, now+3, 1)
+ // lastPostUpdateAt will be post#4 (now+3), even though we exported it above, because LastPostId will exclude it
+ batch002 := shared.GetBatchPath(jobExportDir, now+3, now+6, 2)
+ batches := []string{batch001, batch002}
+
+ return JobResults{
+ batches: batches,
+ jobExportDir: jobExportDir,
+ }
+}
+
+func generateE2ETestType1Results(t *testing.T, th *api4.TestHelper, exportType, attachmentDir, exportDir string,
+ attachmentBackend, exportBackend filestore.FileBackend, testStopping bool) JobResults {
+ // This tests (reading the files exported and testing the actual exported data):
+ // - job system exports the complete time from beginning to end; i.e., it doesn't use the post updateAt values as the bounds, it uses the start time and end time of the job.
+ // - job system uses previous job's end time as the start for the next batch
+ // - user joins and leaves before the first post in the first batch (but after the job start time)
+ // - user joins and leaves before the first post in the second batch
+ // - user joins and leaves before the first post in the last batch
+ // - user joins and leaves after the last post in the last batch (but before the job end time)
+ // - channel with no posts but user activity (one user joins and leaves after start of batch period but before first post)
+ // - channel with no posts but user activity (one user leaves after last post but before end of batch period)
+ // - worked, but making sure we test for it specifically in e2e:
+ // - exports with multiple channels (this wasn't tested before)
+ // - user joins before job start time and stays (should record user's original join time, not the start of the job)
+ // - attachments are recorded with correct names in the xml and content in the files
+ // - a post from a user who wasn't a member in the channel creates a record for the user entering the channel at the start of the batch and leaving at the end of the batch (to be discussed with end user, this seems wrong)
+
+ // Also tests the `BatchSize+1` logic in the worker, because we have 9 posts and batch size of 3.
+
+ start := model.GetMillis()
+
+ th.App.UpdateConfig(func(cfg *model.Config) {
+ *cfg.MessageExportSettings.EnableExport = true
+ *cfg.MessageExportSettings.ExportFromTimestamp = 0
+ *cfg.MessageExportSettings.BatchSize = 3
+ *cfg.MessageExportSettings.ExportFormat = exportType
+ *cfg.FileSettings.DriverName = model.ImageDriverLocal
+ *cfg.FileSettings.Directory = attachmentDir
+
+ if exportDir != attachmentDir {
+ *cfg.FileSettings.DedicatedExportStore = true
+ *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
+ *cfg.FileSettings.ExportDirectory = exportDir
+ }
+ })
+
+ // Users:
+ users := make([]*model.User, 0)
+ user, err := th.App.Srv().Store().User().Save(th.Context, &model.User{
+ Username: "user1",
+ Email: "user1@email",
+ })
+ require.NoError(t, err)
+ users = append(users, user)
+ user, err = th.App.Srv().Store().User().Save(th.Context, &model.User{
+ Username: "user2",
+ Email: "user2@email",
+ })
+ require.NoError(t, err)
+ users = append(users, user)
+ user, err = th.App.Srv().Store().User().Save(th.Context, &model.User{
+ Username: "user3",
+ Email: "user3@email",
+ })
+ require.NoError(t, err)
+ users = append(users, user)
+ user, err = th.App.Srv().Store().User().Save(th.Context, &model.User{
+ Username: "user4",
+ Email: "user4@email",
+ })
+ require.NoError(t, err)
+ users = append(users, user)
+ user, err = th.App.Srv().Store().User().Save(th.Context, &model.User{
+ Username: "user5",
+ Email: "user5@email",
+ })
+ require.NoError(t, err)
+ users = append(users, user)
+ user, err = th.App.Srv().Store().User().Save(th.Context, &model.User{
+ Username: "user6",
+ Email: "user6@email",
+ })
+ require.NoError(t, err)
+ users = append(users, user)
+ user, err = th.App.Srv().Store().User().Save(th.Context, &model.User{
+ Username: "user7",
+ Email: "user7@email",
+ })
+ require.NoError(t, err)
+ users = append(users, user)
+ user, err = th.App.Srv().Store().User().Save(th.Context, &model.User{
+ Username: "user8",
+ Email: "user8@email",
+ })
+ require.NoError(t, err)
+ users = append(users, user)
+
+ channel2, err := th.App.Srv().Store().Channel().Save(th.Context, &model.Channel{
+ DisplayName: "the Channel Two",
+ Name: "channel_two_name",
+ Type: model.ChannelTypePrivate,
+ TeamId: th.BasicTeam.Id,
+ CreatorId: th.BasicUser.Id,
+ }, 999)
+ require.NoError(t, err)
+
+ // channel3 will have only one user leaving during export time
+ channel3, err := th.App.Srv().Store().Channel().Save(th.Context, &model.Channel{
+ DisplayName: "the Channel Three",
+ Name: "channel_three_name",
+ Type: model.ChannelTypeOpen,
+ TeamId: th.BasicTeam.Id,
+ CreatorId: th.BasicUser.Id,
+ }, 999)
+ require.NoError(t, err)
+
+ // channel4 will have only one user joining during export time
+ channel4, err := th.App.Srv().Store().Channel().Save(th.Context, &model.Channel{
+ DisplayName: "the Channel Four",
+ Name: "channel_four_name",
+ Type: model.ChannelTypeOpen,
+ TeamId: th.BasicTeam.Id,
+ CreatorId: th.BasicUser.Id,
+ }, 999)
+ require.NoError(t, err)
+
+ // Save 9 posts so that we have the following batches:
+ createUpdateTimes := []int64{
+ start + 10, start + 14, start + 20, // batch 1: start-20, posts start at 10
+ start + 23, start + 25, start + 30, // batch 2: 20-30, posts start at 23
+ start + 36, start + 37, start + 40, // batch 3: 30-40, posts start at 36
+ }
+
+ jl := []joinLeave{
+ {start - 5, 0}, // user 1 never leaves
+ {start + 7, start + 8}, // user 2 joins and leaves before first post (but after startTime)
+ {start + 11, start + 15}, // user 3 joins and leaves during first batch
+ {start + 21, start + 22}, // user 4 joins and leaves during second batch but before second batch's post
+ {start + 32, start + 35}, // user 5 joins and leaves during third batch but before third batch's post
+ {start + 55, start + 57}, // user 6 joins and leaves after the last batch's last post but before when export is run
+ {start - 100, start + 5}, // user 7 joins channel3 before start time and leaves before batch 1
+ {start + 59, 0}, // user 8 joins channel4 after batch 3 (but before end)
+ }
+
+ // user 1 joins before start time and stays (and posts)
+ err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[0].Id, channel2.Id, jl[0].join)
+ require.NoError(t, err)
+ // user 2 joins and leaves before first post (but after startTime)
+ err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[1].Id, channel2.Id, jl[1].join)
+ require.NoError(t, err)
+ err = th.App.Srv().Store().ChannelMemberHistory().LogLeaveEvent(users[1].Id, channel2.Id, jl[1].leave)
+ require.NoError(t, err)
+ // user 3 joins and leaves during first batch
+ err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[2].Id, channel2.Id, jl[2].join)
+ require.NoError(t, err)
+ err = th.App.Srv().Store().ChannelMemberHistory().LogLeaveEvent(users[2].Id, channel2.Id, jl[2].leave)
+ require.NoError(t, err)
+ // user 4 joins and leaves during second batch but before second batch's post
+ err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[3].Id, channel2.Id, jl[3].join)
+ require.NoError(t, err)
+ err = th.App.Srv().Store().ChannelMemberHistory().LogLeaveEvent(users[3].Id, channel2.Id, jl[3].leave)
+ // user 5 joins and leaves during third batch but before third batch's post
+ require.NoError(t, err)
+ err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[4].Id, channel2.Id, jl[4].join)
+ require.NoError(t, err)
+ err = th.App.Srv().Store().ChannelMemberHistory().LogLeaveEvent(users[4].Id, channel2.Id, jl[4].leave)
+ require.NoError(t, err)
+ // user 6 joins and leaves after the last batch's last post
+ err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[5].Id, channel2.Id, jl[5].join)
+ require.NoError(t, err)
+ err = th.App.Srv().Store().ChannelMemberHistory().LogLeaveEvent(users[5].Id, channel2.Id, jl[5].leave)
+ require.NoError(t, err)
+
+ // user 7 joins channel3 before start time and leaves before batch 1
+ err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[6].Id, channel3.Id, start-100)
+ require.NoError(t, err)
+ err = th.App.Srv().Store().ChannelMemberHistory().LogLeaveEvent(users[6].Id, channel3.Id, start+5)
+ require.NoError(t, err)
+ // user 8 joins channel4 after batch 3 (but before end)
+ err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[7].Id, channel4.Id, start+59)
+ require.NoError(t, err)
+
+ assertNumPostsToExport(t, th, 0, start, model.GetMillis())
+
+ var attachments []*model.FileInfo
+ var contents []string
+ var posts []*model.Post
+ for i, updateAt := range createUpdateTimes {
+ post, err2 := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: users[0].Id,
+ Message: fmt.Sprintf("message %d", i),
+ CreateAt: updateAt,
+ UpdateAt: updateAt,
+ FileIds: []string{fmt.Sprintf("test%d", i)},
+ })
+ require.NoError(t, err2)
+ posts = append(posts, post)
+ time.Sleep(time.Millisecond)
+
+ attachmentContent := fmt.Sprintf("Hello there %d", i)
+ attachmentPath := fmt.Sprintf("path/to/attachments/file_%d.txt", i)
+ _, err = attachmentBackend.WriteFile(bytes.NewBufferString(attachmentContent), attachmentPath)
+ require.NoError(t, err)
+
+ info, err2 := th.App.Srv().Store().FileInfo().Save(th.Context, &model.FileInfo{
+ Id: st.NewTestID(),
+ CreatorId: post.UserId,
+ PostId: post.Id,
+ CreateAt: updateAt,
+ UpdateAt: updateAt,
+ Path: attachmentPath,
+ })
+ require.NoError(t, err2)
+ attachments = append(attachments, info)
+ contents = append(contents, attachmentContent)
+ }
+
+ // Test that it's picking up a previous successful job
+ var previousJob *model.Job
+ previousJob, err = th.App.Srv().Store().Job().GetNewestJobByStatusesAndType([]string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport)
+ require.Error(t, err)
+ require.Nil(t, previousJob)
+
+ _, err = th.App.Srv().Store().Job().Save(&model.Job{
+ Id: "blah",
+ Type: model.JobTypeMessageExport,
+ Priority: 0,
+ CreateAt: 0,
+ StartAt: 0,
+ LastActivityAt: 0,
+ Status: model.JobStatusSuccess,
+ Progress: 100,
+ Data: map[string]string{shared.JobDataBatchStartTime: strconv.Itoa(int(start))},
+ })
+ require.NoError(t, err)
+
+ previousJob, err = th.App.Srv().Store().Job().GetNewestJobByStatusesAndType([]string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport)
+ require.NoError(t, err)
+ require.NotNilf(t, previousJob, "prevJob")
+
+ var prevUpdatedAt int64
+ if timestamp, prevExists := previousJob.Data[shared.JobDataBatchStartTime]; prevExists {
+ prevUpdatedAt, err = strconv.ParseInt(timestamp, 10, 64)
+ require.NoError(t, err)
+ }
+ require.Equal(t, prevUpdatedAt, start)
+
+ // move past the last post time
+ time.Sleep(100 * time.Millisecond)
+
+ // check number of messages to be exported
+ until := model.GetMillis()
+ assertNumPostsToExport(t, th, 9, start, until)
+
+ // Now run the exports
+ var job *model.Job
+ if testStopping {
+ var jobData map[string]string
+ // manually create the job (which will start right away, so we need to wait for it below, after we use its id.
+ job, _, err = th.SystemAdminClient.CreateJob(context.Background(), &model.Job{
+ Type: "message_export",
+ Data: map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)}})
+ require.NoError(t, err)
+
+ // Stop the export after the second batch by stopping the Worker.
+ batchCount := 0
+ testEndOfBatchCb = func(worker *MessageExportWorker) {
+ batchCount++
+ if batchCount == 2 {
+ job = getMostRecentJobWithId(t, th, job.Id)
+ jobData = job.Data
+
+ // let the job continue, but stop Worker, check we went back to Pending, then start the Worker.
+ go func() {
+ worker.Stop()
+ checkJobForStatus(t, th, job.Id, model.JobStatusPending)
+ worker.Run()
+ checkJobForStatus(t, th, job.Id, model.JobStatusInProgress)
+ }()
+ }
+ }
+
+ // Wait for the rest of the exports to finish
+ checkJobForStatus(t, th, job.Id, model.JobStatusSuccess)
+ job = getMostRecentJobWithId(t, th, job.Id)
+ testEndOfBatchCb = nil
+ jobDataInvariantsShouldBeEqual(t, jobData, job.Data)
+ } else {
+ job = runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
+ }
+
+ warnings, err := strconv.Atoi(job.Data[shared.JobDataWarningCount])
+ require.NoError(t, err)
+ require.Equal(t, 0, warnings)
+
+ assertNumExported(t, 9, job.Data)
+
+ jobExportDir := job.Data[shared.JobDataExportDir]
+ jobEndTime, err := strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 10, 64)
+ require.NoError(t, err)
+
+ batchTimes := []batchStartEndTimes{
+ {start: prevUpdatedAt, end: createUpdateTimes[2]},
+ {start: createUpdateTimes[2], end: createUpdateTimes[5]},
+ {start: createUpdateTimes[5], end: jobEndTime},
+ }
+
+ batch001 := shared.GetBatchPath(jobExportDir, batchTimes[0].start, batchTimes[0].end, 1)
+ batch002 := shared.GetBatchPath(jobExportDir, batchTimes[1].start, batchTimes[1].end, 2)
+ batch003 := shared.GetBatchPath(jobExportDir, batchTimes[2].start, batchTimes[2].end, 3)
+ files, err := exportBackend.ListDirectory(jobExportDir)
+ require.NoError(t, err)
+ batches := []string{batch001, batch002, batch003}
+ require.ElementsMatch(t, batches, files)
+
+ return JobResults{
+ start: start,
+ joinLeaves: jl,
+ users: users,
+ createUpdateTimes: createUpdateTimes,
+ attachments: attachments,
+ contents: contents,
+ jobEndTime: jobEndTime,
+ batches: batches,
+ posts: posts,
+ channels: []*model.Channel{channel2, channel3, channel4},
+ teams: []*model.Team{th.BasicTeam},
+ batchTimes: batchTimes,
+ }
+}
+
+func generateE2ETestType2Results(t *testing.T, th *api4.TestHelper, exportType, attachmentDir, exportDir string,
+ attachmentBackend, exportBackend filestore.FileBackend) JobResults {
+ th.App.UpdateConfig(func(cfg *model.Config) {
+ *cfg.MessageExportSettings.EnableExport = true
+ *cfg.MessageExportSettings.ExportFromTimestamp = 0
+ *cfg.MessageExportSettings.BatchSize = 3
+ *cfg.MessageExportSettings.ExportFormat = exportType
+ *cfg.FileSettings.DriverName = model.ImageDriverLocal
+ *cfg.FileSettings.Directory = attachmentDir
+
+ if exportDir != attachmentDir {
+ *cfg.FileSettings.DedicatedExportStore = true
+ *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
+ *cfg.FileSettings.ExportDirectory = exportDir
+ }
+ })
+
+ start := model.GetMillis()
+
+ // Users:
+ users := make([]*model.User, 0)
+ user, err := th.App.Srv().Store().User().Save(th.Context, &model.User{
+ Username: "user1",
+ Email: "user1@email",
+ })
+ require.NoError(t, err)
+ users = append(users, user)
+ user, err = th.App.Srv().Store().User().Save(th.Context, &model.User{
+ Username: "user2",
+ Email: "user2@email",
+ })
+ require.NoError(t, err)
+ users = append(users, user)
+
+ channel2, err := th.App.Srv().Store().Channel().Save(th.Context, &model.Channel{
+ DisplayName: "the Channel Two",
+ Name: "channel_two_name",
+ Type: model.ChannelTypePrivate,
+ TeamId: th.BasicTeam.Id,
+ CreatorId: th.BasicUser.Id,
+ }, 999)
+ require.NoError(t, err)
+
+ createUpdateTimes := []int64{start + 10, start + 14}
+
+ jl := []joinLeave{
+ {start - 5, 0}, // user 1 never leaves
+ }
+
+ // user 1 joins before start time and stays (and posts)
+ err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[0].Id, channel2.Id, jl[0].join)
+ require.NoError(t, err)
+
+ assertNumPostsToExport(t, th, 0, start, model.GetMillis())
+
+ var posts []*model.Post
+
+ // first post from user 1 (member)
+ post, err := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: users[0].Id,
+ Message: "message 1",
+ CreateAt: createUpdateTimes[0],
+ UpdateAt: createUpdateTimes[0],
+ })
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+
+ post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: users[1].Id,
+ Message: "message 2",
+ CreateAt: createUpdateTimes[1],
+ UpdateAt: createUpdateTimes[1],
+ })
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+
+ // Test that it's picking up a previous successful job
+ var previousJob *model.Job
+ previousJob, err = th.App.Srv().Store().Job().GetNewestJobByStatusesAndType([]string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport)
+ require.Error(t, err)
+ require.Nil(t, previousJob)
+
+ _, err = th.App.Srv().Store().Job().Save(&model.Job{
+ Id: "blah",
+ Type: model.JobTypeMessageExport,
+ Priority: 0,
+ CreateAt: 0,
+ StartAt: 0,
+ LastActivityAt: 0,
+ Status: model.JobStatusSuccess,
+ Progress: 100,
+ Data: map[string]string{shared.JobDataBatchStartTime: strconv.Itoa(int(start))},
+ })
+ require.NoError(t, err)
+
+ previousJob, err = th.App.Srv().Store().Job().GetNewestJobByStatusesAndType([]string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport)
+ require.NoError(t, err)
+ require.NotNilf(t, previousJob, "prevJob")
+
+ var prevUpdatedAt int64
+ if timestamp, prevExists := previousJob.Data[shared.JobDataBatchStartTime]; prevExists {
+ prevUpdatedAt, err = strconv.ParseInt(timestamp, 10, 64)
+ require.NoError(t, err)
+ }
+ require.Equal(t, prevUpdatedAt, start)
+
+ // move past the last post time
+ time.Sleep(30 * time.Millisecond)
+
+ // check number of messages to be exported
+ until := model.GetMillis()
+ assertNumPostsToExport(t, th, 2, start, until)
+
+ // Now run the exports
+ job := runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
+
+ warnings, err := strconv.Atoi(job.Data[shared.JobDataWarningCount])
+ require.NoError(t, err)
+ require.Equal(t, 0, warnings)
+
+ assertNumExported(t, 2, job.Data)
+
+ jobExportDir := job.Data[shared.JobDataExportDir]
+ jobEndTime, err := strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 10, 64)
+ require.NoError(t, err)
+
+ batch001 := shared.GetBatchPath(jobExportDir, prevUpdatedAt, jobEndTime, 1)
+ files, err := exportBackend.ListDirectory(jobExportDir)
+ require.NoError(t, err)
+ batches := []string{batch001}
+ require.ElementsMatch(t, batches, files)
+
+ return JobResults{
+ start: start,
+ joinLeaves: jl,
+ users: users,
+ createUpdateTimes: createUpdateTimes,
+ jobEndTime: jobEndTime,
+ batches: batches,
+ posts: posts,
+ channels: []*model.Channel{channel2},
+ teams: []*model.Team{th.BasicTeam},
+ batchTimes: []batchStartEndTimes{{start, jobEndTime}},
+ }
+}
+
+// Type3Results specific data needed to be returned by this test only
+type Type3Results struct {
+ message1DeleteAt int64
+ updatedPost2 *model.Post
+ message3AndFileInfoDeleteAt int64
+ deletedPost3 *model.Post
+}
+
+func generateE2ETestType3Results(t *testing.T, th *api4.TestHelper, exportType, attachmentDir, exportDir string,
+ attachmentBackend, exportBackend filestore.FileBackend) (JobResults, Type3Results) {
+ // This tests (reading the files exported and testing the exported xml):
+ // - post create at field is set
+ // - post deleted fields are set
+ // - post updated (not edited)
+ // - post deleted with a deleted file
+ // - post edited (new message created with original message, old message updated)
+ // - post edited with 3 simultaneous posts in-between - forward
+ // - post edited but falls on the batch boundary (originalId is in batch 1, newId is batch 2)
+
+ start := model.GetMillis()
+
+ // Users:
+ users := make([]*model.User, 0)
+ user, err := th.App.Srv().Store().User().Save(th.Context, &model.User{
+ Username: "user1",
+ Email: "user1@email",
+ })
+ require.NoError(t, err)
+ users = append(users, user)
+
+ // only testing one channel
+ channel2, err := th.App.Srv().Store().Channel().Save(th.Context, &model.Channel{
+ DisplayName: "the Channel Two",
+ Name: "channel_two_name",
+ Type: model.ChannelTypePrivate,
+ TeamId: th.BasicTeam.Id,
+ CreatorId: th.BasicUser.Id,
+ }, 999)
+ require.NoError(t, err)
+
+ // user 1 joins before start time and stays (and posts)
+ user1JoinTime := start - 100
+ err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[0].Id, channel2.Id, user1JoinTime)
+ require.NoError(t, err)
+
+ jl := []joinLeave{
+ {user1JoinTime, 0}, // user 1 never leaves
+ }
+
+ assertNumPostsToExport(t, th, 0, start, model.GetMillis())
+
+ var attachments []*model.FileInfo
+ var contents []string
+ var posts []*model.Post
+
+ // 0 - post create
+ post, err := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: users[0].Id,
+ Message: "message 0",
+ })
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+
+ // 1 - post deleted
+ post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: users[0].Id,
+ Message: "message 1",
+ })
+ require.NoError(t, err)
+ message1DeleteAt := model.GetMillis()
+ err = th.App.Srv().Store().Post().Delete(th.Context, post.Id, message1DeleteAt, users[0].Id)
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+
+ // 2 - post updated not edited (e.g., reaction)
+ post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: users[0].Id,
+ Message: "message 2",
+ })
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+ _, err = th.App.Srv().Store().Reaction().Save(&model.Reaction{
+ UserId: users[0].Id,
+ PostId: post.Id,
+ EmojiName: "smile",
+ ChannelId: channel2.Id,
+ })
+ require.NoError(t, err)
+ updatedPost2, err := th.App.Srv().Store().Post().GetSingle(th.Context, post.Id, false)
+ require.NoError(t, err)
+
+ // 3 - post deleted with a deleted file
+ post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: users[0].Id,
+ Message: "message 3",
+ FileIds: []string{"test3"},
+ })
+ require.NoError(t, err)
+ time.Sleep(100 * time.Millisecond)
+ message3AndFileInfoDeleteAt := model.GetMillis()
+ err = th.App.Srv().Store().Post().Delete(th.Context, post.Id, message3AndFileInfoDeleteAt, users[0].Id)
+ require.NoError(t, err)
+ deletedPost3, err := th.App.Srv().Store().Post().GetSingle(th.Context, post.Id, true)
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+
+ // Message for deleted file -- NOT INCLUDED IN THE BATCH SIZE
+ attachmentContent := "Hello there message 3"
+ attachmentPath := "path/to/attachments/file_3.txt"
+ _, err = attachmentBackend.WriteFile(bytes.NewBufferString(attachmentContent), attachmentPath)
+ require.NoError(t, err)
+ info, err2 := th.App.Srv().Store().FileInfo().Save(th.Context, &model.FileInfo{
+ Id: st.NewTestID(),
+ CreatorId: post.UserId,
+ PostId: post.Id,
+ CreateAt: post.CreateAt,
+ UpdateAt: message3AndFileInfoDeleteAt,
+ Path: attachmentPath,
+ DeleteAt: message3AndFileInfoDeleteAt,
+ })
+ require.NoError(t, err2)
+ attachments = append(attachments, info)
+ time.Sleep(100 * time.Millisecond)
+ contents = append(contents, attachmentContent)
+
+ // 4 - original post
+ post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: users[0].Id,
+ Message: "message 4",
+ })
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+ // 5 - post edited
+ post, err = th.App.Srv().Store().Post().Update(th.Context, &model.Post{
+ Id: post.Id,
+ CreateAt: post.CreateAt,
+ EditAt: model.GetMillis(),
+ ChannelId: channel2.Id,
+ UserId: users[0].Id,
+ Message: "edited message 4",
+ }, post)
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+
+ // 6 - post edited but falls on the batch boundary
+ // original post, but gets modified by the next edit
+ post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: users[0].Id,
+ Message: "message 6",
+ })
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+
+ // 7 - new post with original message
+ // update returns "new" post, which is the old post modified
+ post, err = th.App.Srv().Store().Post().Update(th.Context, &model.Post{
+ Id: post.Id,
+ CreateAt: post.CreateAt,
+ EditAt: model.GetMillis(),
+ ChannelId: channel2.Id,
+ UserId: users[0].Id,
+ Message: "edited message 6",
+ }, post)
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+
+ require.Len(t, posts, 8)
+ // therefore, need a batch size of 7
+
+ // use the config fallback
+ th.App.UpdateConfig(func(cfg *model.Config) {
+ *cfg.MessageExportSettings.EnableExport = true
+ *cfg.MessageExportSettings.ExportFromTimestamp = start
+ *cfg.MessageExportSettings.BatchSize = 7
+ *cfg.MessageExportSettings.ExportFormat = exportType
+ *cfg.FileSettings.DriverName = model.ImageDriverLocal
+ *cfg.FileSettings.Directory = attachmentDir
+
+ if exportDir != attachmentDir {
+ *cfg.FileSettings.DedicatedExportStore = true
+ *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
+ *cfg.FileSettings.ExportDirectory = exportDir
+ }
+ })
+
+ // check number of messages to be exported
+ until := model.GetMillis()
+ assertNumPostsToExport(t, th, 8, start, until)
+
+ // Now run the exports
+ job := runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
+
+ assertNumExported(t, 8, job.Data)
+
+ jobExportDir := job.Data[shared.JobDataExportDir]
+ jobEndTime, err := strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 10, 64)
+ require.NoError(t, err)
+
+ batchTimes := []batchStartEndTimes{
+ {start, posts[7].UpdateAt},
+ {posts[7].UpdateAt, jobEndTime},
+ }
+
+ // using posts[7] because it's updateAt is what posts[6] is changed to (after the edit)
+ batch001 := shared.GetBatchPath(jobExportDir, batchTimes[0].start, batchTimes[0].end, 1)
+ batch002 := shared.GetBatchPath(jobExportDir, batchTimes[1].start, batchTimes[1].end, 2)
+ files, err := exportBackend.ListDirectory(jobExportDir)
+ require.NoError(t, err)
+ batches := []string{batch001, batch002}
+ require.ElementsMatch(t, batches, files)
+
+ return JobResults{
+ start: start,
+ users: users,
+ joinLeaves: jl,
+ jobEndTime: jobEndTime,
+ batches: batches,
+ posts: posts,
+ attachments: attachments,
+ contents: contents,
+ channels: []*model.Channel{channel2},
+ teams: []*model.Team{th.BasicTeam},
+ batchTimes: batchTimes,
+ }, Type3Results{
+ message1DeleteAt: message1DeleteAt,
+ updatedPost2: updatedPost2,
+ message3AndFileInfoDeleteAt: message3AndFileInfoDeleteAt,
+ deletedPost3: deletedPost3,
+ }
+}
+
+func generateE2ETestType4Results(t *testing.T, th *api4.TestHelper, exportType, attachmentDir, exportDir string,
+ attachmentBackend, exportBackend filestore.FileBackend) JobResults {
+ start := model.GetMillis()
+
+ // Users:
+ users := make([]*model.User, 0)
+ user, err := th.App.Srv().Store().User().Save(th.Context, &model.User{
+ Username: "user1",
+ Email: "user1@email",
+ })
+ require.NoError(t, err)
+ users = append(users, user)
+
+ // only testing one channel
+ channel2, err := th.App.Srv().Store().Channel().Save(th.Context, &model.Channel{
+ DisplayName: "the Channel Two",
+ Name: "channel_two_name",
+ Type: model.ChannelTypePrivate,
+ TeamId: th.BasicTeam.Id,
+ CreatorId: th.BasicUser.Id,
+ }, 999)
+ require.NoError(t, err)
+
+ // user 1 joins before start time and stays (and posts)
+ user1JoinTime := model.GetMillis() - 200
+ err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[0].Id, channel2.Id, user1JoinTime)
+ require.NoError(t, err)
+
+ jl := []joinLeave{
+ {user1JoinTime, 0}, // user 1 never leaves
+ }
+
+ // This tests (reading the files exported and testing the exported xml):
+ // - post edited with 3 simultaneous posts in-between
+ assertNumPostsToExport(t, th, 0, start, model.GetMillis())
+
+ var posts []*model.Post
+
+ // 0 - post edited with 3 simultaneous posts in-between - forward
+ // original post with edited message
+ originalPost, err := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: users[0].Id,
+ Message: "message 0",
+ })
+ require.NoError(t, err)
+ posts = append(posts, originalPost)
+ time.Sleep(100 * time.Millisecond)
+
+ // 1 - edited post
+ post, err := th.App.Srv().Store().Post().Update(th.Context, &model.Post{
+ Id: originalPost.Id,
+ CreateAt: originalPost.CreateAt,
+ EditAt: model.GetMillis(),
+ ChannelId: channel2.Id,
+ UserId: users[0].Id,
+ Message: "edited message 0",
+ }, originalPost)
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+
+ simultaneous := post.UpdateAt
+
+ // 2 - post 1 at same updateAt
+ post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: users[0].Id,
+ Message: "message 2",
+ CreateAt: simultaneous,
+ })
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+
+ // 3 - post 2 at same updateAt
+ post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: users[0].Id,
+ Message: "message 3",
+ CreateAt: simultaneous,
+ })
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+
+ // 4 - post 3 in-between
+ post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: users[0].Id,
+ Message: "message 4",
+ CreateAt: simultaneous,
+ })
+ require.NoError(t, err)
+ posts = append(posts, post)
+ // Use the config fallback for simplicity
+
+ th.App.UpdateConfig(func(cfg *model.Config) {
+ *cfg.MessageExportSettings.EnableExport = true
+ *cfg.MessageExportSettings.ExportFromTimestamp = start
+ *cfg.MessageExportSettings.BatchSize = 10
+ *cfg.MessageExportSettings.ExportFormat = exportType
+ *cfg.FileSettings.DriverName = model.ImageDriverLocal
+ *cfg.FileSettings.Directory = attachmentDir
+
+ if exportDir != attachmentDir {
+ *cfg.FileSettings.DedicatedExportStore = true
+ *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
+ *cfg.FileSettings.ExportDirectory = exportDir
+ }
+ })
+
+ // check number of messages to be exported
+ until := model.GetMillis()
+ assertNumPostsToExport(t, th, 5, start, until)
+
+ // Now run the exports
+ job := runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
+ // cleanup for next run
+ _, err = th.App.Srv().Store().Job().Delete(job.Id)
+ require.NoError(t, err)
+
+ assertNumExported(t, 5, job.Data)
+
+ jobExportDir := job.Data[shared.JobDataExportDir]
+ jobEndTime, err := strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 10, 64)
+ require.NoError(t, err)
+
+ batchTimes := []batchStartEndTimes{{start, jobEndTime}}
+
+ batch001 := shared.GetBatchPath(jobExportDir, batchTimes[0].start, batchTimes[0].end, 1)
+ files, err := exportBackend.ListDirectory(jobExportDir)
+ require.NoError(t, err)
+ batches := []string{batch001}
+ require.ElementsMatch(t, batches, files)
+
+ return JobResults{
+ start: start,
+ users: users,
+ joinLeaves: jl,
+ jobEndTime: jobEndTime,
+ batches: batches,
+ posts: posts,
+ channels: []*model.Channel{channel2},
+ teams: []*model.Team{th.BasicTeam},
+ batchTimes: batchTimes,
+ }
+}
+
+// Type5Results specific data needed to be returned by this test only
+type Type5Results struct {
+ message0DeleteAt int64
+ updatedPost1 *model.Post
+ zipBytes [][]byte
+}
+
+// generateE2ETestType5Results does 4 jobs, returns an array of each job's data to use in testing
+func generateE2ETestType5Results(t *testing.T, th *api4.TestHelper, exportType, attachmentDir, exportDir string,
+ attachmentBackend, exportBackend filestore.FileBackend) ([]JobResults, []Type5Results) {
+ // This tests (reading the files exported and testing the exported file):
+ // - post deleted in current job: shows created post, then deleted post
+ // - post deleted in current job but different batch: shows created post (in second batch), then deleted post
+ // - post created in previous job, deleted in current job: shows only deleted post in current job
+ // (and same for updated post)
+
+ start := model.GetMillis()
+
+ channel2, err := th.App.Srv().Store().Channel().Save(th.Context, &model.Channel{
+ DisplayName: "the Channel Two",
+ Name: "channel_two_name",
+ Type: model.ChannelTypePrivate,
+ TeamId: th.BasicTeam.Id,
+ CreatorId: th.BasicUser.Id,
+ }, 999)
+ require.NoError(t, err)
+
+ // user 1 joins before start time and stays (and posts)
+ user1JoinTime := model.GetMillis() - 100
+ err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(th.BasicUser.Id, channel2.Id, user1JoinTime)
+ require.NoError(t, err)
+
+ // Job 1: post deleted in current job: shows created post, then deleted post
+ assertNumPostsToExport(t, th, 0, start, model.GetMillis())
+
+ var posts []*model.Post
+
+ // post create
+ post, err := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: th.BasicUser.Id,
+ Message: "message 0",
+ })
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+
+ // post deleted
+ message0DeleteAt := model.GetMillis()
+ err = th.App.Srv().Store().Post().Delete(th.Context, post.Id, message0DeleteAt, th.BasicUser.Id)
+ require.NoError(t, err)
+
+ // use the config fallback
+ th.App.UpdateConfig(func(cfg *model.Config) {
+ *cfg.MessageExportSettings.EnableExport = true
+ *cfg.MessageExportSettings.ExportFromTimestamp = start
+ *cfg.MessageExportSettings.BatchSize = 2
+ *cfg.MessageExportSettings.ExportFormat = exportType
+ *cfg.FileSettings.DriverName = model.ImageDriverLocal
+ *cfg.FileSettings.Directory = attachmentDir
+
+ if exportDir != attachmentDir {
+ *cfg.FileSettings.DedicatedExportStore = true
+ *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
+ *cfg.FileSettings.ExportDirectory = exportDir
+ }
+ })
+
+ // check number of messages to be exported -- will be 1 (because one message deleted)
+ until := model.GetMillis()
+ assertNumPostsToExport(t, th, 1, start, until)
+
+ // Now run the exports
+ job := runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
+
+ assertNumExported(t, 1, job.Data)
+
+ jobExportDir := job.Data[shared.JobDataExportDir]
+ jobEndTime, err := strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 10, 64)
+ require.NoError(t, err)
+
+ batchTimes := []batchStartEndTimes{
+ {start: start, end: jobEndTime},
+ }
+
+ batch001 := shared.GetBatchPath(jobExportDir, batchTimes[0].start, batchTimes[0].end, 1)
+ files, err := exportBackend.ListDirectory(jobExportDir)
+ require.NoError(t, err)
+ batches := []string{batch001}
+ require.ElementsMatch(t, batches, files)
+
+ zipBytes, err := exportBackend.ReadFile(batches[0])
+ require.NoError(t, err)
+
+ var setupReturns []JobResults
+ var setupType5Returns []Type5Results
+
+ setupReturns = append(setupReturns, JobResults{
+ posts: posts,
+ })
+ setupType5Returns = append(setupType5Returns, Type5Results{
+ message0DeleteAt: message0DeleteAt,
+ zipBytes: [][]byte{zipBytes},
+ })
+
+ // Cleanup for next job
+ err = os.RemoveAll(exportDir)
+ assert.NoError(t, err)
+ err = os.RemoveAll(attachmentDir)
+ assert.NoError(t, err)
+ _, err = th.App.Srv().Store().Job().Delete(job.Id)
+ assert.NoError(t, err)
+
+ //
+ // Job 2
+ //
+
+ // Job 2: post deleted in current job, shows up in second batch because it was deleted after the "second" post
+ start = model.GetMillis()
+
+ posts = make([]*model.Post, 0)
+
+ // post create -- this will be the one deleted
+ post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: th.BasicUser.Id,
+ Message: "message 0",
+ })
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+
+ // post create -- this is the "second" post, but it will show up first because first post is deleted after
+ post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: th.BasicUser.Id,
+ Message: "message 1",
+ })
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+
+ // post deleted -- first post deleted
+ message0DeleteAt = model.GetMillis()
+ err = th.App.Srv().Store().Post().Delete(th.Context, posts[0].Id, message0DeleteAt, th.BasicUser.Id)
+ require.NoError(t, err)
+
+ // use the config fallback
+ th.App.UpdateConfig(func(cfg *model.Config) {
+ *cfg.MessageExportSettings.ExportFromTimestamp = start
+ *cfg.MessageExportSettings.BatchSize = 1
+ })
+
+ // check number of messages to be exported
+ until = model.GetMillis()
+ assertNumPostsToExport(t, th, 2, start, until)
+
+ // Now run the exports
+ job = runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
+
+ assertNumExported(t, 2, job.Data)
+
+ jobExportDir = job.Data[shared.JobDataExportDir]
+ jobEndTime, err = strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 10, 64)
+ require.NoError(t, err)
+
+ // use the message1 updateAt, because the message0's updateAt is now after
+ batch001 = shared.GetBatchPath(jobExportDir, start, posts[1].UpdateAt, 1)
+ batch002 := shared.GetBatchPath(jobExportDir, posts[1].UpdateAt, jobEndTime, 2)
+ files, err = exportBackend.ListDirectory(jobExportDir)
+ require.NoError(t, err)
+ batches = []string{batch001, batch002}
+ require.ElementsMatch(t, batches, files)
+
+ zipBytes, err = exportBackend.ReadFile(batches[0])
+ require.NoError(t, err)
+ zipBytes2, err := exportBackend.ReadFile(batches[1])
+ require.NoError(t, err)
+
+ setupReturns = append(setupReturns, JobResults{
+ posts: posts,
+ })
+ setupType5Returns = append(setupType5Returns, Type5Results{
+ message0DeleteAt: message0DeleteAt,
+ zipBytes: [][]byte{zipBytes, zipBytes2},
+ })
+
+ // Cleanup for next job
+ err = os.RemoveAll(exportDir)
+ assert.NoError(t, err)
+ err = os.RemoveAll(attachmentDir)
+ assert.NoError(t, err)
+ _, err = th.App.Srv().Store().Job().Delete(job.Id)
+ assert.NoError(t, err)
+
+ //
+ // Job 3
+ //
+
+ // Job 3: post created in previous job, deleted in current job: shows only deleted post in current job
+ start = model.GetMillis()
+
+ posts = make([]*model.Post, 0)
+
+ // post create -- this will be the one deleted in second job
+ post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: th.BasicUser.Id,
+ Message: "message 0",
+ })
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+
+ // post create -- this will be the one updated in second job
+ post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: th.BasicUser.Id,
+ Message: "message 1",
+ })
+ require.NoError(t, err)
+ posts = append(posts, post)
+
+ // use the config fallback
+ th.App.UpdateConfig(func(cfg *model.Config) {
+ *cfg.MessageExportSettings.ExportFromTimestamp = start
+ *cfg.MessageExportSettings.BatchSize = 10
+ })
+
+ until = model.GetMillis()
+ assertNumPostsToExport(t, th, 2, start, until)
+
+ // Now run the exports
+ job = runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
+
+ assertNumExported(t, 2, job.Data)
+
+ jobExportDir = job.Data[shared.JobDataExportDir]
+ jobEndTime, err = strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 10, 64)
+ require.NoError(t, err)
+
+ batch001 = shared.GetBatchPath(jobExportDir, start, jobEndTime, 1)
+ files, err = exportBackend.ListDirectory(jobExportDir)
+ require.NoError(t, err)
+ batches = []string{batch001}
+ require.ElementsMatch(t, batches, files)
+
+ zipBytes, err = exportBackend.ReadFile(batches[0])
+ require.NoError(t, err)
+
+ setupReturns = append(setupReturns, JobResults{
+ posts: posts,
+ })
+ setupType5Returns = append(setupType5Returns, Type5Results{
+ zipBytes: [][]byte{zipBytes},
+ })
+
+ // Now, clean up outputs for next job
+ err = os.RemoveAll(exportDir)
+ assert.NoError(t, err)
+ err = os.RemoveAll(attachmentDir)
+ assert.NoError(t, err)
+ _, err = th.App.Srv().Store().Job().Delete(job.Id)
+ assert.NoError(t, err)
+
+ //
+ // Job 4
+ //
+ start = model.GetMillis()
+
+ // make a copy of posts so the one we sent earlier doesn't get modified and cause difficult to detect bugs
+ posts = append([]*model.Post{}, posts...)
+
+ // post create -- filler
+ post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: channel2.Id,
+ UserId: th.BasicUser.Id,
+ Message: "message 1",
+ })
+ require.NoError(t, err)
+ posts = append(posts, post)
+ time.Sleep(100 * time.Millisecond)
+
+ // post deleted -- first post deleted (the first one exported earlier)
+ message0DeleteAt = model.GetMillis()
+ err = th.App.Srv().Store().Post().Delete(th.Context, posts[0].Id, message0DeleteAt, th.BasicUser.Id)
+ require.NoError(t, err)
+
+ // post updated -- second post updated (the second one exported earlier)
+ _, err = th.App.Srv().Store().Reaction().Save(&model.Reaction{
+ UserId: th.BasicUser.Id,
+ PostId: posts[1].Id,
+ EmojiName: "smile",
+ ChannelId: channel2.Id,
+ })
+ require.NoError(t, err)
+ updatedPost1, err := th.App.Srv().Store().Post().GetSingle(th.Context, posts[1].Id, false)
+ require.NoError(t, err)
+
+ // use the config fallback
+ th.App.UpdateConfig(func(cfg *model.Config) {
+ *cfg.MessageExportSettings.ExportFromTimestamp = start
+ })
+
+ // check number of messages to be exported
+ // filler post, deleted post, and updated post
+ until = model.GetMillis()
+ assertNumPostsToExport(t, th, 3, start, until)
+
+ // Now run the exports
+ job = runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
+
+ assertNumExported(t, 3, job.Data)
+
+ jobExportDir = job.Data[shared.JobDataExportDir]
+ jobEndTime, err = strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 10, 64)
+ require.NoError(t, err)
+
+ batch001 = shared.GetBatchPath(jobExportDir, start, jobEndTime, 1)
+ files, err = exportBackend.ListDirectory(jobExportDir)
+ require.NoError(t, err)
+ batches = []string{batch001}
+ require.ElementsMatch(t, batches, files)
+
+ zipBytes, err = exportBackend.ReadFile(batches[0])
+ require.NoError(t, err)
+
+ setupReturns = append(setupReturns, JobResults{
+ posts: posts,
+ })
+ setupType5Returns = append(setupType5Returns, Type5Results{
+ message0DeleteAt: message0DeleteAt,
+ updatedPost1: updatedPost1,
+ zipBytes: [][]byte{zipBytes},
+ })
+
+ // Cleanup
+ err = os.RemoveAll(exportDir)
+ assert.NoError(t, err)
+ err = os.RemoveAll(attachmentDir)
+ assert.NoError(t, err)
+ _, err = th.App.Srv().Store().Job().Delete(job.Id)
+ assert.NoError(t, err)
+
+ return setupReturns, setupType5Returns
+}
diff --git a/server/enterprise/message_export/scheduler.go b/server/enterprise/message_export/scheduler.go
index f100fb3fcc..99aa440d7e 100644
--- a/server/enterprise/message_export/scheduler.go
+++ b/server/enterprise/message_export/scheduler.go
@@ -39,7 +39,7 @@ func (s *MessageExportScheduler) NextScheduleTime(cfg *model.Config, now time.Ti
if err != nil {
s.jobServer.Logger().Error(
"Cannot determine next schedule time for message export. DailyRunTime config value is invalid.",
- mlog.String("DailyRunTime", *cfg.MessageExportSettings.DailyRunTime),
+ mlog.String("daily_run_time", *cfg.MessageExportSettings.DailyRunTime),
)
return nil
}
diff --git a/server/enterprise/message_export/shared/export_data.go b/server/enterprise/message_export/shared/export_data.go
new file mode 100644
index 0000000000..3ac2f29c9a
--- /dev/null
+++ b/server/enterprise/message_export/shared/export_data.go
@@ -0,0 +1,543 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package shared
+
+import (
+ "sort"
+
+ "github.com/mattermost/mattermost/server/public/model"
+)
+
+type UserType string
+
+const (
+ User UserType = "user"
+ Bot UserType = "bot"
+)
+
+type PostExport struct {
+ model.MessageExport // the MessageExport that this PostExport is providing more information for
+ UserType UserType // the type of the person that sent the post: "user" or "bot"
+
+ // Allows us to differentiate between:
+ // - "EditedOriginalMsg": the newly created message (new Id), which holds the pre-edited message contents. The
+ // "EditedNewMsgId" field will point to the message (original Id) which has the post-edited message content.
+ // - "EditedNewMsg": the post-edited message content. This is confusing, so be careful: in the db, this EditedNewMsg
+ // is actually the original messageId because we wanted an edited message to have the same messageId as the
+ // pre-edited message. But for the purposes of exporting and to keep the mental model clear for end-users, we are
+ // calling this the EditedNewMsg and EditedNewMsgId, because this will hold the NEW post-edited message contents,
+ // and that's what's important to the end-user viewing the export.
+ // - "UpdatedNoMsgChange": the message content hasn't changed, but the post was updated for some reason (reaction,
+ // replied-to, a reply was edited, a reply was deleted (as of 10.2), perhaps other reasons)
+ // - "Deleted": the message was deleted.
+ // - "FileDeleted": this message is recording that a file was deleted.
+ UpdatedType PostUpdatedType
+ UpdateAt int64 // if this is an updated post, this is the updated time (same as deleted time for deleted posts).
+
+ // when a message is edited, the EditedOriginalMsg points to the message Id that now has the newly edited message.
+ EditedNewMsgId string
+ Message string // the text body of the post
+ PreviewsPost string // the post id of the post that is previewed by the permalink preview feature
+ AttachmentCreates []*FileUploadStartExport // the post's attachments that were uploaded this export period
+ AttachmentDeletes []PostExport // the post's attachments that were deleted
+ FileInfo *model.FileInfo // if this was a file PostExport, FileInfo will contain that info. Otherwise, nil.
+}
+
+type FileUploadStartExport struct {
+ model.MessageExport // the post that this upload was attached to
+ UserEmail string // the email of the person that sent the file
+ UploadStartTime int64 // utc timestamp (seconds), time at which the user started the upload. Example: 1366611728
+ FileInfo *model.FileInfo
+}
+
+type FileUploadStopExport struct {
+ model.MessageExport // the post that this upload was attached to
+ UserEmail string // the email of the person that sent the file
+ UploadStopTime int64 // utc timestamp (seconds), time at which the user finished the upload. Example: 1366611728
+ Status string // set to either "Completed" or "Failed" depending on the outcome of the upload operation
+ FileInfo *model.FileInfo
+}
+
+type ChannelExport struct {
+ ChannelId string
+ ChannelType model.ChannelType
+ ChannelName string
+ DisplayName string
+ StartTime int64 // utc timestamp (milliseconds), start of export period or create time of channel, whichever is greater.
+ EndTime int64 // utc timestamp (milliseconds), end of export period or delete time of channel, whichever is lesser.
+ Posts []PostExport
+ Files []*model.FileInfo
+ DeletedFiles []PostExport
+ UploadStarts []*FileUploadStartExport
+ UploadStops []*FileUploadStopExport
+ JoinEvents []JoinExport // start with a list of all users who were present in the channel during the export period
+ LeaveEvents []LeaveExport // finish with a list of all users who were present in the channel during the export period
+
+ // Used by csv, ignored by others
+ TeamId string
+ TeamName string
+ TeamDisplayName string
+}
+
+type JoinExport struct {
+ UserId string
+ Username string
+ UserEmail string // the email of the person that joined the channel
+ UserType UserType // the type of the user that joined the channel
+ JoinTime int64 // utc timestamp (seconds), time at which the user joined. Example: 1366611728
+
+ // when the user left (or batch endTime if they didn't leave). Only used by GlobalRelay
+ LeaveTime int64
+}
+
+type LeaveExport struct {
+ UserId string
+ Username string
+ UserEmail string // the email of the person that left the channel
+ UserType UserType // the type of the user that left the channel
+ LeaveTime int64 // utc timestamp (seconds), time at which the user left. Example: 1366611728
+
+ // ClosedOut indicates this is a "leave" event created by closing out the channel at the end of an export period.
+ // Actiance requires all users to be closed out at the end of an export period (each join has a matching leave).
+ ClosedOut bool
+}
+
+type GenericExportData struct {
+ Exports []ChannelExport
+ Metadata Metadata
+ Results RunExportResults
+}
+
+// GetGenericExportData assembles all the data in an exportType-agnostic way. Each exportType will process this data into
+// the specific format they need to export.
+func GetGenericExportData(p ExportParams) (GenericExportData, error) {
+ // postAuthorsByChannel is a map so that we don't store duplicate authors
+ postAuthorsByChannel := make(map[string]map[string]ChannelMember)
+ metadata := Metadata{
+ Channels: p.ChannelMetadata,
+ MessagesCount: 0,
+ AttachmentsCount: 0,
+ StartTime: p.BatchStartTime,
+ EndTime: p.BatchEndTime,
+ }
+
+ var results RunExportResults
+ channelsInThisBatch := make(map[string]bool)
+ postsByChannel := make(map[string][]PostExport)
+ filesByChannel := make(map[string][]*model.FileInfo)
+ uploadStartsByChannel := make(map[string][]*FileUploadStartExport)
+ uploadStopsByChannel := make(map[string][]*FileUploadStopExport)
+ deletedFilesByChannel := make(map[string][]PostExport)
+
+ processPostAttachments := func(post *model.MessageExport, postExport PostExport, originalPostThatWillBeDeletedLater bool) error {
+ // originalPostThatWillBeDeletedLater means we are recording this message's original file starts and stops,
+ // before it was deleted (we'll record that next call to this function)
+ //
+ // NOTE: there is an edge case here: the original post is not deleted but the attachment has been deleted.
+ // See the note in the postToAttachmentsEntries func.
+
+ channelId := *post.ChannelId
+ uploadedFiles, startUploads, stopUploads, deleteFileMessages, err :=
+ postToAttachmentsEntries(post, p.Db, originalPostThatWillBeDeletedLater)
+ if err != nil {
+ return err
+ }
+ uploadStartsByChannel[channelId] = append(uploadStartsByChannel[channelId], startUploads...)
+ uploadStopsByChannel[channelId] = append(uploadStopsByChannel[channelId], stopUploads...)
+ deletedFilesByChannel[channelId] = append(deletedFilesByChannel[channelId], deleteFileMessages...)
+ filesByChannel[channelId] = append(filesByChannel[channelId], uploadedFiles...)
+
+ postExport.AttachmentCreates = startUploads
+ postExport.AttachmentDeletes = deleteFileMessages
+ postsByChannel[channelId] = append(postsByChannel[channelId], postExport)
+
+ results.UploadedFiles += len(startUploads)
+ results.DeletedFiles += len(deleteFileMessages)
+
+ // only count uploaded files (not deleted files)
+ if err := metadata.UpdateCounts(channelId, 1, len(startUploads)); err != nil {
+ return err
+ }
+
+ return nil
+ }
+
+ for _, post := range p.Posts {
+ channelId := *post.ChannelId
+ channelsInThisBatch[channelId] = true
+
+ // Was the post deleted (not an edited post), and originally posted during the current job window?
+ // If so, we need to record it. It may actually belong in an earlier batch, but there's no way to know that
+ // before now because of the way we export posts (by updateAt).
+ if IsDeletedMsg(post) && !isEditedOriginalMsg(post) && *post.PostCreateAt >= p.JobStartTime {
+ results.CreatedPosts++
+ postExport := createdPostToExportEntry(post)
+ if err := processPostAttachments(post, postExport, true); err != nil {
+ return GenericExportData{}, err
+ }
+ }
+ var postExport PostExport
+ postExport, results = getPostExport(post, results)
+
+ if err := processPostAttachments(post, postExport, false); err != nil {
+ return GenericExportData{}, err
+ }
+
+ if _, ok := postAuthorsByChannel[channelId]; !ok {
+ postAuthorsByChannel[channelId] = make(map[string]ChannelMember)
+ }
+ postAuthorsByChannel[channelId][*post.UserId] = ChannelMember{
+ UserId: *post.UserId,
+ Email: *post.UserEmail,
+ Username: *post.Username,
+ IsBot: post.IsBot,
+ }
+ }
+
+ // If the channel is not in channelsInThisBatch (i.e. if it didn't have a post), we need to check if it had
+ // user activity between this batch's startTime-endTime. If so, add it to the channelsInThisBatch.
+ for id := range p.ChannelMetadata {
+ if !channelsInThisBatch[id] {
+ if ChannelHasActivity(p.ChannelMemberHistories[id], p.BatchStartTime, p.BatchEndTime) {
+ channelsInThisBatch[id] = true
+ }
+ }
+ }
+
+ // Build the channel exports for the channels that had post or user join/leave activity this batch.
+ channelExports := make([]ChannelExport, 0, len(channelsInThisBatch))
+ for id := range channelsInThisBatch {
+ c := metadata.Channels[id]
+
+ joinEvents, leaveEvents := getJoinsAndLeaves(p.BatchStartTime, p.BatchEndTime,
+ p.ChannelMemberHistories[id], postAuthorsByChannel[id])
+
+ // We don't have teamName and teamDisplayName from the channelMetaData, but we have it from MessageExport.
+ // However, if we don't have posts for this channel (only joins and leaves), then we don't have it at all.
+ var teamName, teamDisplayName string
+ if posts, ok := postsByChannel[id]; ok {
+ if len(posts) > 0 {
+ teamName = model.SafeDereference(posts[0].TeamName)
+ teamDisplayName = model.SafeDereference(posts[0].TeamDisplayName)
+ }
+ }
+
+ channelExports = append(channelExports, ChannelExport{
+ ChannelId: c.ChannelId,
+ ChannelType: c.ChannelType,
+ ChannelName: c.ChannelName,
+ DisplayName: c.ChannelDisplayName,
+ StartTime: p.BatchStartTime,
+ EndTime: p.BatchEndTime,
+ Posts: postsByChannel[id],
+ Files: filesByChannel[id],
+ DeletedFiles: deletedFilesByChannel[id],
+ UploadStarts: uploadStartsByChannel[id],
+ UploadStops: uploadStopsByChannel[id],
+ JoinEvents: joinEvents,
+ LeaveEvents: leaveEvents,
+ TeamId: model.SafeDereference(c.TeamId),
+ TeamName: teamName,
+ TeamDisplayName: teamDisplayName,
+ })
+
+ results.Joins += len(joinEvents)
+ results.Leaves += len(leaveEvents)
+ }
+
+ return GenericExportData{channelExports, metadata, results}, nil
+}
+
+// postToAttachmentsEntries returns every fileInfo as uploadedFiles. It also adds each file into the lists:
+//
+// startUploads, stopUploads, and deleteFileMessages (for ActianceExport).
+// If onlyDeleted = true, only export the deleted entry (if it exists).
+func postToAttachmentsEntries(post *model.MessageExport, db MessageExportStore, ignoreDeleted bool) (
+ uploadedFiles []*model.FileInfo, startUploads []*FileUploadStartExport, stopUploads []*FileUploadStopExport, deleteFileMessages []PostExport, err error) {
+ // if the post included any files, we need to add special elements to the export.
+ //
+ // NOTE: there is an edge case here: the original post is not deleted but the attachment has been deleted.
+ // 1. If the attachment is deleted sometime later in the future but the post has not been updated (updateAt == createAt)
+ // then we won't ever get here, there's nothing we can do.
+ // 2. If the attachment is deleted within this export period, or the post is updated this export period and we
+ // now see that the attachment is deleted, then we have to output 2 files here:
+ // a) the original file attachment
+ // b) the deleted file attachment
+ // These have to be added the start/stop and deletedFileMessages
+
+ if len(post.PostFileIds) == 0 {
+ return
+ }
+
+ uploadedFiles, err = db.FileInfo().GetForPost(*post.PostId, true, true, false)
+ if err != nil {
+ return
+ }
+
+ for _, fileInfo := range uploadedFiles {
+ if fileInfo.DeleteAt > 0 && !ignoreDeleted {
+ deleteFileMessages = append(deleteFileMessages, deleteFileToExportEntry(post, fileInfo))
+
+ // this was a deleted file, so do not record its start and stop. If the original message was sent in this
+ // batch, the file transfer will have been exported earlier up when the original message was exported.
+ //
+ // However, because of the edge case above, we still need to record start and stop if the post is not deleted.
+ if IsDeletedMsg(post) {
+ continue
+ } // not deleted, so need to add start and stop below.
+ }
+
+ // insert a record of the file upload into the export file
+ // path to exported file is relative to the fileAttachmentFilestore root,
+ // which could be different from the exportFilestore root
+ startUploads = append(startUploads, &FileUploadStartExport{
+ MessageExport: *post,
+ UserEmail: *post.UserEmail,
+ UploadStartTime: *post.PostCreateAt,
+ FileInfo: fileInfo,
+ })
+
+ stopUploads = append(stopUploads, &FileUploadStopExport{
+ MessageExport: *post,
+ UserEmail: *post.UserEmail,
+ UploadStopTime: *post.PostCreateAt,
+ Status: "Completed",
+ FileInfo: fileInfo,
+ })
+ }
+ return
+}
+
+func getPostExport(post *model.MessageExport, results RunExportResults) (PostExport, RunExportResults) {
+ // We have three "kinds" of posts:
+ // (using "1" and "2" for simplicity)
+ // - created: Id = new, CreateAt = 1, UpdateAt = 1, DeleteAt = 0
+ // - deleted: Id = orig, CreateAt = orig, UpdateAt = 2, DeleteAt = 2, props: deleteBy
+ // - edited: old post gets "created": Id = new, CreateAt = 1, UpdateAt = 2, DeleteAt = 2, originalId: orig
+ // existing post modified: Id = orig, CreateAt = 1, UpdateAt = 2, DeleteAt = 0
+ //
+ // We also have other ways for a post to be updated:
+ // - a root post in a thread is replied to, when a reply is edited, or (as of 10.2) when a reply is deleted
+
+ if isEditedOriginalMsg(post) {
+ // Post has been edited. This is the original message.
+ results.EditedOrigMsgPosts++
+ return editedOriginalMsgToExportEntry(post), results
+ } else if IsDeletedMsg(post) {
+ // Post is deleted
+ results.DeletedPosts++
+ return deletedPostToExportEntry(post, "delete "+*post.PostMessage), results
+ } else if *post.PostUpdateAt > *post.PostCreateAt {
+ // Post has been updated. But what kind?
+ if model.SafeDereference(post.PostEditAt) > 0 {
+ // This is an edited post.
+ results.EditedNewMsgPosts++
+ return editedNewMsgToExportEntry(post), results
+ }
+ // This is just an updated post (e.g. reaction)
+ results.UpdatedPosts++
+ return updatedPostToExportEntry(post), results
+ }
+ // Post is newly created:
+ // *post.PostCreateAt == *post.PostUpdateAt && (post.PostDeleteAt == nil || *post.PostDeleteAt == 0)
+ // but also fallback to this in case there is missing data, which is better than not exporting anything.
+ results.CreatedPosts++
+ return createdPostToExportEntry(post), results
+}
+
+func getJoinsAndLeaves(startTime int64, endTime int64, channelMembersHistory []*model.ChannelMemberHistoryResult,
+ postAuthors map[string]ChannelMember) ([]JoinExport, []LeaveExport) {
+ var leaveEvents []LeaveExport
+
+ joins, leaves := GetJoinsAndLeavesForChannel(startTime, endTime, channelMembersHistory, postAuthors)
+ joinsById := make(map[string]JoinExport, len(joins))
+ type StillMemberInfo struct {
+ time int64
+ userType UserType
+ userId string
+ username string
+ }
+ stillMember := map[string]StillMemberInfo{}
+ for _, join := range joins {
+ userType := User
+ if join.IsBot {
+ userType = Bot
+ }
+ joinsById[join.UserId] = JoinExport{
+ UserId: join.UserId,
+ Username: join.Username,
+ UserEmail: join.Email,
+ JoinTime: join.Datetime,
+ UserType: userType,
+ LeaveTime: endTime,
+ }
+ if value, ok := stillMember[join.Email]; !ok {
+ stillMember[join.Email] = StillMemberInfo{time: join.Datetime, userType: userType, userId: join.UserId, username: join.Username}
+ } else if join.Datetime > value.time {
+ stillMember[join.Email] = StillMemberInfo{time: join.Datetime, userType: userType, userId: join.UserId, username: join.Username}
+ }
+ }
+ for _, leave := range leaves {
+ userType := User
+ if leave.IsBot {
+ userType = Bot
+ }
+ leaveEvents = append(leaveEvents, LeaveExport{
+ UserId: leave.UserId,
+ Username: leave.Username,
+ UserEmail: leave.Email,
+ LeaveTime: leave.Datetime,
+ UserType: userType,
+ })
+ if leave.Datetime > stillMember[leave.Email].time {
+ delete(stillMember, leave.Email)
+ }
+
+ // record their leave in their initial join
+ if join, ok := joinsById[leave.UserId]; ok {
+ join.LeaveTime = leave.Datetime
+ joinsById[leave.UserId] = join
+ }
+ }
+
+ // Closing-out the channel for Actiance (each join must have a matching leave).
+ for email := range stillMember {
+ leaveEvents = append(leaveEvents, LeaveExport{
+ UserId: stillMember[email].userId,
+ Username: stillMember[email].username,
+ LeaveTime: endTime,
+ UserEmail: email,
+ UserType: stillMember[email].userType,
+ ClosedOut: true,
+ })
+ }
+
+ joinEvents := make([]JoinExport, 0, len(joinsById))
+ for _, v := range joinsById {
+ joinEvents = append(joinEvents, v)
+ }
+
+ sort.Slice(joinEvents, func(i, j int) bool {
+ if joinEvents[i].JoinTime == joinEvents[j].JoinTime {
+ return joinEvents[i].UserEmail < joinEvents[j].UserEmail
+ }
+ return joinEvents[i].JoinTime < joinEvents[j].JoinTime
+ })
+
+ sort.Slice(leaveEvents, func(i, j int) bool {
+ if leaveEvents[i].LeaveTime == leaveEvents[j].LeaveTime {
+ return leaveEvents[i].UserEmail < leaveEvents[j].UserEmail
+ }
+ return leaveEvents[i].LeaveTime < leaveEvents[j].LeaveTime
+ })
+
+ return joinEvents, leaveEvents
+}
+
+func isEditedOriginalMsg(post *model.MessageExport) bool {
+ return model.SafeDereference(post.PostDeleteAt) > 0 && model.SafeDereference(post.PostOriginalId) != ""
+}
+
+func createdPostToExportEntry(post *model.MessageExport) PostExport {
+ userType := User
+ if post.IsBot {
+ userType = Bot
+ }
+ return PostExport{
+ MessageExport: *post,
+ Message: *post.PostMessage,
+ UserType: userType,
+ PreviewsPost: post.PreviewID(),
+ }
+}
+
+func deletedPostToExportEntry(post *model.MessageExport, newMsg string) PostExport {
+ userType := User
+ if post.IsBot {
+ userType = Bot
+ }
+ return PostExport{
+ MessageExport: *post,
+ UpdateAt: *post.PostDeleteAt,
+ UpdatedType: Deleted,
+ Message: newMsg,
+ UserType: userType,
+ PreviewsPost: post.PreviewID(),
+ }
+}
+
+func editedOriginalMsgToExportEntry(post *model.MessageExport) PostExport {
+ userType := User
+ if post.IsBot {
+ userType = Bot
+ }
+ return PostExport{
+ MessageExport: *post,
+ UpdateAt: *post.PostUpdateAt,
+ UpdatedType: EditedOriginalMsg,
+ Message: *post.PostMessage,
+ UserType: userType,
+ PreviewsPost: post.PreviewID(),
+ EditedNewMsgId: *post.PostOriginalId,
+ }
+}
+
+func editedNewMsgToExportEntry(post *model.MessageExport) PostExport {
+ userType := User
+ if post.IsBot {
+ userType = Bot
+ }
+ return PostExport{
+ MessageExport: *post,
+ UpdateAt: *post.PostUpdateAt,
+ UpdatedType: EditedNewMsg,
+ Message: *post.PostMessage,
+ UserType: userType,
+ PreviewsPost: post.PreviewID(),
+ }
+}
+
+func updatedPostToExportEntry(post *model.MessageExport) PostExport {
+ userType := User
+ if post.IsBot {
+ userType = Bot
+ }
+ return PostExport{
+ MessageExport: *post,
+ UpdateAt: *post.PostUpdateAt,
+ UpdatedType: UpdatedNoMsgChange,
+ Message: *post.PostMessage,
+ UserType: userType,
+ PreviewsPost: post.PreviewID(),
+ }
+}
+
+func deleteFileToExportEntry(post *model.MessageExport, fileInfo *model.FileInfo) PostExport {
+ userType := User
+ if post.IsBot {
+ userType = Bot
+ }
+ return PostExport{
+ MessageExport: *post,
+ UpdateAt: fileInfo.DeleteAt,
+ UpdatedType: FileDeleted,
+ Message: "delete " + fileInfo.Path,
+ UserType: userType,
+ PreviewsPost: post.PreviewID(),
+ FileInfo: fileInfo,
+ }
+}
+
+func UploadStartToExportEntry(u *FileUploadStartExport) PostExport {
+ userType := User
+ if u.IsBot {
+ userType = Bot
+ }
+ return PostExport{
+ MessageExport: u.MessageExport,
+ UpdateAt: u.FileInfo.UpdateAt,
+ UserType: userType,
+ FileInfo: u.FileInfo,
+ }
+}
diff --git a/server/enterprise/message_export/shared/export_data_test.go b/server/enterprise/message_export/shared/export_data_test.go
new file mode 100644
index 0000000000..2231b96f4d
--- /dev/null
+++ b/server/enterprise/message_export/shared/export_data_test.go
@@ -0,0 +1,523 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package shared
+
+import (
+ "encoding/json"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+ "github.com/stretchr/testify/require"
+
+ "github.com/mattermost/mattermost/server/public/model"
+ "github.com/mattermost/mattermost/server/v8/channels/api4"
+ "github.com/mattermost/mattermost/server/v8/channels/jobs"
+ "github.com/mattermost/mattermost/server/v8/channels/store/storetest"
+)
+
+func postToMessageExport(t *testing.T, p *model.Post, u *model.User, c *model.Channel, team *model.Team) model.MessageExport {
+ t.Helper()
+ props, err := json.Marshal(p.GetProps())
+ assert.NoError(t, err)
+
+ return model.MessageExport{
+ TeamId: &team.Id,
+ TeamName: &team.Name,
+ TeamDisplayName: &team.DisplayName,
+ ChannelId: &c.Id,
+ ChannelName: &c.Name,
+ ChannelDisplayName: &c.DisplayName,
+ ChannelType: &c.Type,
+ UserId: &u.Id,
+ UserEmail: model.NewPointer(u.Email),
+ Username: &u.Username,
+ IsBot: false,
+ PostId: model.NewPointer(p.Id),
+ PostCreateAt: model.NewPointer(p.CreateAt),
+ PostUpdateAt: &p.UpdateAt,
+ PostDeleteAt: &p.DeleteAt,
+ PostEditAt: &p.EditAt,
+ PostMessage: &p.Message,
+ PostType: &p.Type,
+ PostRootId: &p.RootId,
+ PostProps: model.NewPointer(string(props)),
+ PostOriginalId: &p.OriginalId,
+ PostFileIds: p.FileIds,
+ }
+}
+
+func Test_getPostExport(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skipping test in short mode.")
+ }
+
+ jobs.DefaultWatcherPollingInterval = 100
+ th := api4.SetupEnterprise(t).InitBasic()
+ th.App.Srv().SetLicense(model.NewTestLicense("message_export"))
+ defer th.TearDown()
+
+ // the post exports from the db will be random (because they all have the same updateAt), so do it a few times
+ for i := 0; i < 10; i++ {
+ time.Sleep(time.Millisecond)
+ start := model.GetMillis()
+
+ count, err := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{ExcludeSystemPosts: true, SincePostID: "", SinceUpdateAt: start})
+ require.NoError(t, err)
+ require.Equal(t, 0, int(count))
+
+ var posts []*model.Post
+
+ // 0 - post edited with 3 simultaneous posts in-between - forward
+ // original post with edited message
+ originalPost, err := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: th.BasicChannel.Id,
+ UserId: th.BasicUser.Id,
+ Message: "message 0",
+ })
+ require.NoError(t, err)
+ require.NotEqual(t, 0, originalPost.UpdateAt, "originalPost's updateAt was zero, test 1")
+ posts = append(posts, originalPost)
+
+ // If we don't sleep, the two messages might not have different CreateAt and UpdateAts
+ time.Sleep(time.Millisecond)
+
+ // 1 - edited post
+ post, err := th.App.Srv().Store().Post().Update(th.Context, &model.Post{
+ Id: originalPost.Id,
+ CreateAt: originalPost.CreateAt,
+ EditAt: model.GetMillis(),
+ ChannelId: th.BasicChannel.Id,
+ UserId: th.BasicUser.Id,
+ Message: "edited message 0",
+ }, originalPost)
+ require.NoError(t, err)
+ require.NotEqual(t, 0, originalPost.UpdateAt, "originalPost's updateAt was zero, test 2")
+ require.NotEqual(t, 0, post.UpdateAt, "edited post's updateAt was zero, test 2")
+ posts = append(posts, post)
+
+ simultaneous := post.UpdateAt
+
+ // Add 8 other posts at the same updateAt
+ for j := 1; j <= 8; j++ {
+ // 2 - post 1 at same updateAt
+ post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+ ChannelId: th.BasicChannel.Id,
+ UserId: th.BasicUser.Id,
+ Message: fmt.Sprintf("message %d", j),
+ CreateAt: simultaneous,
+ })
+ require.NoError(t, err)
+ require.NotEqual(t, 0, post.UpdateAt)
+ posts = append(posts, post)
+ }
+
+ // Use the config fallback for simplicity
+ th.App.UpdateConfig(func(cfg *model.Config) {
+ *cfg.MessageExportSettings.EnableExport = true
+ *cfg.MessageExportSettings.ExportFromTimestamp = start
+ *cfg.MessageExportSettings.BatchSize = 10
+ })
+
+ // the messages can be in any order because all have equal `updateAt`s
+ expectedExports := []PostExport{
+ {
+ MessageExport: postToMessageExport(t, posts[0], th.BasicUser, th.BasicChannel, th.BasicTeam),
+ UserType: "user",
+ Message: posts[0].Message,
+ UpdateAt: posts[1].UpdateAt, // the edit update at
+ UpdatedType: EditedOriginalMsg,
+ EditedNewMsgId: posts[1].Id,
+ },
+ {
+ MessageExport: postToMessageExport(t, posts[1], th.BasicUser, th.BasicChannel, th.BasicTeam),
+ UserType: "user",
+ Message: posts[1].Message,
+ UpdateAt: posts[1].UpdateAt,
+ UpdatedType: EditedNewMsg,
+ },
+ }
+
+ for j := 2; j < 10; j++ {
+ expectedExports = append(expectedExports, PostExport{
+ MessageExport: postToMessageExport(t, posts[j], th.BasicUser, th.BasicChannel, th.BasicTeam),
+ UserType: "user",
+ Message: posts[j].Message,
+ })
+ }
+
+ actualMessageExports, _, err := th.App.Srv().Store().Compliance().MessageExport(th.Context, model.MessageExportCursor{
+ LastPostUpdateAt: start,
+ UntilUpdateAt: model.GetMillis(),
+ }, 10)
+ require.NoError(t, err)
+ require.Len(t, actualMessageExports, 10)
+ for _, export := range actualMessageExports {
+ require.NotEqual(t, 0, *export.PostUpdateAt)
+ }
+ results := RunExportResults{}
+ var actualExports []PostExport
+
+ for _, m := range actualMessageExports {
+ var postExport PostExport
+ postExport, results = getPostExport(m, results)
+ actualExports = append(actualExports, postExport)
+ }
+
+ require.ElementsMatch(t, expectedExports, actualExports, fmt.Sprintf("batch %d", i))
+ }
+}
+
+func TestPostToAttachmentsEntries(t *testing.T) {
+ chanTypeDirect := model.ChannelTypeDirect
+ tt := []struct {
+ name string
+ post model.MessageExport
+ attachments []*model.FileInfo
+ expectedStarts []*FileUploadStartExport
+ expectedStops []*FileUploadStopExport
+ expectedFileInfos []*model.FileInfo
+ expectedDeleteFileMessages []PostExport
+ ignoreDeleted bool
+ expectError bool
+ }{
+ {
+ name: "no-attachments",
+ post: model.MessageExport{
+ ChannelId: model.NewPointer("Test"),
+ ChannelDisplayName: model.NewPointer("Test"),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostMessage: model.NewPointer("Some message"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("test"),
+ Username: model.NewPointer("test"),
+ ChannelType: &chanTypeDirect,
+ },
+ attachments: nil,
+ expectedStarts: nil,
+ expectedStops: nil,
+ expectedFileInfos: nil,
+ expectedDeleteFileMessages: nil,
+ expectError: false,
+ },
+ {
+ name: "one-attachment",
+ post: model.MessageExport{
+ PostId: model.NewPointer("test"),
+ ChannelId: model.NewPointer("Test"),
+ ChannelDisplayName: model.NewPointer("Test"),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostMessage: model.NewPointer("Some message"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("test"),
+ Username: model.NewPointer("test"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{"12345"},
+ },
+ attachments: []*model.FileInfo{
+ {Name: "test", Id: "12345", Path: "filename.txt"},
+ },
+ expectedStarts: []*FileUploadStartExport{
+ {UserEmail: "test@test.com", UploadStartTime: 1,
+ FileInfo: &model.FileInfo{Id: "12345", Name: "test", Path: "filename.txt"}},
+ },
+ expectedStops: []*FileUploadStopExport{
+ {UserEmail: "test@test.com", UploadStopTime: 1,
+ FileInfo: &model.FileInfo{Id: "12345", Name: "test", Path: "filename.txt"}, Status: "Completed"},
+ },
+ expectedFileInfos: []*model.FileInfo{
+ {Name: "test", Id: "12345", Path: "filename.txt"},
+ },
+ expectedDeleteFileMessages: nil,
+ expectError: false,
+ },
+ {
+ name: "two-attachment",
+ post: model.MessageExport{
+ PostId: model.NewPointer("test"),
+ ChannelId: model.NewPointer("Test"),
+ ChannelDisplayName: model.NewPointer("Test"),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostMessage: model.NewPointer("Some message"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("test"),
+ Username: model.NewPointer("test"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{"12345", "54321"},
+ },
+ attachments: []*model.FileInfo{
+ {Name: "test", Id: "12345", Path: "filename.txt"},
+ {Name: "test2", Id: "54321", Path: "filename2.txt"},
+ },
+ expectedStarts: []*FileUploadStartExport{
+ {UserEmail: "test@test.com", UploadStartTime: 1,
+ FileInfo: &model.FileInfo{Id: "12345", Name: "test", Path: "filename.txt"}},
+ {UserEmail: "test@test.com", UploadStartTime: 1,
+ FileInfo: &model.FileInfo{Id: "54321", Name: "test2", Path: "filename2.txt"}},
+ },
+ expectedStops: []*FileUploadStopExport{
+ {UserEmail: "test@test.com", UploadStopTime: 1,
+ FileInfo: &model.FileInfo{Id: "12345", Name: "test", Path: "filename.txt"}, Status: "Completed"},
+ {UserEmail: "test@test.com", UploadStopTime: 1,
+ FileInfo: &model.FileInfo{Id: "54321", Name: "test2", Path: "filename2.txt"}, Status: "Completed"},
+ },
+ expectedFileInfos: []*model.FileInfo{
+ {Name: "test", Id: "12345", Path: "filename.txt"},
+ {Name: "test2", Id: "54321", Path: "filename2.txt"},
+ },
+ expectedDeleteFileMessages: nil,
+ expectError: false,
+ },
+ {
+ name: "one-attachment-deleted",
+ post: model.MessageExport{
+ PostId: model.NewPointer("test"),
+ ChannelId: model.NewPointer("Test"),
+ ChannelDisplayName: model.NewPointer("Test"),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostDeleteAt: model.NewPointer(int64(2)),
+ PostMessage: model.NewPointer("Some message"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("test"),
+ Username: model.NewPointer("test"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{"12345", "54321"},
+ },
+ attachments: []*model.FileInfo{
+ {Name: "test", Id: "12345", Path: "filename.txt", DeleteAt: 2},
+ },
+ expectedStarts: []*FileUploadStartExport{
+ {UserEmail: "test@test.com", UploadStartTime: 1,
+ FileInfo: &model.FileInfo{Id: "12345", Name: "test", Path: "filename.txt", DeleteAt: 2}},
+ },
+ expectedStops: []*FileUploadStopExport{
+ {UserEmail: "test@test.com", UploadStopTime: 1,
+ FileInfo: &model.FileInfo{Id: "12345", Name: "test", Path: "filename.txt", DeleteAt: 2}, Status: "Completed"},
+ },
+ expectedFileInfos: []*model.FileInfo{
+ {Name: "test", Id: "12345", Path: "filename.txt", DeleteAt: 2},
+ },
+ expectedDeleteFileMessages: []PostExport{
+ {
+ UserType: "user",
+ UpdatedType: FileDeleted,
+ UpdateAt: 2,
+ Message: "delete " + "filename.txt",
+ FileInfo: &model.FileInfo{Id: "12345", Name: "test", Path: "filename.txt", DeleteAt: 2},
+ },
+ },
+ expectError: false,
+ },
+ {
+ name: "one-attachment-deleted, ignore it (only record the creation)",
+ post: model.MessageExport{
+ PostId: model.NewPointer("test"),
+ ChannelId: model.NewPointer("Test"),
+ ChannelDisplayName: model.NewPointer("Test"),
+ PostCreateAt: model.NewPointer(int64(1)),
+ PostDeleteAt: model.NewPointer(int64(2)),
+ PostMessage: model.NewPointer("Some message"),
+ UserEmail: model.NewPointer("test@test.com"),
+ UserId: model.NewPointer("test"),
+ Username: model.NewPointer("test"),
+ ChannelType: &chanTypeDirect,
+ PostFileIds: []string{"12345", "54321"},
+ },
+ attachments: []*model.FileInfo{
+ {Name: "test", Id: "12345", Path: "filename.txt", DeleteAt: 2},
+ },
+ expectedStarts: []*FileUploadStartExport{
+ {UserEmail: "test@test.com", UploadStartTime: 1,
+ FileInfo: &model.FileInfo{Name: "test", Id: "12345", Path: "filename.txt", DeleteAt: 2}},
+ },
+ expectedStops: []*FileUploadStopExport{
+ {UserEmail: "test@test.com", UploadStopTime: 1,
+ FileInfo: &model.FileInfo{Name: "test", Id: "12345", Path: "filename.txt", DeleteAt: 2}, Status: "Completed"},
+ },
+ expectedFileInfos: []*model.FileInfo{
+ {Name: "test", Id: "12345", Path: "filename.txt", DeleteAt: 2},
+ },
+ expectedDeleteFileMessages: nil,
+ ignoreDeleted: true,
+ expectError: false,
+ },
+ }
+
+ for _, tc := range tt {
+ t.Run(tc.name, func(t *testing.T) {
+ mockStore := &storetest.Store{}
+ defer mockStore.AssertExpectations(t)
+
+ if len(tc.attachments) > 0 {
+ call := mockStore.FileInfoStore.On("GetForPost", *tc.post.PostId, true, true, false)
+ call.Run(func(args mock.Arguments) {
+ call.Return(tc.attachments, nil)
+ })
+ }
+ files, uploadStarts, uploadStops, deleteFileMessages, err := postToAttachmentsEntries(&tc.post, NewMessageExportStore(mockStore), tc.ignoreDeleted)
+ if tc.expectError {
+ assert.Error(t, err)
+ } else {
+ assert.NoError(t, err)
+ }
+
+ // Need to add the post to the expected starts (to save from copy pasting in the test cases above)
+ for _, s := range tc.expectedStarts {
+ s.MessageExport = tc.post
+ }
+
+ // Need to add the post to the expected stops
+ for _, s := range tc.expectedStops {
+ s.MessageExport = tc.post
+ }
+
+ // Need to add the post to the expected deleted
+ var expectedDeleted []PostExport
+ for _, s := range tc.expectedDeleteFileMessages {
+ s.MessageExport = tc.post
+ expectedDeleted = append(expectedDeleted, s)
+ }
+
+ assert.Equal(t, tc.expectedStarts, uploadStarts)
+ assert.Equal(t, tc.expectedStops, uploadStops)
+ assert.Equal(t, tc.expectedFileInfos, files)
+ assert.Equal(t, expectedDeleted, deleteFileMessages)
+ })
+ }
+}
+
+func TestGetJoinLeavePosts(t *testing.T) {
+ mockStore := &storetest.Store{}
+ defer mockStore.AssertExpectations(t)
+
+ // This would have been retrieved during CalculateChannelExports
+ channelMemberHistories := map[string][]*model.ChannelMemberHistoryResult{
+ "good-request-1": {
+ {JoinTime: 1, UserId: "test1", UserEmail: "test1", Username: "test1"},
+ {JoinTime: 2, LeaveTime: model.NewPointer(int64(3)), UserId: "test2", UserEmail: "test2", Username: "test2"},
+ {JoinTime: 3, UserId: "test3", UserEmail: "test3", Username: "test3"},
+ },
+ "good-request-2": {
+ {JoinTime: 4, UserId: "test4", UserEmail: "test4", Username: "test4"},
+ {JoinTime: 5, LeaveTime: model.NewPointer(int64(6)), UserId: "test5", UserEmail: "test5", Username: "test5"},
+ {JoinTime: 6, UserId: "test6", UserEmail: "test6", Username: "test6"},
+ },
+ }
+
+ var joins []JoinExport
+ var leaves []LeaveExport
+ for _, id := range []string{"good-request-1", "good-request-2"} {
+ newJoins, newLeaves := getJoinsAndLeaves(
+ 1,
+ 7,
+ channelMemberHistories[id],
+ nil,
+ )
+ joins = append(joins, newJoins...)
+ leaves = append(leaves, newLeaves...)
+ }
+
+ assert.Len(t, joins, 6)
+ assert.Equal(t, JoinExport{
+ UserId: "test1",
+ Username: "test1",
+ UserEmail: "test1",
+ UserType: User,
+ JoinTime: 1,
+ LeaveTime: 7,
+ }, joins[0])
+ assert.Equal(t, JoinExport{
+ UserId: "test2",
+ Username: "test2",
+ UserEmail: "test2",
+ UserType: User,
+ JoinTime: 2,
+ LeaveTime: 3,
+ }, joins[1])
+ assert.Equal(t, JoinExport{
+ UserId: "test3",
+ Username: "test3",
+ UserEmail: "test3",
+ UserType: User,
+ JoinTime: 3,
+ LeaveTime: 7,
+ }, joins[2])
+ assert.Equal(t, JoinExport{
+ UserId: "test4",
+ Username: "test4",
+ UserEmail: "test4",
+ UserType: User,
+ JoinTime: 4,
+ LeaveTime: 7,
+ }, joins[3])
+ assert.Equal(t, JoinExport{
+ UserId: "test5",
+ Username: "test5",
+ UserEmail: "test5",
+ UserType: User,
+ JoinTime: 5,
+ LeaveTime: 6,
+ }, joins[4])
+ assert.Equal(t, JoinExport{
+ UserId: "test6",
+ Username: "test6",
+ UserEmail: "test6",
+ UserType: User,
+ JoinTime: 6,
+ LeaveTime: 7,
+ }, joins[5])
+
+ // remember that getJoinsAndLeaves sorts _for each channel_
+ assert.Len(t, leaves, 6)
+ // 1st channel:
+ assert.Equal(t, LeaveExport{
+ UserId: "test2",
+ Username: "test2",
+ UserEmail: "test2",
+ UserType: User,
+ LeaveTime: 3,
+ }, leaves[0])
+ assert.Equal(t, LeaveExport{
+ UserId: "test1",
+ Username: "test1",
+ UserEmail: "test1",
+ UserType: User,
+ LeaveTime: 7,
+ ClosedOut: true,
+ }, leaves[1])
+ assert.Equal(t, LeaveExport{
+ UserId: "test3",
+ Username: "test3",
+ UserEmail: "test3",
+ UserType: User,
+ LeaveTime: 7,
+ ClosedOut: true,
+ }, leaves[2])
+ // 2nd channel:
+ assert.Equal(t, LeaveExport{
+ UserId: "test5",
+ Username: "test5",
+ UserEmail: "test5",
+ UserType: User,
+ LeaveTime: 6,
+ }, leaves[3])
+ assert.Equal(t, LeaveExport{
+ UserId: "test4",
+ Username: "test4",
+ UserEmail: "test4",
+ UserType: User,
+ LeaveTime: 7,
+ ClosedOut: true,
+ }, leaves[4])
+ assert.Equal(t, LeaveExport{
+ UserId: "test6",
+ Username: "test6",
+ UserEmail: "test6",
+ UserType: User,
+ LeaveTime: 7,
+ ClosedOut: true,
+ }, leaves[5])
+}
diff --git a/server/enterprise/message_export/shared/main_test.go b/server/enterprise/message_export/shared/main_test.go
new file mode 100644
index 0000000000..51dc92daa0
--- /dev/null
+++ b/server/enterprise/message_export/shared/main_test.go
@@ -0,0 +1,21 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package shared
+
+import (
+ "testing"
+
+ "github.com/mattermost/mattermost/server/v8/channels/api4"
+ "github.com/mattermost/mattermost/server/v8/channels/testlib"
+)
+
+var mainHelper *testlib.MainHelper
+
+func TestMain(m *testing.M) {
+ mainHelper = testlib.NewMainHelper()
+ defer mainHelper.Close()
+ api4.SetMainHelper(mainHelper)
+
+ mainHelper.Main(m)
+}
diff --git a/server/enterprise/message_export/shared/shared.go b/server/enterprise/message_export/shared/shared.go
new file mode 100644
index 0000000000..ef2de996c2
--- /dev/null
+++ b/server/enterprise/message_export/shared/shared.go
@@ -0,0 +1,609 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package shared
+
+import (
+ "encoding/json"
+ "fmt"
+ "path"
+ "strconv"
+ "time"
+
+ "github.com/pkg/errors"
+
+ "github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+ "github.com/mattermost/mattermost/server/v8/platform/shared/templates"
+
+ "github.com/mattermost/mattermost/server/public/model"
+ "github.com/mattermost/mattermost/server/public/shared/mlog"
+ "github.com/mattermost/mattermost/server/public/shared/request"
+)
+
+const (
+ MissingFileMessage = "File missing for post; cannot copy file to archive"
+
+ EstimatedPostCount = 10_000_000
+
+ // JobDataBatchStartTime is the posts.updateat value from the previous batch. Posts are selected using
+ // keyset pagination sorted by (posts.updateat, posts.id).
+ JobDataBatchStartTime = "batch_start_time"
+
+ // JobDataJobStartTime is the start of the job (doesn't change across batches)
+ JobDataJobStartTime = "job_start_time"
+
+ // JobDataBatchStartId is the posts.id value from the previous batch.
+ JobDataBatchStartId = "batch_start_id"
+
+ // JobDataJobEndTime is the point up to which this job is exporting. It is the time the job was started,
+ // i.e., we export everything from the end of previous batch to the moment this batch started.
+ JobDataJobEndTime = "job_end_time"
+
+ JobDataJobStartId = "job_start_id"
+ JobDataExportType = "export_type"
+ JobDataBatchSize = "batch_size"
+ JobDataChannelBatchSize = "channel_batch_size"
+ JobDataChannelHistoryBatchSize = "channel_history_batch_size"
+ JobDataMessagesExported = "messages_exported"
+ JobDataWarningCount = "warning_count"
+ JobDataIsDownloadable = "is_downloadable"
+ JobDataExportDir = "export_dir"
+ JobDataBatchNumber = "job_batch_number"
+ JobDataTotalPostsExpected = "total_posts_expected"
+)
+
+type PostUpdatedType string
+
+const (
+ EditedOriginalMsg PostUpdatedType = "EditedOriginalMsg"
+ EditedNewMsg PostUpdatedType = "EditedNewMsg"
+ UpdatedNoMsgChange PostUpdatedType = "UpdatedNoMsgChange"
+ Deleted PostUpdatedType = "Deleted"
+ FileDeleted PostUpdatedType = "FileDeleted"
+)
+
+// JobData keeps the current state of the job.
+// When used by a worker, all fields in JobDataExported are exported to the job's job.Data prop bag.
+type JobData struct {
+ JobDataExported
+
+ ExportPeriodStartTime int64
+
+ // This section is the current state of the export
+ ChannelMetadata map[string]*MetadataChannel
+ ChannelMemberHistories map[string][]*model.ChannelMemberHistoryResult
+ Cursor model.MessageExportCursor
+ PostsToExport []*model.MessageExport
+ BatchEndTime int64
+ BatchPath string
+ MessageExportMs []int64
+ ProcessingPostsMs []int64
+ ProcessingXmlMs []int64
+ TransferringFilesMs []int64
+ TransferringZipMs []int64
+ TotalBatchMs []int64
+ Finished bool
+}
+
+type JobDataExported struct {
+ ExportType string
+ ExportDir string
+ BatchStartTime int64
+ BatchStartId string
+ JobStartTime int64
+ JobEndTime int64
+ JobStartId string
+ BatchSize int
+ ChannelBatchSize int
+ ChannelHistoryBatchSize int
+ BatchNumber int
+ TotalPostsExpected int
+ MessagesExported int
+ WarningCount int
+ IsDownloadable bool
+}
+
+func JobDataToStringMap(jd JobData) map[string]string {
+ ret := make(map[string]string)
+ ret[JobDataExportType] = jd.ExportType
+ ret[JobDataExportDir] = jd.ExportDir
+ ret[JobDataBatchStartTime] = strconv.FormatInt(jd.BatchStartTime, 10)
+ ret[JobDataBatchStartId] = jd.BatchStartId
+ ret[JobDataJobStartTime] = strconv.FormatInt(jd.JobStartTime, 10)
+ ret[JobDataJobEndTime] = strconv.FormatInt(jd.JobEndTime, 10)
+ ret[JobDataJobStartId] = jd.JobStartId
+ ret[JobDataBatchSize] = strconv.Itoa(jd.BatchSize)
+ ret[JobDataChannelBatchSize] = strconv.Itoa(jd.ChannelBatchSize)
+ ret[JobDataChannelHistoryBatchSize] = strconv.Itoa(jd.ChannelHistoryBatchSize)
+ ret[JobDataBatchNumber] = strconv.Itoa(jd.BatchNumber)
+ ret[JobDataTotalPostsExpected] = strconv.Itoa(jd.TotalPostsExpected)
+ ret[JobDataMessagesExported] = strconv.Itoa(jd.MessagesExported)
+ ret[JobDataWarningCount] = strconv.Itoa(jd.WarningCount)
+ ret[JobDataIsDownloadable] = strconv.FormatBool(jd.IsDownloadable)
+ return ret
+}
+
+func StringMapToJobDataWithZeroValues(sm map[string]string) (JobData, error) {
+ var jd JobData
+ var err error
+
+ jd.ExportType = sm[JobDataExportType]
+ jd.ExportDir = sm[JobDataExportDir]
+
+ batchStartTime, ok := sm[JobDataBatchStartTime]
+ if !ok {
+ batchStartTime = "0"
+ }
+ if jd.BatchStartTime, err = strconv.ParseInt(batchStartTime, 10, 64); err != nil {
+ return jd, errors.Wrap(err, "error converting JobDataBatchStartTime")
+ }
+
+ jd.BatchStartId = sm[JobDataBatchStartId]
+
+ jobStartTime, ok := sm[JobDataJobStartTime]
+ if !ok {
+ jobStartTime = "0"
+ }
+ if jd.JobStartTime, err = strconv.ParseInt(jobStartTime, 10, 64); err != nil {
+ return jd, errors.Wrap(err, "error converting JobDataJobStartTime")
+ }
+
+ jobEndTime, ok := sm[JobDataJobEndTime]
+ if !ok {
+ jobEndTime = "0"
+ }
+ if jd.JobEndTime, err = strconv.ParseInt(jobEndTime, 10, 64); err != nil {
+ return jd, errors.Wrap(err, "error converting JobDataJobEndTime")
+ }
+
+ jd.JobStartId = sm[JobDataJobStartId]
+
+ jobBatchSize, ok := sm[JobDataBatchSize]
+ if !ok {
+ jobBatchSize = "0"
+ }
+ if jd.BatchSize, err = strconv.Atoi(jobBatchSize); err != nil {
+ return jd, errors.Wrap(err, "error converting JobDataBatchSize")
+ }
+
+ channelBatchSize, ok := sm[JobDataChannelBatchSize]
+ if !ok {
+ channelBatchSize = "0"
+ }
+ if jd.ChannelBatchSize, err = strconv.Atoi(channelBatchSize); err != nil {
+ return jd, errors.Wrap(err, "error converting JobDataChannelBatchSize")
+ }
+
+ channelHistoryBatchSize, ok := sm[JobDataChannelHistoryBatchSize]
+ if !ok {
+ channelHistoryBatchSize = "0"
+ }
+ if jd.ChannelHistoryBatchSize, err = strconv.Atoi(channelHistoryBatchSize); err != nil {
+ return jd, errors.Wrap(err, "error converting JobDataChannelHistoryBatchSize")
+ }
+
+ batchNumber, ok := sm[JobDataBatchNumber]
+ if !ok {
+ batchNumber = "0"
+ }
+ if jd.BatchNumber, err = strconv.Atoi(batchNumber); err != nil {
+ return jd, errors.Wrap(err, "error converting JobDataBatchNumber")
+ }
+
+ totalPostsExpected, ok := sm[JobDataTotalPostsExpected]
+ if !ok {
+ totalPostsExpected = "0"
+ }
+ if jd.TotalPostsExpected, err = strconv.Atoi(totalPostsExpected); err != nil {
+ return jd, errors.Wrap(err, "error converting JobDataTotalPostsExpected")
+ }
+
+ messagesExported, ok := sm[JobDataMessagesExported]
+ if !ok {
+ messagesExported = "0"
+ }
+ if jd.MessagesExported, err = strconv.Atoi(messagesExported); err != nil {
+ return jd, errors.Wrap(err, "error converting JobDataMessagesExported")
+ }
+
+ warningCount, ok := sm[JobDataWarningCount]
+ if !ok {
+ warningCount = "0"
+ }
+ if jd.WarningCount, err = strconv.Atoi(warningCount); err != nil {
+ return jd, errors.Wrap(err, "error converting JobDataWarningCount")
+ }
+
+ isDownloadable, ok := sm[JobDataIsDownloadable]
+ if !ok {
+ isDownloadable = "0"
+ }
+ if jd.IsDownloadable, err = strconv.ParseBool(isDownloadable); err != nil {
+ return jd, errors.Wrap(err, "error converting JobDataIsDownloadable")
+ }
+
+ return jd, nil
+}
+
+type BackendParams struct {
+ Config *model.Config
+ Store MessageExportStore
+ FileAttachmentBackend filestore.FileBackend
+ ExportBackend filestore.FileBackend
+ HtmlTemplates *templates.Container
+}
+
+type ExportParams struct {
+ ExportType string
+ ChannelMetadata map[string]*MetadataChannel
+ Posts []*model.MessageExport
+ ChannelMemberHistories map[string][]*model.ChannelMemberHistoryResult
+ JobStartTime int64
+ BatchPath string
+ BatchStartTime int64
+ BatchEndTime int64
+ Config *model.Config
+ Db MessageExportStore
+ FileAttachmentBackend filestore.FileBackend
+ ExportBackend filestore.FileBackend
+ Templates *templates.Container
+}
+
+type WriteExportResult struct {
+ TransferringFilesMs int64
+ ProcessingXmlMs int64
+ TransferringZipMs int64
+ NumWarnings int
+}
+
+type RunExportResults struct {
+ CreatedPosts int
+ EditedOrigMsgPosts int
+ EditedNewMsgPosts int
+ UpdatedPosts int
+ DeletedPosts int
+ UploadedFiles int
+ DeletedFiles int
+ NumChannels int
+ Joins int
+ Leaves int
+ ProcessingPostsMs int64
+ WriteExportResult
+}
+
+type ChannelMemberJoin struct {
+ UserId string
+ IsBot bool
+ Email string
+ Username string
+ Datetime int64
+}
+
+type ChannelMemberLeave struct {
+ UserId string
+ IsBot bool
+ Email string
+ Username string
+ Datetime int64
+}
+
+type ChannelMember struct {
+ UserId string
+ IsBot bool
+ Email string
+ Username string
+}
+
+type MetadataChannel struct {
+ TeamId *string
+ TeamName *string
+ TeamDisplayName *string
+ ChannelId string
+ ChannelName string
+ ChannelDisplayName string
+ ChannelType model.ChannelType
+ RoomId string
+ StartTime int64
+ EndTime int64
+ MessagesCount int
+ AttachmentsCount int
+}
+
+type Metadata struct {
+ Channels map[string]*MetadataChannel
+ MessagesCount int
+ AttachmentsCount int
+ StartTime int64
+ EndTime int64
+}
+
+func (metadata *Metadata) UpdateCounts(channelId string, numMessages int, numAttachments int) error {
+ _, ok := metadata.Channels[channelId]
+ if !ok {
+ return fmt.Errorf("could not find channelId for post in metadata.Channels")
+ }
+
+ metadata.Channels[channelId].AttachmentsCount += numAttachments
+ metadata.AttachmentsCount += numAttachments
+ metadata.Channels[channelId].MessagesCount += numMessages
+ metadata.MessagesCount += numMessages
+
+ return nil
+}
+
+// GetInitialExportPeriodData calculates and caches the channel memberships, channel metadata, and the TotalPostsExpected.
+func GetInitialExportPeriodData(rctx request.CTX, store MessageExportStore, data JobData, reportProgress func(string)) (JobData, error) {
+ // Counting all posts may fail or timeout when the posts table is large. If this happens, log a warning, but carry
+ // on with the job anyway. The only issue is that the progress % reporting will be inaccurate.
+ count, err := store.Post().AnalyticsPostCount(&model.PostCountOptions{ExcludeSystemPosts: true, SincePostID: data.JobStartId, SinceUpdateAt: data.ExportPeriodStartTime, UntilUpdateAt: data.JobEndTime})
+ if err != nil {
+ rctx.Logger().Warn("Worker: Failed to fetch total post count for job. An estimated value will be used for progress reporting.", mlog.Err(err))
+ data.TotalPostsExpected = EstimatedPostCount
+ } else {
+ data.TotalPostsExpected = int(count)
+ }
+
+ rctx.Logger().Info("Expecting to export total posts", mlog.Int("total_posts", data.TotalPostsExpected))
+
+ // Every time we claim the job, we need to gather the membership data that every batch will use.
+ // If we're here, then either this is the start of the job, or the job was stopped (e.g., the worker stopped)
+ // and we've claimed it again. Either way, we need to recalculate channel and member history data.
+ data.ChannelMetadata, data.ChannelMemberHistories, err = CalculateChannelExports(rctx,
+ ChannelExportsParams{
+ Store: store,
+ ExportPeriodStartTime: data.ExportPeriodStartTime,
+ ExportPeriodEndTime: data.JobEndTime,
+ ChannelBatchSize: data.ChannelBatchSize,
+ ChannelHistoryBatchSize: data.ChannelHistoryBatchSize,
+ ReportProgressMessage: reportProgress,
+ })
+ if err != nil {
+ return data, err
+ }
+
+ data.Cursor = model.MessageExportCursor{
+ LastPostUpdateAt: data.BatchStartTime,
+ LastPostId: data.BatchStartId,
+ UntilUpdateAt: data.JobEndTime,
+ }
+
+ return data, nil
+}
+
+type ChannelExportsParams struct {
+ Store MessageExportStore
+ ExportPeriodStartTime int64
+ ExportPeriodEndTime int64
+ ChannelBatchSize int
+ ChannelHistoryBatchSize int
+ ReportProgressMessage func(message string)
+}
+
+// CalculateChannelExports returns the channel info ( map[channelId]*MetadataChannel ) and the channel user
+// joins/leaves ( map[channelId][]*model.ChannelMemberHistoryResult ) for any channel that has had activity
+// (posts or user join/leaves) between ExportPeriodStartTime and ExportPeriodEndTime.
+func CalculateChannelExports(rctx request.CTX, opt ChannelExportsParams) (map[string]*MetadataChannel, map[string][]*model.ChannelMemberHistoryResult, error) {
+ // Which channels had user activity in the export period?
+ activeChannelIds, err := opt.Store.ChannelMemberHistory().GetChannelsWithActivityDuring(opt.ExportPeriodStartTime, opt.ExportPeriodEndTime)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ if len(activeChannelIds) == 0 {
+ return nil, nil, nil
+ }
+
+ rctx.Logger().Debug("Started CalculateChannelExports", mlog.Int("export_period_start_time", opt.ExportPeriodStartTime), mlog.Int("export_period_end_time", opt.ExportPeriodEndTime), mlog.Int("num_active_channel_ids", len(activeChannelIds)))
+ message := rctx.T("ent.message_export.actiance_export.calculate_channel_exports.channel_message", model.StringMap{"NumChannels": strconv.Itoa(len(activeChannelIds))})
+ opt.ReportProgressMessage(message)
+
+ // For each channel, get its metadata.
+ channelMetadata := make(map[string]*MetadataChannel, len(activeChannelIds))
+
+ // Use batches to reduce db load and network waste.
+ for pos := 0; pos < len(activeChannelIds); pos += opt.ChannelBatchSize {
+ upTo := min(pos+opt.ChannelBatchSize, len(activeChannelIds))
+ batch := activeChannelIds[pos:upTo]
+ channels, err := opt.Store.Channel().GetMany(batch, true)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ for _, channel := range channels {
+ channelMetadata[channel.Id] = &MetadataChannel{
+ TeamId: model.NewPointer(channel.TeamId),
+ ChannelId: channel.Id,
+ ChannelName: channel.Name,
+ ChannelDisplayName: channel.DisplayName,
+ ChannelType: channel.Type,
+ RoomId: fmt.Sprintf("%v - %v", ChannelTypeDisplayName(channel.Type), channel.Id),
+ StartTime: opt.ExportPeriodStartTime,
+ EndTime: opt.ExportPeriodEndTime,
+ }
+ }
+ }
+
+ historiesByChannelId := make(map[string][]*model.ChannelMemberHistoryResult, len(activeChannelIds))
+
+ var batchTimes []int64
+
+ // Now that we have metadata, get channelMemberHistories for each channel.
+ // Use batches to reduce total db load and network waste.
+ for pos := 0; pos < len(activeChannelIds); pos += opt.ChannelHistoryBatchSize {
+ // This may take a while, so update the system console UI.
+ message := rctx.T("ent.message_export.actiance_export.calculate_channel_exports.activity_message", model.StringMap{
+ "NumChannels": strconv.Itoa(len(activeChannelIds)),
+ "NumCompleted": strconv.Itoa(pos),
+ })
+ opt.ReportProgressMessage(message)
+
+ start := time.Now()
+
+ upTo := min(pos+opt.ChannelHistoryBatchSize, len(activeChannelIds))
+ batch := activeChannelIds[pos:upTo]
+ channelMemberHistories, err := opt.Store.ChannelMemberHistory().GetUsersInChannelDuring(opt.ExportPeriodStartTime, opt.ExportPeriodEndTime, batch)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ batchTimes = append(batchTimes, time.Since(start).Milliseconds())
+
+ // collect the channelMemberHistories by channelId
+ for _, entry := range channelMemberHistories {
+ historiesByChannelId[entry.ChannelId] = append(historiesByChannelId[entry.ChannelId], entry)
+ }
+ }
+
+ rctx.Logger().Info("GetUsersInChannelDuring batch times", mlog.Array("batch_times", batchTimes))
+
+ return channelMetadata, historiesByChannelId, nil
+}
+
+// ChannelHasActivity returns true if the channel (represented by the []*model.ChannelMemberHistoryResult slice)
+// had user activity between startTime and endTime
+func ChannelHasActivity(cmhs []*model.ChannelMemberHistoryResult, startTime int64, endTime int64) bool {
+ for _, cmh := range cmhs {
+ if (cmh.JoinTime >= startTime && cmh.JoinTime <= endTime) ||
+ (cmh.LeaveTime != nil && *cmh.LeaveTime >= startTime && *cmh.LeaveTime <= endTime) {
+ return true
+ }
+ }
+ return false
+}
+
+func GetJoinsAndLeavesForChannel(startTime int64, endTime int64, channelMembersHistory []*model.ChannelMemberHistoryResult,
+ postAuthors map[string]ChannelMember) ([]ChannelMemberJoin, []ChannelMemberLeave) {
+ var joins []ChannelMemberJoin
+ var leaves []ChannelMemberLeave
+
+ alreadyJoined := make(map[string]bool)
+ for _, cmh := range channelMembersHistory {
+ if cmh.UserDeleteAt > 0 && cmh.UserDeleteAt < startTime {
+ continue
+ }
+
+ if cmh.JoinTime > endTime {
+ continue
+ }
+
+ if cmh.LeaveTime != nil && *cmh.LeaveTime < startTime {
+ continue
+ }
+
+ if cmh.JoinTime <= endTime {
+ joins = append(joins, ChannelMemberJoin{
+ UserId: cmh.UserId,
+ IsBot: cmh.IsBot,
+ Email: cmh.UserEmail,
+ Username: cmh.Username,
+ Datetime: cmh.JoinTime,
+ })
+ alreadyJoined[cmh.UserId] = true
+ }
+
+ if cmh.LeaveTime != nil && *cmh.LeaveTime <= endTime {
+ leaves = append(leaves, ChannelMemberLeave{
+ UserId: cmh.UserId,
+ IsBot: cmh.IsBot,
+ Email: cmh.UserEmail,
+ Username: cmh.Username,
+ Datetime: *cmh.LeaveTime,
+ })
+ }
+ }
+
+ for _, member := range postAuthors {
+ if alreadyJoined[member.UserId] {
+ continue
+ }
+
+ joins = append(joins, ChannelMemberJoin{
+ UserId: member.UserId,
+ IsBot: member.IsBot,
+ Email: member.Email,
+ Username: member.Username,
+ Datetime: startTime,
+ })
+ }
+ return joins, leaves
+}
+
+// GetPostAttachments if the post included any files, we need to add special elements to the export.
+func GetPostAttachments(db MessageExportStore, post *model.MessageExport) ([]*model.FileInfo, error) {
+ if len(post.PostFileIds) == 0 {
+ return []*model.FileInfo{}, nil
+ }
+
+ attachments, err := db.FileInfo().GetForPost(*post.PostId, true, true, false)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get file info for a post: %w", err)
+ }
+ return attachments, nil
+}
+
+func ChannelTypeDisplayName(channelType model.ChannelType) string {
+ return map[model.ChannelType]string{
+ model.ChannelTypeOpen: "public",
+ model.ChannelTypePrivate: "private",
+ model.ChannelTypeDirect: "direct",
+ model.ChannelTypeGroup: "group",
+ }[channelType]
+}
+
+func GetBatchPath(exportDir string, prevPostUpdateAt int64, lastPostUpdateAt int64, batchNumber int) string {
+ if exportDir == "" {
+ exportDir = path.Join(model.ComplianceExportPath, time.Now().Format(model.ComplianceExportDirectoryFormat))
+ }
+ return path.Join(exportDir,
+ fmt.Sprintf("batch%03d-%d-%d.zip", batchNumber, prevPostUpdateAt, lastPostUpdateAt))
+}
+
+// GetExportBackend returns the file backend where the export will be created.
+func GetExportBackend(rctx request.CTX, config *model.Config) (filestore.FileBackend, error) {
+ insecure := config.ServiceSettings.EnableInsecureOutgoingConnections
+ skipVerify := insecure != nil && *insecure
+
+ if config.FileSettings.DedicatedExportStore != nil && *config.FileSettings.DedicatedExportStore {
+ rctx.Logger().Debug("Worker: using dedicated export filestore", mlog.String("driver_name", *config.FileSettings.ExportDriverName))
+ backend, errFileBack := filestore.NewExportFileBackend(filestore.NewExportFileBackendSettingsFromConfig(&config.FileSettings, true, skipVerify))
+ if errFileBack != nil {
+ return nil, errFileBack
+ }
+
+ return backend, nil
+ }
+
+ backend, err := filestore.NewFileBackend(filestore.NewFileBackendSettingsFromConfig(&config.FileSettings, true, skipVerify))
+ if err != nil {
+ return nil, err
+ }
+ return backend, nil
+}
+
+// GetFileAttachmentBackend returns the file backend where file attachments are
+// located for messages that will be exported. This may be the same backend
+// where the export will be created.
+func GetFileAttachmentBackend(rctx request.CTX, config *model.Config) (filestore.FileBackend, error) {
+ insecure := config.ServiceSettings.EnableInsecureOutgoingConnections
+
+ backend, err := filestore.NewFileBackend(filestore.NewFileBackendSettingsFromConfig(&config.FileSettings, true, insecure != nil && *insecure))
+ if err != nil {
+ return nil, err
+ }
+ return backend, nil
+}
+
+func IsDeletedMsg(post *model.MessageExport) bool {
+ if model.SafeDereference(post.PostDeleteAt) > 0 && post.PostProps != nil {
+ props := map[string]any{}
+ err := json.Unmarshal([]byte(*post.PostProps), &props)
+ if err != nil {
+ return false
+ }
+
+ if _, ok := props[model.PostPropsDeleteBy]; ok {
+ return true
+ }
+ }
+ return false
+}
diff --git a/server/enterprise/message_export/shared/shared_test.go b/server/enterprise/message_export/shared/shared_test.go
new file mode 100644
index 0000000000..7e7761eda5
--- /dev/null
+++ b/server/enterprise/message_export/shared/shared_test.go
@@ -0,0 +1,286 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package shared
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/mattermost/mattermost/server/public/model"
+)
+
+func TestGetJoinsAndLeavesForChannel(t *testing.T) {
+ channel := MetadataChannel{
+ StartTime: 100,
+ EndTime: 200,
+ ChannelId: "good-request-1",
+ TeamId: model.NewPointer("test"),
+ TeamName: model.NewPointer("test"),
+ TeamDisplayName: model.NewPointer("test"),
+ ChannelName: "test",
+ ChannelDisplayName: "test",
+ ChannelType: "O",
+ }
+
+ tt := []struct {
+ name string
+ channel MetadataChannel
+ membersHistory []*model.ChannelMemberHistoryResult
+ usersInPosts map[string]ChannelMember
+ expectedJoins int
+ expectedLeaves int
+ }{
+ {
+ name: "no-joins-no-leaves",
+ channel: channel,
+ membersHistory: nil,
+ usersInPosts: nil,
+ expectedJoins: 0,
+ expectedLeaves: 0,
+ },
+ {
+ name: "joins-and-leaves-outside-the-range",
+ channel: channel,
+ membersHistory: []*model.ChannelMemberHistoryResult{
+ {JoinTime: 1, LeaveTime: model.NewPointer(int64(10)), UserId: "test", UserEmail: "test", Username: "test"},
+ {JoinTime: 250, LeaveTime: model.NewPointer(int64(260)), UserId: "test", UserEmail: "test", Username: "test"},
+ {JoinTime: 300, UserId: "test", UserEmail: "test", Username: "test"},
+ },
+ usersInPosts: nil,
+ expectedJoins: 0,
+ expectedLeaves: 0,
+ },
+ {
+ name: "join-and-leave-during-the-range",
+ channel: channel,
+ membersHistory: []*model.ChannelMemberHistoryResult{
+ {JoinTime: 100, LeaveTime: model.NewPointer(int64(150)), UserId: "test", UserEmail: "test", Username: "test"},
+ },
+ usersInPosts: nil,
+ expectedJoins: 1,
+ expectedLeaves: 1,
+ },
+ {
+ name: "join-during-and-leave-after-the-range",
+ channel: channel,
+ membersHistory: []*model.ChannelMemberHistoryResult{
+ {JoinTime: 150, LeaveTime: model.NewPointer(int64(300)), UserId: "test", UserEmail: "test", Username: "test"},
+ },
+ usersInPosts: nil,
+ expectedJoins: 1,
+ expectedLeaves: 0,
+ },
+ {
+ name: "join-before-and-leave-during-the-range",
+ channel: channel,
+ membersHistory: []*model.ChannelMemberHistoryResult{
+ {JoinTime: 99, LeaveTime: model.NewPointer(int64(150)), UserId: "test", UserEmail: "test", Username: "test"},
+ },
+ usersInPosts: nil,
+ expectedJoins: 1,
+ expectedLeaves: 1,
+ },
+ {
+ name: "join-before-and-leave-after-the-range",
+ channel: channel,
+ membersHistory: []*model.ChannelMemberHistoryResult{
+ {JoinTime: 99, LeaveTime: model.NewPointer(int64(350)), UserId: "test", UserEmail: "test", Username: "test"},
+ },
+ usersInPosts: nil,
+ expectedJoins: 1,
+ expectedLeaves: 0,
+ },
+ {
+ name: "implicit-joins",
+ channel: channel,
+ membersHistory: nil,
+ usersInPosts: map[string]ChannelMember{
+ "test1": {UserId: "test1", Email: "test1", Username: "test1"},
+ "test2": {UserId: "test2", Email: "test2", Username: "test2"},
+ },
+ expectedJoins: 2,
+ expectedLeaves: 0,
+ },
+ {
+ name: "implicit-joins-with-explicit-joins",
+ channel: channel,
+ membersHistory: []*model.ChannelMemberHistoryResult{
+ {JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test1", UserEmail: "test1", Username: "test1"},
+ {JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test3", UserEmail: "test3", Username: "test3"},
+ },
+ usersInPosts: map[string]ChannelMember{
+ "test1": {UserId: "test1", Email: "test1", Username: "test1"},
+ "test2": {UserId: "test2", Email: "test2", Username: "test2"},
+ },
+ expectedJoins: 3,
+ expectedLeaves: 2,
+ },
+ {
+ name: "join-leave-and-join-again",
+ channel: channel,
+ membersHistory: []*model.ChannelMemberHistoryResult{
+ {JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test1", UserEmail: "test1", Username: "test1"},
+ {JoinTime: 160, LeaveTime: model.NewPointer(int64(180)), UserId: "test1", UserEmail: "test1", Username: "test1"},
+ },
+ usersInPosts: nil,
+ expectedJoins: 2,
+ expectedLeaves: 2,
+ },
+ {
+ name: "deactivated-members-dont-show",
+ channel: channel,
+ membersHistory: []*model.ChannelMemberHistoryResult{
+ {JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 50},
+ {JoinTime: 160, LeaveTime: model.NewPointer(int64(180)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 50},
+ },
+ usersInPosts: nil,
+ expectedJoins: 0,
+ expectedLeaves: 0,
+ },
+ {
+ name: "deactivated-members-show-if-deleted-after-latest-export",
+ channel: channel,
+ membersHistory: []*model.ChannelMemberHistoryResult{
+ {JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 150},
+ {JoinTime: 160, LeaveTime: model.NewPointer(int64(180)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 150},
+ },
+ usersInPosts: nil,
+ expectedJoins: 2,
+ expectedLeaves: 2,
+ },
+ {
+ name: "deactivated-members-show-and-dont-show",
+ channel: channel,
+ membersHistory: []*model.ChannelMemberHistoryResult{
+ {JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 50},
+ {JoinTime: 160, LeaveTime: model.NewPointer(int64(180)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 150},
+ },
+ usersInPosts: nil,
+ expectedJoins: 1,
+ expectedLeaves: 1,
+ },
+ }
+
+ for _, tc := range tt {
+ t.Run(tc.name, func(t *testing.T) {
+ joins, leaves := GetJoinsAndLeavesForChannel(tc.channel.StartTime, tc.channel.EndTime, tc.membersHistory, tc.usersInPosts)
+ assert.Len(t, joins, tc.expectedJoins)
+ assert.Len(t, leaves, tc.expectedLeaves)
+ })
+ }
+}
+
+func Test_GetBatchPath(t *testing.T) {
+ tests := []struct {
+ name string
+ exportDir string
+ prevPostUpdateAt int64
+ lastPostUpdateAt int64
+ batchNumber int
+ want string
+ }{
+ {
+ name: "all args given",
+ exportDir: "/export/test_dir",
+ prevPostUpdateAt: 123,
+ lastPostUpdateAt: 456,
+ batchNumber: 21,
+ want: "/export/test_dir/batch021-123-456.zip",
+ },
+ {
+ name: "exportDir blank",
+ exportDir: "",
+ prevPostUpdateAt: 12345,
+ lastPostUpdateAt: 456789,
+ batchNumber: 921,
+ want: model.ComplianceExportPath + "/" + time.Now().Format(model.ComplianceExportDirectoryFormat) + "/batch921-12345-456789.zip",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ assert.Equalf(t, tt.want, GetBatchPath(tt.exportDir, tt.prevPostUpdateAt, tt.lastPostUpdateAt, tt.batchNumber), "GetBatchPath(%v, %v, %v, %v)", tt.exportDir, tt.prevPostUpdateAt, tt.lastPostUpdateAt, tt.batchNumber)
+ })
+ }
+}
+
+func TestJobDataToStringMap_and_StringMapToJobData(t *testing.T) {
+ jd := JobData{
+ JobDataExported: JobDataExported{
+ ExportType: "cli_message_export",
+ ExportDir: "/here/there/34234-123",
+ BatchStartTime: 45,
+ BatchStartId: "34arsitenaorsten",
+ JobStartTime: 99,
+ JobEndTime: 1234,
+ JobStartId: "99abcdef34",
+ BatchSize: 2000,
+ ChannelBatchSize: 30000,
+ ChannelHistoryBatchSize: 30,
+ BatchNumber: 4,
+ TotalPostsExpected: 999999,
+ MessagesExported: 343499,
+ WarningCount: 39,
+ },
+ ExportPeriodStartTime: 123456, // not exported
+ BatchEndTime: 999999999, // not exported
+ BatchPath: "/another/path/123-123", // not exported
+ }
+
+ strMap := JobDataToStringMap(jd)
+
+ expected := make(map[string]string)
+ expected[JobDataExportType] = "cli_message_export"
+ expected[JobDataExportDir] = "/here/there/34234-123"
+ expected[JobDataBatchStartTime] = "45"
+ expected[JobDataBatchStartId] = "34arsitenaorsten"
+ expected[JobDataJobStartTime] = "99"
+ expected[JobDataJobEndTime] = "1234"
+ expected[JobDataJobStartId] = "99abcdef34"
+ expected[JobDataBatchSize] = "2000"
+ expected[JobDataChannelBatchSize] = "30000"
+ expected[JobDataChannelHistoryBatchSize] = "30"
+ expected[JobDataBatchNumber] = "4"
+ expected[JobDataTotalPostsExpected] = "999999"
+ expected[JobDataMessagesExported] = "343499"
+ expected[JobDataWarningCount] = "39"
+ expected[JobDataIsDownloadable] = "false"
+
+ for k, v := range expected {
+ val, ok := strMap[k]
+ assert.True(t, ok)
+ assert.Equal(t, v, val)
+ }
+
+ // not exported:
+ for _, k := range []string{"export_period_start_time", "batch_end_time", "batch_path"} {
+ _, ok := strMap[k]
+ assert.False(t, ok)
+ }
+
+ // zero the fields that weren't exported:
+ jd.ExportPeriodStartTime = 0
+ jd.BatchEndTime = 0
+ jd.BatchPath = ""
+
+ // now convert back
+ jd2, err := StringMapToJobDataWithZeroValues(strMap)
+ assert.NoError(t, err)
+ assert.Equal(t, jd, jd2)
+
+ // and test bad conversion (just a couple):
+ badStrMap := map[string]string{JobDataJobStartTime: "56aaa"}
+ _, err = StringMapToJobDataWithZeroValues(badStrMap)
+ assert.Error(t, err)
+ badStrMap = map[string]string{JobDataJobEndTime: "blah blah"}
+ _, err = StringMapToJobDataWithZeroValues(badStrMap)
+ assert.Error(t, err)
+
+ // test that zero values are used when not present
+ emptyStrMap := make(map[string]string)
+ emptyJd, err := StringMapToJobDataWithZeroValues(emptyStrMap)
+ assert.NoError(t, err)
+ assert.Equal(t, JobData{}, emptyJd)
+}
diff --git a/server/enterprise/message_export/shared/store.go b/server/enterprise/message_export/shared/store.go
new file mode 100644
index 0000000000..9855491fb5
--- /dev/null
+++ b/server/enterprise/message_export/shared/store.go
@@ -0,0 +1,36 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package shared
+
+import (
+ "github.com/mattermost/mattermost/server/public/model"
+ "github.com/mattermost/mattermost/server/v8/channels/store"
+)
+
+type MessageExportStore interface {
+ Post() store.PostStore
+ ChannelMemberHistory() store.ChannelMemberHistoryStore
+ Channel() store.ChannelStore
+ Compliance() store.ComplianceStore
+ FileInfo() MEFileInfoStore
+}
+
+type MEFileInfoStore interface {
+ GetForPost(postID string, readFromMaster, includeDeleted, allowFromCache bool) ([]*model.FileInfo, error)
+}
+
+type messageExportStore struct {
+ store.Store
+}
+
+// NewMessageExportStore returns a wrapped store.Store. Why? Because the CLI tool needs to "override" the GetForPost
+// method in FileStore. Non-CLI code only needs to use the store.Store's methods. This can be removed when we deprecate
+// the CLI tool. Tracked at MM-61139.
+func NewMessageExportStore(s store.Store) messageExportStore {
+ return messageExportStore{s}
+}
+
+func (ss messageExportStore) FileInfo() MEFileInfoStore {
+ return ss.Store.FileInfo()
+}
diff --git a/server/enterprise/message_export/testdata/actianceE2E1Batch1ch2.tmpl b/server/enterprise/message_export/testdata/actianceE2E1Batch1ch2.tmpl
new file mode 100644
index 0000000000..78c9bbae2f
--- /dev/null
+++ b/server/enterprise/message_export/testdata/actianceE2E1Batch1ch2.tmpl
@@ -0,0 +1,101 @@
+
+ private - channel_two_name - %s
+ %d
+
+ user1@email
+ user
+ %d
+ user1@email
+
+
+ user2@email
+ user
+ %d
+ user2@email
+
+
+ user3@email
+ user
+ %d
+ user3@email
+
+
+ %s
+ user1@email
+ user
+ %d
+ message 0
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_0.txt
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_0.txt
+ Completed
+
+
+ %s
+ user1@email
+ user
+ %d
+ message 1
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_1.txt
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_1.txt
+ Completed
+
+
+ %s
+ user1@email
+ user
+ %d
+ message 2
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_2.txt
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_2.txt
+ Completed
+
+
+ user2@email
+ user
+ %d
+ user2@email
+
+
+ user3@email
+ user
+ %d
+ user3@email
+
+
+ user1@email
+ user
+ %d
+ user1@email
+
+ %d
+
diff --git a/server/enterprise/message_export/testdata/actianceE2E1Batch1ch3.tmpl b/server/enterprise/message_export/testdata/actianceE2E1Batch1ch3.tmpl
new file mode 100644
index 0000000000..2a246c4690
--- /dev/null
+++ b/server/enterprise/message_export/testdata/actianceE2E1Batch1ch3.tmpl
@@ -0,0 +1,17 @@
+
+ public - channel_three_name - %s
+ %d
+
+ user7@email
+ user
+ %d
+ user7@email
+
+
+ user7@email
+ user
+ %d
+ user7@email
+
+ %d
+
diff --git a/server/enterprise/message_export/testdata/actianceE2E1Batch2.tmpl b/server/enterprise/message_export/testdata/actianceE2E1Batch2.tmpl
new file mode 100644
index 0000000000..696344dfc3
--- /dev/null
+++ b/server/enterprise/message_export/testdata/actianceE2E1Batch2.tmpl
@@ -0,0 +1,92 @@
+
+
+
+ private - channel_two_name - %s
+ %d
+
+ user1@email
+ user
+ %d
+ user1@email
+
+
+ user4@email
+ user
+ %d
+ user4@email
+
+
+ %s
+ user1@email
+ user
+ %d
+ message 3
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_3.txt
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_3.txt
+ Completed
+
+
+ %s
+ user1@email
+ user
+ %d
+ message 4
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_4.txt
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_4.txt
+ Completed
+
+
+ %s
+ user1@email
+ user
+ %d
+ message 5
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_5.txt
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_5.txt
+ Completed
+
+
+ user4@email
+ user
+ %d
+ user4@email
+
+
+ user1@email
+ user
+ %d
+ user1@email
+
+ %d
+
+
diff --git a/server/enterprise/message_export/testdata/actianceE2E1Batch3ch2.tmpl b/server/enterprise/message_export/testdata/actianceE2E1Batch3ch2.tmpl
new file mode 100644
index 0000000000..a3050015e2
--- /dev/null
+++ b/server/enterprise/message_export/testdata/actianceE2E1Batch3ch2.tmpl
@@ -0,0 +1,101 @@
+
+ private - channel_two_name - %s
+ %d
+
+ user1@email
+ user
+ %d
+ user1@email
+
+
+ user5@email
+ user
+ %d
+ user5@email
+
+
+ user6@email
+ user
+ %d
+ user6@email
+
+
+ %s
+ user1@email
+ user
+ %d
+ message 6
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_6.txt
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_6.txt
+ Completed
+
+
+ %s
+ user1@email
+ user
+ %d
+ message 7
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_7.txt
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_7.txt
+ Completed
+
+
+ %s
+ user1@email
+ user
+ %d
+ message 8
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_8.txt
+
+
+ user1@email
+ %d
+
+ path/to/attachments/file_8.txt
+ Completed
+
+
+ user5@email
+ user
+ %d
+ user5@email
+
+
+ user6@email
+ user
+ %d
+ user6@email
+
+
+ user1@email
+ user
+ %d
+ user1@email
+
+ %d
+
diff --git a/server/enterprise/message_export/testdata/actianceE2E1Batch3ch4.tmpl b/server/enterprise/message_export/testdata/actianceE2E1Batch3ch4.tmpl
new file mode 100644
index 0000000000..28f2a25217
--- /dev/null
+++ b/server/enterprise/message_export/testdata/actianceE2E1Batch3ch4.tmpl
@@ -0,0 +1,17 @@
+
+ public - channel_four_name - %s
+ %d
+
+ user8@email
+ user
+ %d
+ user8@email
+
+
+ user8@email
+ user
+ %d
+ user8@email
+
+ %d
+
diff --git a/server/enterprise/message_export/testdata/actianceE2E2.tmpl b/server/enterprise/message_export/testdata/actianceE2E2.tmpl
new file mode 100644
index 0000000000..6e7e0042a4
--- /dev/null
+++ b/server/enterprise/message_export/testdata/actianceE2E2.tmpl
@@ -0,0 +1,46 @@
+
+
+
+ private - channel_two_name - %s
+ %d
+
+ user1@email
+ user
+ %d
+ user1@email
+
+
+ user2@email
+ user
+ %d
+ user2@email
+
+
+ %s
+ user1@email
+ user
+ %d
+ message 1
+
+
+ %s
+ user2@email
+ user
+ %d
+ message 2
+
+
+ user1@email
+ user
+ %d
+ user1@email
+
+
+ user2@email
+ user
+ %d
+ user2@email
+
+ %d
+
+
diff --git a/server/enterprise/message_export/testdata/actianceXMLHeader.tmpl b/server/enterprise/message_export/testdata/actianceXMLHeader.tmpl
new file mode 100644
index 0000000000..00082370b9
--- /dev/null
+++ b/server/enterprise/message_export/testdata/actianceXMLHeader.tmpl
@@ -0,0 +1,3 @@
+
+
+%s%s
diff --git a/server/enterprise/message_export/testdata/csvE2E1Batch1.tmpl b/server/enterprise/message_export/testdata/csvE2E1Batch1.tmpl
new file mode 100644
index 0000000000..f2c741c78e
--- /dev/null
+++ b/server/enterprise/message_export/testdata/csvE2E1Batch1.tmpl
@@ -0,0 +1,14 @@
+Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
+%[16]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
+%[17]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[7]s,user2@email,user2,,,,User user2 (user2@email) joined the channel,enter,user,
+%[18]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[7]s,user2@email,user2,,,,User user2 (user2@email) left the channel,leave,user,
+%[19]d,%[19]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[10]s,,,message 0,message,user,
+%[19]d,%[19]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[10]s,,,(files/%[10]s/%[13]s-file_0.txt),attachment,user,
+%[20]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[8]s,user3@email,user3,,,,User user3 (user3@email) joined the channel,enter,user,
+%[21]d,%[21]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[11]s,,,message 1,message,user,
+%[21]d,%[21]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[11]s,,,(files/%[11]s/%[14]s-file_1.txt),attachment,user,
+%[22]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[8]s,user3@email,user3,,,,User user3 (user3@email) left the channel,leave,user,
+%[23]d,0,,%[1]s,,,%[5]s,channel_three_name,the Channel Three,public,%[9]s,user7@email,user7,,,,User user7 (user7@email) was already in the channel,previously-joined,user,
+%[24]d,0,,%[1]s,,,%[5]s,channel_three_name,the Channel Three,public,%[9]s,user7@email,user7,,,,User user7 (user7@email) left the channel,leave,user,
+%[25]d,%[25]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[12]s,,,message 2,message,user,
+%[25]d,%[25]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[12]s,,,(files/%[12]s/%[15]s-file_2.txt),attachment,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E1Batch2.tmpl b/server/enterprise/message_export/testdata/csvE2E1Batch2.tmpl
new file mode 100644
index 0000000000..0d95a21e23
--- /dev/null
+++ b/server/enterprise/message_export/testdata/csvE2E1Batch2.tmpl
@@ -0,0 +1,10 @@
+Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
+%[13]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
+%[14]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user4@email,user4,,,,User user4 (user4@email) joined the channel,enter,user,
+%[15]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user4@email,user4,,,,User user4 (user4@email) left the channel,leave,user,
+%[16]d,%[16]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,message 3,message,user,
+%[16]d,%[16]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,(files/%[7]s/%[10]s-file_3.txt),attachment,user,
+%[17]d,%[17]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[8]s,,,message 4,message,user,
+%[17]d,%[17]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[8]s,,,(files/%[8]s/%[11]s-file_4.txt),attachment,user,
+%[18]d,%[18]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,message 5,message,user,
+%[18]d,%[18]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[12]s-file_5.txt),attachment,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E1Batch3.tmpl b/server/enterprise/message_export/testdata/csvE2E1Batch3.tmpl
new file mode 100644
index 0000000000..4eb6ab7625
--- /dev/null
+++ b/server/enterprise/message_export/testdata/csvE2E1Batch3.tmpl
@@ -0,0 +1,13 @@
+Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
+%[16]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
+%[17]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[7]s,user5@email,user5,,,,User user5 (user5@email) joined the channel,enter,user,
+%[18]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[7]s,user5@email,user5,,,,User user5 (user5@email) left the channel,leave,user,
+%[19]d,%[19]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[10]s,,,message 6,message,user,
+%[19]d,%[19]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[10]s,,,(files/%[10]s/%[13]s-file_6.txt),attachment,user,
+%[20]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[11]s,,,message 7,message,user,
+%[20]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[11]s,,,(files/%[11]s/%[14]s-file_7.txt),attachment,user,
+%[21]d,%[21]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[12]s,,,message 8,message,user,
+%[21]d,%[21]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[12]s,,,(files/%[12]s/%[15]s-file_8.txt),attachment,user,
+%[22]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[8]s,user6@email,user6,,,,User user6 (user6@email) joined the channel,enter,user,
+%[23]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[8]s,user6@email,user6,,,,User user6 (user6@email) left the channel,leave,user,
+%[24]d,0,,%[1]s,,,%[5]s,channel_four_name,the Channel Four,public,%[9]s,user8@email,user8,,,,User user8 (user8@email) joined the channel,enter,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E2Batch1.tmpl b/server/enterprise/message_export/testdata/csvE2E2Batch1.tmpl
new file mode 100644
index 0000000000..bd0250d5da
--- /dev/null
+++ b/server/enterprise/message_export/testdata/csvE2E2Batch1.tmpl
@@ -0,0 +1,5 @@
+Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
+%[9]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
+%[10]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user2@email,user2,,,,User user2 (user2@email) was already in the channel,previously-joined,user,
+%[11]d,%[11]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,message 1,message,user,
+%[12]d,%[12]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user2@email,user2,%[8]s,,,message 2,message,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E3Batch1Perm1.tmpl b/server/enterprise/message_export/testdata/csvE2E3Batch1Perm1.tmpl
new file mode 100644
index 0000000000..e879f4000d
--- /dev/null
+++ b/server/enterprise/message_export/testdata/csvE2E3Batch1Perm1.tmpl
@@ -0,0 +1,13 @@
+Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
+%[14]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
+%[15]d,%[15]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[6]s,,,message 0,message,user,
+%[16]d,%[17]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,message 1,message,user,
+%[16]d,%[17]d,Deleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,delete message 1,message,user,
+%[18]d,%[25]d,UpdatedNoMsgChange,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[8]s,,,message 2,message,user,
+%[19]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,message 3,message,user,
+%[19]d,%[20]d,Deleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,delete message 3,message,user,
+%[19]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[13]s-file_3.txt),attachment,user,
+%[19]d,%[20]d,FileDeleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[13]s-file_3.txt),deleted attachment,user,
+%[21]d,%[22]d,EditedOriginalMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[10]s,%[11]s,,message 4,message,user,
+%[21]d,%[22]d,EditedNewMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[11]s,,,edited message 4,message,user,
+%[23]d,%[26]d,EditedOriginalMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[12]s,%[24]s,,message 6,message,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E3Batch1Perm2.tmpl b/server/enterprise/message_export/testdata/csvE2E3Batch1Perm2.tmpl
new file mode 100644
index 0000000000..15166d1fa3
--- /dev/null
+++ b/server/enterprise/message_export/testdata/csvE2E3Batch1Perm2.tmpl
@@ -0,0 +1,13 @@
+Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
+%[14]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
+%[15]d,%[15]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[6]s,,,message 0,message,user,
+%[16]d,%[17]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,message 1,message,user,
+%[16]d,%[17]d,Deleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,delete message 1,message,user,
+%[18]d,%[25]d,UpdatedNoMsgChange,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[8]s,,,message 2,message,user,
+%[19]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,message 3,message,user,
+%[19]d,%[20]d,Deleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,delete message 3,message,user,
+%[19]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[13]s-file_3.txt),attachment,user,
+%[19]d,%[20]d,FileDeleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[13]s-file_3.txt),deleted attachment,user,
+%[21]d,%[22]d,EditedNewMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[11]s,,,edited message 4,message,user,
+%[21]d,%[22]d,EditedOriginalMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[10]s,%[11]s,,message 4,message,user,
+%[23]d,%[26]d,EditedOriginalMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[12]s,%[24]s,,message 6,message,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E3Batch1Perm3.tmpl b/server/enterprise/message_export/testdata/csvE2E3Batch1Perm3.tmpl
new file mode 100644
index 0000000000..0ff6103a4c
--- /dev/null
+++ b/server/enterprise/message_export/testdata/csvE2E3Batch1Perm3.tmpl
@@ -0,0 +1,13 @@
+Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
+%[14]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
+%[15]d,%[15]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[6]s,,,message 0,message,user,
+%[16]d,%[17]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,message 1,message,user,
+%[16]d,%[17]d,Deleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,delete message 1,message,user,
+%[18]d,%[24]d,UpdatedNoMsgChange,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[8]s,,,message 2,message,user,
+%[19]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,message 3,message,user,
+%[19]d,%[20]d,Deleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,delete message 3,message,user,
+%[19]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[13]s-file_3.txt),attachment,user,
+%[19]d,%[20]d,FileDeleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[13]s-file_3.txt),deleted attachment,user,
+%[21]d,%[22]d,EditedOriginalMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[10]s,%[11]s,,message 4,message,user,
+%[21]d,%[22]d,EditedNewMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[11]s,,,edited message 4,message,user,
+%[25]d,%[26]d,EditedNewMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[12]s,,,edited message 6,message,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E3Batch1Perm4.tmpl b/server/enterprise/message_export/testdata/csvE2E3Batch1Perm4.tmpl
new file mode 100644
index 0000000000..ba121187d7
--- /dev/null
+++ b/server/enterprise/message_export/testdata/csvE2E3Batch1Perm4.tmpl
@@ -0,0 +1,13 @@
+Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
+%[14]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
+%[15]d,%[15]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[6]s,,,message 0,message,user,
+%[16]d,%[17]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,message 1,message,user,
+%[16]d,%[17]d,Deleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,delete message 1,message,user,
+%[18]d,%[24]d,UpdatedNoMsgChange,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[8]s,,,message 2,message,user,
+%[19]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,message 3,message,user,
+%[19]d,%[20]d,Deleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,delete message 3,message,user,
+%[19]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[13]s-file_3.txt),attachment,user,
+%[19]d,%[20]d,FileDeleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[13]s-file_3.txt),deleted attachment,user,
+%[21]d,%[22]d,EditedNewMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[11]s,,,edited message 4,message,user,
+%[21]d,%[22]d,EditedOriginalMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[10]s,%[11]s,,message 4,message,user,
+%[25]d,%[26]d,EditedNewMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[12]s,,,edited message 6,message,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E3Batch2Perm1.tmpl b/server/enterprise/message_export/testdata/csvE2E3Batch2Perm1.tmpl
new file mode 100644
index 0000000000..aba0e2a347
--- /dev/null
+++ b/server/enterprise/message_export/testdata/csvE2E3Batch2Perm1.tmpl
@@ -0,0 +1,3 @@
+Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
+%[9]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
+%[8]d,%[10]d,EditedOriginalMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[6]s,%[7]s,,message 6,message,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E3Batch2Perm2.tmpl b/server/enterprise/message_export/testdata/csvE2E3Batch2Perm2.tmpl
new file mode 100644
index 0000000000..cbf6719310
--- /dev/null
+++ b/server/enterprise/message_export/testdata/csvE2E3Batch2Perm2.tmpl
@@ -0,0 +1,3 @@
+Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
+%[8]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
+%[7]d,%[9]d,EditedNewMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[6]s,,,edited message 6,message,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E4Batch1.tmpl b/server/enterprise/message_export/testdata/csvE2E4Batch1.tmpl
new file mode 100644
index 0000000000..3fbf4f31bc
--- /dev/null
+++ b/server/enterprise/message_export/testdata/csvE2E4Batch1.tmpl
@@ -0,0 +1,6 @@
+%[11]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
+%[12]d,%[14]d,EditedOriginalMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[6]s,%[7]s,,message 0,message,user,
+%[12]d,%[14]d,EditedNewMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,edited message 0,message,user,
+%[13]d,%[14]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[8]s,,,message 2,message,user,
+%[13]d,%[14]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,message 3,message,user,
+%[13]d,%[14]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[10]s,,,message 4,message,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E5Job1.tmpl b/server/enterprise/message_export/testdata/csvE2E5Job1.tmpl
new file mode 100644
index 0000000000..b803ead72b
--- /dev/null
+++ b/server/enterprise/message_export/testdata/csvE2E5Job1.tmpl
@@ -0,0 +1,2 @@
+1732245676086,czoko3u8oidrtcgt58qi3afmrc,faketeambfc64t,dn_qtkwchcc7jng5mo87qw5gzxojo,zao11dtduinninqwtbjn3c6jzw,fakechannelsucatskrmt,dn_pyo5j7yqui865qt69ikoi6sgcr,public,741xw3i71frkmks64of8myh4gc,tfxgyrwfbbymzqz3c74sysbqao@localhost,fakeuser6b4riuzx5b,pf4y8d9s77domftewwfzk4fn5o,,,message 0,message,user,
+1732245676088,czoko3u8oidrtcgt58qi3afmrc,faketeambfc64t,dn_qtkwchcc7jng5mo87qw5gzxojo,zao11dtduinninqwtbjn3c6jzw,fakechannelsucatskrmt,dn_pyo5j7yqui865qt69ikoi6sgcr,public,741xw3i71frkmks64of8myh4gc,tfxgyrwfbbymzqz3c74sysbqao@localhost,fakeuser6b4riuzx5b,pf4y8d9s77domftewwfzk4fn5o,,,delete message 0,message,user,
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch1.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch1.tmpl
new file mode 100644
index 0000000000..abbac799f9
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E1Batch1.tmpl
@@ -0,0 +1,137 @@
+Mattermost Compliance Export
+
+Conversation Summary
+
+
+ - TeamId:%[13]s
+ - TeamName:%[14]s
+ - TeamDisplayName:%[15]s
+ - ChannelId:%[22]s
+ - ChannelName:channel_two_name
+ - ChannelDisplayName:the Channel Two
+ - Started:%[1]s
+ - Ended:%[2]s
+ - Duration:0 seconds
+
+
+
+
+ UserId
|
+ Username
|
+ UserType
|
+ Email |
+ Joined |
+ Left |
+ Duration |
+ Messages |
+
+ =20
+
+ | %[16]s |
+ @user1 |
+ user |
+ user1@email |
+ %[3]s |
+ %[4]s |
+ 0 seconds |
+ 3 |
+
+
+
+ | %[17]s |
+ @user2 |
+ user |
+ user2@email |
+ %[5]s |
+ %[6]s |
+ 0 seconds |
+ 0 |
+
+
+
+ | %[18]s |
+ @user3 |
+ user |
+ user3@email |
+ %[7]s |
+ %[8]s |
+ 0 seconds |
+ 0 |
+
+
+
+
+Messages
+
+
+ =20
+-
+ %[19]s
+ %[9]s
+ @user1
+ %[16]s
+ @user1
+ user
+ (user1@email)
+ message 0
+
+
+-
+ %[19]s
+ %[9]s
+ @user1
+ %[16]s
+ @user1
+ user
+ (user1@email)
+ Uploaded file
+
+
+-
+ %[20]s
+ %[10]s
+ @user1
+ %[16]s
+ @user1
+ user
+ (user1@email)
+ message 1
+
+
+-
+ %[20]s
+ %[10]s
+ @user1
+ %[16]s
+ @user1
+ user
+ (user1@email)
+ Uploaded file
+
+
+-
+ %[21]s
+ %[11]s
+ @user1
+ %[16]s
+ @user1
+ user
+ (user1@email)
+ message 2
+
+
+-
+ %[21]s
+ %[11]s
+ @user1
+ %[16]s
+ @user1
+ user
+ (user1@email)
+ Uploaded file
+
+
+
+
+
+Exported on %[12]s
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch1Ch3.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch1Ch3.tmpl
new file mode 100644
index 0000000000..3461fba402
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E1Batch1Ch3.tmpl
@@ -0,0 +1,47 @@
+Conversation Summary
+
+
+ - TeamId:%[1]s
+ - TeamName:
+ - TeamDisplayName:
+ - ChannelId:%[2]s
+ - ChannelName:channel_three_name
+ - ChannelDisplayName:the Channel Three
+ - Started:%[3]s
+ - Ended:%[4]s
+ - Duration:0 seconds
+
+
+
+
+ UserId
|
+ Username
|
+ UserType
|
+ Email |
+ Joined |
+ Left |
+ Duration |
+ Messages |
+
+ =20
+
+ | %[5]s |
+ @user7 |
+ user |
+ user7@email |
+ %[6]s |
+ %[7]s |
+ 0 seconds |
+ 0 |
+
+
+
+
+Messages
+
+
+Exported on %[8]s
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch1Summary.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch1Summary.tmpl
new file mode 100644
index 0000000000..9855c22aaf
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E1Batch1Summary.tmpl
@@ -0,0 +1,29 @@
+--------------------
+Conversation Summary
+--------------------
+
+* TeamId: %[14]s
+* TeamName: %[15]s
+* TeamDisplayName: %[16]s
+* ChannelId: %[13]s
+* ChannelName: channel_two_name
+* ChannelDisplayName: %[1]s
+* Started: %[2]s
+* Ended: %[3]s
+* Duration: 0 seconds
+
+UserId
+Username
+UserType
+Email Joined Left Duration Messages %[17]s @user1 user user1@email %[4]s %[5]s 0 seconds 3 %[18]s @user2 user user2@email %[6]s %[7]s 0 seconds 0 %[19]s @user3 user user3@email %[8]s %[9]s 0 seconds 0
+
+--------
+Messages
+--------
+
+* %[20]s %[10]s @user1 %[17]s @user1 user (user1@email) message 0
+* %[20]s %[10]s @user1 %[17]s @user1 user (user1@email) Uploaded file
+* %[21]s %[11]s @user1 %[17]s @user1 user (user1@email) message 1
+* %[21]s %[11]s @user1 %[17]s @user1 user (user1@email) Uploaded file
+* %[22]s %[12]s @user1 %[17]s @user1 user (user1@email) message 2
+* %[22]s %[12]s @user1 %[17]s @user1 user (user1@email) Uploaded file
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch1SummaryCh3.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch1SummaryCh3.tmpl
new file mode 100644
index 0000000000..ecc9eba10f
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E1Batch1SummaryCh3.tmpl
@@ -0,0 +1,22 @@
+--------------------
+Conversation Summary
+--------------------
+
+* TeamId: %[1]s
+* TeamName:
+* TeamDisplayName:
+* ChannelId: %[4]s
+* ChannelName: channel_three_name
+* ChannelDisplayName: the Channel Three
+* Started: %[5]s
+* Ended: %[6]s
+* Duration: 0 seconds
+
+UserId
+Username
+UserType
+Email Joined Left Duration Messages %[7]s @user7 user user7@email %[8]s %[9]s 0 seconds 0
+
+--------
+Messages
+--------
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch2.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch2.tmpl
new file mode 100644
index 0000000000..81cea56645
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E1Batch2.tmpl
@@ -0,0 +1,126 @@
+Mattermost Compliance Export
+
+Conversation Summary
+
+
+ - TeamId:%[11]s
+ - TeamName:%[12]s
+ - TeamDisplayName:%[13]s
+ - ChannelId:%[20]s
+ - ChannelName:channel_two_name
+ - ChannelDisplayName:the Channel Two
+ - Started:%[1]s
+ - Ended:%[2]s
+ - Duration:0 seconds
+
+
+
+
+ UserId
|
+ Username
|
+ UserType
|
+ Email |
+ Joined |
+ Left |
+ Duration |
+ Messages |
+
+ =20
+
+ | %[14]s |
+ @user1 |
+ user |
+ user1@email |
+ %[3]s |
+ %[4]s |
+ 0 seconds |
+ 3 |
+
+
+
+ | %[16]s |
+ @user4 |
+ user |
+ user4@email |
+ %[5]s |
+ %[6]s |
+ 0 seconds |
+ 0 |
+
+
+
+
+Messages
+
+
+ =20
+-
+ %[17]s
+ %[7]s
+ @user1
+ %[14]s
+ @user1
+ user
+ (user1@email)
+ message 3
+
+
+-
+ %[17]s
+ %[7]s
+ @user1
+ %[14]s
+ @user1
+ user
+ (user1@email)
+ Uploaded file
+
+
+-
+ %[18]s
+ %[8]s
+ @user1
+ %[14]s
+ @user1
+ user
+ (user1@email)
+ message 4
+
+
+-
+ %[18]s
+ %[8]s
+ @user1
+ %[14]s
+ @user1
+ user
+ (user1@email)
+ Uploaded file
+
+
+-
+ %[19]s
+ %[9]s
+ @user1
+ %[14]s
+ @user1
+ user
+ (user1@email)
+ message 5
+
+
+-
+ %[19]s
+ %[9]s
+ @user1
+ %[14]s
+ @user1
+ user
+ (user1@email)
+ Uploaded file
+
+
+
+
+
+Exported on %[10]s
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch2Summary.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch2Summary.tmpl
new file mode 100644
index 0000000000..5197e603df
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E1Batch2Summary.tmpl
@@ -0,0 +1,29 @@
+--------------------
+Conversation Summary
+--------------------
+
+* TeamId: %[11]s
+* TeamName: %[12]s
+* TeamDisplayName: %[13]s
+* ChannelId: %[10]s
+* ChannelName: channel_two_name
+* ChannelDisplayName: the Channel Two
+* Started: %[1]s
+* Ended: %[2]s
+* Duration: 0 seconds
+
+UserId
+Username
+UserType
+Email Joined Left Duration Messages %[14]s @user1 user user1@email %[3]s %[4]s 0 seconds 3 %[15]s @user4 user user4@email %[5]s %[6]s 0 seconds 0
+
+--------
+Messages
+--------
+
+* %[16]s %[7]s @user1 %[14]s @user1 user (user1@email) message 3
+* %[16]s %[7]s @user1 %[14]s @user1 user (user1@email) Uploaded file
+* %[17]s %[8]s @user1 %[14]s @user1 user (user1@email) message 4
+* %[17]s %[8]s @user1 %[14]s @user1 user (user1@email) Uploaded file
+* %[18]s %[9]s @user1 %[14]s @user1 user (user1@email) message 5
+* %[18]s %[9]s @user1 %[14]s @user1 user (user1@email) Uploaded file
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch3.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch3.tmpl
new file mode 100644
index 0000000000..3aa981c8ad
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E1Batch3.tmpl
@@ -0,0 +1,137 @@
+Mattermost Compliance Export
+
+Conversation Summary
+
+
+ - TeamId:%[13]s
+ - TeamName:%[14]s
+ - TeamDisplayName:%[15]s
+ - ChannelId:%[22]s
+ - ChannelName:channel_two_name
+ - ChannelDisplayName:the Channel Two
+ - Started:%[1]s
+ - Ended:%[2]s
+ - Duration:0 seconds
+
+
+
+
+ UserId
|
+ Username
|
+ UserType
|
+ Email |
+ Joined |
+ Left |
+ Duration |
+ Messages |
+
+ =20
+
+ | %[16]s |
+ @user1 |
+ user |
+ user1@email |
+ %[3]s |
+ %[4]s |
+ 0 seconds |
+ 3 |
+
+
+
+ | %[17]s |
+ @user5 |
+ user |
+ user5@email |
+ %[5]s |
+ %[6]s |
+ 0 seconds |
+ 0 |
+
+
+
+ | %[18]s |
+ @user6 |
+ user |
+ user6@email |
+ %[7]s |
+ %[8]s |
+ 0 seconds |
+ 0 |
+
+
+
+
+Messages
+
+
+ =20
+-
+ %[19]s
+ %[9]s
+ @user1
+ %[16]s
+ @user1
+ user
+ (user1@email)
+ message 6
+
+
+-
+ %[19]s
+ %[9]s
+ @user1
+ %[16]s
+ @user1
+ user
+ (user1@email)
+ Uploaded file
+
+
+-
+ %[20]s
+ %[10]s
+ @user1
+ %[16]s
+ @user1
+ user
+ (user1@email)
+ message 7
+
+
+-
+ %[20]s
+ %[10]s
+ @user1
+ %[16]s
+ @user1
+ user
+ (user1@email)
+ Uploaded file
+
+
+-
+ %[21]s
+ %[11]s
+ @user1
+ %[16]s
+ @user1
+ user
+ (user1@email)
+ message 8
+
+
+-
+ %[21]s
+ %[11]s
+ @user1
+ %[16]s
+ @user1
+ user
+ (user1@email)
+ Uploaded file
+
+
+
+
+
+Exported on %[12]s
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch3Ch4.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch3Ch4.tmpl
new file mode 100644
index 0000000000..a24fc616f9
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E1Batch3Ch4.tmpl
@@ -0,0 +1,47 @@
+Conversation Summary
+
+
+ - TeamId:%[1]s
+ - TeamName:
+ - TeamDisplayName:
+ - ChannelId:%[2]s
+ - ChannelName:channel_four_name
+ - ChannelDisplayName:the Channel Four
+ - Started:%[3]s
+ - Ended:%[4]s
+ - Duration:0 seconds
+
+
+
+
+ UserId
|
+ Username
|
+ UserType
|
+ Email |
+ Joined |
+ Left |
+ Duration |
+ Messages |
+
+ =20
+
+ | %[5]s |
+ @user8 |
+ user |
+ user8@email |
+ %[6]s |
+ %[4]s |
+ 0 seconds |
+ 0 |
+
+
+
+
+Messages
+
+
+Exported on %[8]s
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch3Summary.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch3Summary.tmpl
new file mode 100644
index 0000000000..2554298f02
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E1Batch3Summary.tmpl
@@ -0,0 +1,29 @@
+--------------------
+Conversation Summary
+--------------------
+
+* TeamId: %[13]s
+* TeamName: %[14]s
+* TeamDisplayName: %[15]s
+* ChannelId: %[12]s
+* ChannelName: channel_two_name
+* ChannelDisplayName: the Channel Two
+* Started: %[1]s
+* Ended: %[2]s
+* Duration: 0 seconds
+
+UserId
+Username
+UserType
+Email Joined Left Duration Messages %[16]s @user1 user user1@email %[3]s %[4]s 0 seconds 3 %[22]s @user5 user user5@email %[5]s %[6]s 0 seconds 0 %[23]s @user6 user user6@email %[7]s %[8]s 0 seconds 0
+
+--------
+Messages
+--------
+
+* %[19]s %[9]s @user1 %[16]s @user1 user (user1@email) message 6
+* %[19]s %[9]s @user1 %[16]s @user1 user (user1@email) Uploaded file
+* %[20]s %[10]s @user1 %[16]s @user1 user (user1@email) message 7
+* %[20]s %[10]s @user1 %[16]s @user1 user (user1@email) Uploaded file
+* %[21]s %[11]s @user1 %[16]s @user1 user (user1@email) message 8
+* %[21]s %[11]s @user1 %[16]s @user1 user (user1@email) Uploaded file
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch3SummaryCh4.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch3SummaryCh4.tmpl
new file mode 100644
index 0000000000..910dafdda9
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E1Batch3SummaryCh4.tmpl
@@ -0,0 +1,22 @@
+--------------------
+Conversation Summary
+--------------------
+
+* TeamId: %[1]s
+* TeamName:
+* TeamDisplayName:
+* ChannelId: %[4]s
+* ChannelName: channel_four_name
+* ChannelDisplayName: the Channel Four
+* Started: %[5]s
+* Ended: %[6]s
+* Duration: 0 seconds
+
+UserId
+Username
+UserType
+Email Joined Left Duration Messages %[7]s @user8 user user8@email %[8]s %[6]s 0 seconds 0
+
+--------
+Messages
+--------
diff --git a/server/enterprise/message_export/testdata/grE2E2Batch1.tmpl b/server/enterprise/message_export/testdata/grE2E2Batch1.tmpl
new file mode 100644
index 0000000000..43c641755c
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E2Batch1.tmpl
@@ -0,0 +1,82 @@
+Mattermost Compliance Export
+
+Conversation Summary
+
+
+ - TeamId:%[10]s
+ - TeamName:%[11]s
+ - TeamDisplayName:%[12]s
+ - ChannelId:%[13]s
+ - ChannelName:channel_two_name
+ - ChannelDisplayName:the Channel Two
+ - Started:%[1]s
+ - Ended:%[2]s
+ - Duration:0 seconds
+
+
+
+
+ UserId
|
+ Username
|
+ UserType
|
+ Email |
+ Joined |
+ Left |
+ Duration |
+ Messages |
+
+ =20
+
+ | %[14]s |
+ @user1 |
+ user |
+ user1@email |
+ %[3]s |
+ %[4]s |
+ 0 seconds |
+ 1 |
+
+
+
+ | %[15]s |
+ @user2 |
+ user |
+ user2@email |
+ %[5]s |
+ %[6]s |
+ 0 seconds |
+ 1 |
+
+
+
+
+Messages
+
+
+ =20
+-
+ %[16]s
+ %[7]s
+ @user1
+ %[14]s
+ @user1
+ user
+ (user1@email)
+ message 1
+
+
+-
+ %[17]s
+ %[8]s
+ @user2
+ %[15]s
+ @user2
+ user
+ (user2@email)
+ message 2
+
+
+
+
+
+Exported on %[9]s
diff --git a/server/enterprise/message_export/testdata/grE2E2Batch1Summary.tmpl b/server/enterprise/message_export/testdata/grE2E2Batch1Summary.tmpl
new file mode 100644
index 0000000000..8dbea6c4ea
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E2Batch1Summary.tmpl
@@ -0,0 +1,25 @@
+--------------------
+Conversation Summary
+--------------------
+
+* TeamId: %[9]s
+* TeamName: %[10]s
+* TeamDisplayName: %[11]s
+* ChannelId: %[12]s
+* ChannelName: channel_two_name
+* ChannelDisplayName: the Channel Two
+* Started: %[1]s
+* Ended: %[2]s
+* Duration: 0 seconds
+
+UserId
+Username
+UserType
+Email Joined Left Duration Messages %[13]s @user1 user user1@email %[3]s %[4]s 0 seconds 1 %[14]s @user2 user user2@email %[5]s %[6]s 0 seconds 1
+
+--------
+Messages
+--------
+
+* %[15]s %[7]s @user1 %[13]s @user1 user (user1@email) message 1
+* %[16]s %[8]s @user2 %[14]s @user2 user (user2@email) message 2
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch1Perm1.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch1Perm1.tmpl
new file mode 100644
index 0000000000..f3e66c0c1c
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E3Batch1Perm1.tmpl
@@ -0,0 +1,191 @@
+Mattermost Compliance Export
+
+Conversation Summary
+
+
+ - TeamId:%[13]s
+ - TeamName:%[14]s
+ - TeamDisplayName:%[15]s
+ - ChannelId:%[16]s
+ - ChannelName:channel_two_name
+ - ChannelDisplayName:the Channel Two
+ - Started:%[1]s
+ - Ended:%[2]s
+ - Duration:0 seconds
+
+
+
+
+ UserId
|
+ Username
|
+ UserType
|
+ Email |
+ Joined |
+ Left |
+ Duration |
+ Messages |
+
+ =20
+
+ | %[17]s |
+ @user1 |
+ user |
+ user1@email |
+ %[3]s |
+ %[4]s |
+ 0 seconds |
+ 9 |
+
+
+
+
+Messages
+
+
+ =20
+-
+ %[18]s
+ %[5]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 0
+
+
+-
+ %[19]s
+ %[6]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 1
+
+
+-
+ %[19]s
+ %[6]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ delete message 1
+ Deleted
+ %[26]s
+
+
+
+-
+ %[20]s
+ %[7]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 2
+ UpdatedNoMsgChange
+ %[27]s
+
+
+
+-
+ %[21]s
+ %[8]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 3
+
+
+-
+ %[21]s
+ %[8]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ Uploaded file
+
+
+-
+ %[21]s
+ %[8]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ delete message 3
+ Deleted
+ %[9]s
+
+
+
+-
+ %[21]s
+ %[8]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ Deleted file
+ FileDeleted
+ %[9]s
+
+
+
+-
+ %[22]s
+ %[10]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 4
+ EditedOriginalMsg
+ %[28]s
+ %[23]s
+
+
+-
+ %[23]s
+ %[11]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ edited message 4
+ EditedNewMsg
+ %[28]s
+
+
+
+-
+ %[24]s
+ %[12]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 6
+ EditedOriginalMsg
+ %[29]s
+ %[25]s
+
+
+
+
+
+Exported on %[1]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch1Perm2.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch1Perm2.tmpl
new file mode 100644
index 0000000000..c755f39a45
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E3Batch1Perm2.tmpl
@@ -0,0 +1,191 @@
+Mattermost Compliance Export
+
+Conversation Summary
+
+
+ - TeamId:%[13]s
+ - TeamName:%[14]s
+ - TeamDisplayName:%[15]s
+ - ChannelId:%[16]s
+ - ChannelName:channel_two_name
+ - ChannelDisplayName:the Channel Two
+ - Started:%[1]s
+ - Ended:%[2]s
+ - Duration:0 seconds
+
+
+
+
+ UserId
|
+ Username
|
+ UserType
|
+ Email |
+ Joined |
+ Left |
+ Duration |
+ Messages |
+
+ =20
+
+ | %[17]s |
+ @user1 |
+ user |
+ user1@email |
+ %[3]s |
+ %[4]s |
+ 0 seconds |
+ 9 |
+
+
+
+
+Messages
+
+
+ =20
+-
+ %[18]s
+ %[5]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 0
+
+
+-
+ %[19]s
+ %[6]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 1
+
+
+-
+ %[19]s
+ %[6]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ delete message 1
+ Deleted
+ %[26]s
+
+
+
+-
+ %[20]s
+ %[7]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 2
+ UpdatedNoMsgChange
+ %[27]s
+
+
+
+-
+ %[21]s
+ %[8]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 3
+
+
+-
+ %[21]s
+ %[8]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ Uploaded file
+
+
+-
+ %[21]s
+ %[8]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ delete message 3
+ Deleted
+ %[9]s
+
+
+
+-
+ %[21]s
+ %[8]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ Deleted file
+ FileDeleted
+ %[9]s
+
+
+
+-
+ %[23]s
+ %[11]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ edited message 4
+ EditedNewMsg
+ %[28]s
+
+
+
+-
+ %[22]s
+ %[10]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 4
+ EditedOriginalMsg
+ %[28]s
+ %[23]s
+
+
+-
+ %[24]s
+ %[12]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 6
+ EditedOriginalMsg
+ %[29]s
+ %[25]s
+
+
+
+
+
+Exported on %[1]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch1Perm3.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch1Perm3.tmpl
new file mode 100644
index 0000000000..70fbaf1602
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E3Batch1Perm3.tmpl
@@ -0,0 +1,191 @@
+Mattermost Compliance Export
+
+Conversation Summary
+
+
+ - TeamId:%[13]s
+ - TeamName:%[14]s
+ - TeamDisplayName:%[15]s
+ - ChannelId:%[16]s
+ - ChannelName:channel_two_name
+ - ChannelDisplayName:the Channel Two
+ - Started:%[1]s
+ - Ended:%[2]s
+ - Duration:0 seconds
+
+
+
+
+ UserId
|
+ Username
|
+ UserType
|
+ Email |
+ Joined |
+ Left |
+ Duration |
+ Messages |
+
+ =20
+
+ | %[17]s |
+ @user1 |
+ user |
+ user1@email |
+ %[3]s |
+ %[4]s |
+ 0 seconds |
+ 9 |
+
+
+
+
+Messages
+
+
+ =20
+-
+ %[18]s
+ %[5]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 0
+
+
+-
+ %[19]s
+ %[6]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 1
+
+
+-
+ %[19]s
+ %[6]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ delete message 1
+ Deleted
+ %[26]s
+
+
+
+-
+ %[20]s
+ %[7]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 2
+ UpdatedNoMsgChange
+ %[27]s
+
+
+
+-
+ %[21]s
+ %[8]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 3
+
+
+-
+ %[21]s
+ %[8]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ Uploaded file
+
+
+-
+ %[21]s
+ %[8]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ delete message 3
+ Deleted
+ %[9]s
+
+
+
+-
+ %[21]s
+ %[8]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ Deleted file
+ FileDeleted
+ %[9]s
+
+
+
+-
+ %[22]s
+ %[10]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 4
+ EditedOriginalMsg
+ %[28]s
+ %[23]s
+
+
+-
+ %[23]s
+ %[11]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ edited message 4
+ EditedNewMsg
+ %[28]s
+
+
+
+-
+ %[25]s
+ %[12]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ edited message 6
+ EditedNewMsg
+ %[29]s
+
+
+
+
+
+
+Exported on %[1]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch1Perm4.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch1Perm4.tmpl
new file mode 100644
index 0000000000..118aa32725
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E3Batch1Perm4.tmpl
@@ -0,0 +1,191 @@
+Mattermost Compliance Export
+
+Conversation Summary
+
+
+ - TeamId:%[13]s
+ - TeamName:%[14]s
+ - TeamDisplayName:%[15]s
+ - ChannelId:%[16]s
+ - ChannelName:channel_two_name
+ - ChannelDisplayName:the Channel Two
+ - Started:%[1]s
+ - Ended:%[2]s
+ - Duration:0 seconds
+
+
+
+
+ UserId
|
+ Username
|
+ UserType
|
+ Email |
+ Joined |
+ Left |
+ Duration |
+ Messages |
+
+ =20
+
+ | %[17]s |
+ @user1 |
+ user |
+ user1@email |
+ %[3]s |
+ %[4]s |
+ 0 seconds |
+ 9 |
+
+
+
+
+Messages
+
+
+ =20
+-
+ %[18]s
+ %[5]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 0
+
+
+-
+ %[19]s
+ %[6]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 1
+
+
+-
+ %[19]s
+ %[6]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ delete message 1
+ Deleted
+ %[26]s
+
+
+
+-
+ %[20]s
+ %[7]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 2
+ UpdatedNoMsgChange
+ %[27]s
+
+
+
+-
+ %[21]s
+ %[8]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 3
+
+
+-
+ %[21]s
+ %[8]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ Uploaded file
+
+
+-
+ %[21]s
+ %[8]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ delete message 3
+ Deleted
+ %[9]s
+
+
+
+-
+ %[21]s
+ %[8]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ Deleted file
+ FileDeleted
+ %[9]s
+
+
+
+-
+ %[23]s
+ %[11]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ edited message 4
+ EditedNewMsg
+ %[28]s
+
+
+
+-
+ %[22]s
+ %[10]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ message 4
+ EditedOriginalMsg
+ %[28]s
+ %[23]s
+
+
+-
+ %[25]s
+ %[12]s
+ @user1
+ %[17]s
+ @user1
+ user
+ (user1@email)
+ edited message 6
+ EditedNewMsg
+ %[29]s
+
+
+
+
+
+
+Exported on %[1]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm1.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm1.tmpl
new file mode 100644
index 0000000000..ef64694bf1
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm1.tmpl
@@ -0,0 +1,34 @@
+--------------------
+Conversation Summary
+--------------------
+
+* TeamId: %[13]s
+* TeamName: %[14]s
+* TeamDisplayName: %[15]s
+* ChannelId: %[16]s
+* ChannelName: channel_two_name
+* ChannelDisplayName: the Channel Two
+* Started: %[1]s
+* Ended: %[2]s
+* Duration: 0 seconds
+
+UserId
+Username
+UserType
+Email Joined Left Duration Messages %[17]s @user1 user user1@email %[3]s %[4]s 0 seconds 9
+
+--------
+Messages
+--------
+
+* %[18]s %[5]s @user1 %[17]s @user1 user (user1@email) message 0
+* %[19]s %[6]s @user1 %[17]s @user1 user (user1@email) message 1
+* %[19]s %[6]s @user1 %[17]s @user1 user (user1@email) delete message 1 Deleted %[26]s
+* %[20]s %[7]s @user1 %[17]s @user1 user (user1@email) message 2 UpdatedNoMsgChange %[27]s
+* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) message 3
+* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) Uploaded file
+* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) delete message 3 Deleted %[9]s
+* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) Deleted file FileDeleted %[9]s
+* %[22]s %[10]s @user1 %[17]s @user1 user (user1@email) message 4 EditedOriginalMsg %[28]s %[23]s
+* %[23]s %[11]s @user1 %[17]s @user1 user (user1@email) edited message 4 EditedNewMsg %[28]s
+* %[24]s %[12]s @user1 %[17]s @user1 user (user1@email) message 6 EditedOriginalMsg %[29]s %[25]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm2.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm2.tmpl
new file mode 100644
index 0000000000..8d63a41185
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm2.tmpl
@@ -0,0 +1,34 @@
+--------------------
+Conversation Summary
+--------------------
+
+* TeamId: %[13]s
+* TeamName: %[14]s
+* TeamDisplayName: %[15]s
+* ChannelId: %[16]s
+* ChannelName: channel_two_name
+* ChannelDisplayName: the Channel Two
+* Started: %[1]s
+* Ended: %[2]s
+* Duration: 0 seconds
+
+UserId
+Username
+UserType
+Email Joined Left Duration Messages %[17]s @user1 user user1@email %[3]s %[4]s 0 seconds 9
+
+--------
+Messages
+--------
+
+* %[18]s %[5]s @user1 %[17]s @user1 user (user1@email) message 0
+* %[19]s %[6]s @user1 %[17]s @user1 user (user1@email) message 1
+* %[19]s %[6]s @user1 %[17]s @user1 user (user1@email) delete message 1 Deleted %[26]s
+* %[20]s %[7]s @user1 %[17]s @user1 user (user1@email) message 2 UpdatedNoMsgChange %[27]s
+* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) message 3
+* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) Uploaded file
+* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) delete message 3 Deleted %[9]s
+* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) Deleted file FileDeleted %[9]s
+* %[23]s %[11]s @user1 %[17]s @user1 user (user1@email) edited message 4 EditedNewMsg %[28]s
+* %[22]s %[10]s @user1 %[17]s @user1 user (user1@email) message 4 EditedOriginalMsg %[28]s %[23]s
+* %[24]s %[12]s @user1 %[17]s @user1 user (user1@email) message 6 EditedOriginalMsg %[29]s %[25]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm3.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm3.tmpl
new file mode 100644
index 0000000000..b396e09cfc
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm3.tmpl
@@ -0,0 +1,34 @@
+--------------------
+Conversation Summary
+--------------------
+
+* TeamId: %[13]s
+* TeamName: %[14]s
+* TeamDisplayName: %[15]s
+* ChannelId: %[16]s
+* ChannelName: channel_two_name
+* ChannelDisplayName: the Channel Two
+* Started: %[1]s
+* Ended: %[2]s
+* Duration: 0 seconds
+
+UserId
+Username
+UserType
+Email Joined Left Duration Messages %[17]s @user1 user user1@email %[3]s %[4]s 0 seconds 9
+
+--------
+Messages
+--------
+
+* %[18]s %[5]s @user1 %[17]s @user1 user (user1@email) message 0
+* %[19]s %[6]s @user1 %[17]s @user1 user (user1@email) message 1
+* %[19]s %[6]s @user1 %[17]s @user1 user (user1@email) delete message 1 Deleted %[26]s
+* %[20]s %[7]s @user1 %[17]s @user1 user (user1@email) message 2 UpdatedNoMsgChange %[27]s
+* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) message 3
+* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) Uploaded file
+* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) delete message 3 Deleted %[9]s
+* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) Deleted file FileDeleted %[9]s
+* %[22]s %[10]s @user1 %[17]s @user1 user (user1@email) message 4 EditedOriginalMsg %[28]s %[23]s
+* %[23]s %[11]s @user1 %[17]s @user1 user (user1@email) edited message 4 EditedNewMsg %[28]s
+* %[25]s %[12]s @user1 %[17]s @user1 user (user1@email) edited message 6 EditedNewMsg %[29]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm4.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm4.tmpl
new file mode 100644
index 0000000000..a52c309944
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm4.tmpl
@@ -0,0 +1,34 @@
+--------------------
+Conversation Summary
+--------------------
+
+* TeamId: %[13]s
+* TeamName: %[14]s
+* TeamDisplayName: %[15]s
+* ChannelId: %[16]s
+* ChannelName: channel_two_name
+* ChannelDisplayName: the Channel Two
+* Started: %[1]s
+* Ended: %[2]s
+* Duration: 0 seconds
+
+UserId
+Username
+UserType
+Email Joined Left Duration Messages %[17]s @user1 user user1@email %[3]s %[4]s 0 seconds 9
+
+--------
+Messages
+--------
+
+* %[18]s %[5]s @user1 %[17]s @user1 user (user1@email) message 0
+* %[19]s %[6]s @user1 %[17]s @user1 user (user1@email) message 1
+* %[19]s %[6]s @user1 %[17]s @user1 user (user1@email) delete message 1 Deleted %[26]s
+* %[20]s %[7]s @user1 %[17]s @user1 user (user1@email) message 2 UpdatedNoMsgChange %[27]s
+* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) message 3
+* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) Uploaded file
+* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) delete message 3 Deleted %[9]s
+* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) Deleted file FileDeleted %[9]s
+* %[23]s %[11]s @user1 %[17]s @user1 user (user1@email) edited message 4 EditedNewMsg %[28]s
+* %[22]s %[10]s @user1 %[17]s @user1 user (user1@email) message 4 EditedOriginalMsg %[28]s %[23]s
+* %[25]s %[12]s @user1 %[17]s @user1 user (user1@email) edited message 6 EditedNewMsg %[29]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch2Perm1.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch2Perm1.tmpl
new file mode 100644
index 0000000000..69ae80f23e
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E3Batch2Perm1.tmpl
@@ -0,0 +1,63 @@
+Mattermost Compliance Export
+
+Conversation Summary
+
+
+ - TeamId:%[6]s
+ - TeamName:%[7]s
+ - TeamDisplayName:%[8]s
+ - ChannelId:%[9]s
+ - ChannelName:channel_two_name
+ - ChannelDisplayName:the Channel Two
+ - Started:%[1]s
+ - Ended:%[2]s
+ - Duration:0 seconds
+
+
+
+
+ UserId
|
+ Username
|
+ UserType
|
+ Email |
+ Joined |
+ Left |
+ Duration |
+ Messages |
+
+ =20
+
+ | %[10]s |
+ @user1 |
+ user |
+ user1@email |
+ %[3]s |
+ %[4]s |
+ 0 seconds |
+ 1 |
+
+
+
+
+Messages
+
+
+ =20
+-
+ %[14]s
+ %[5]s
+ @user1
+ %[10]s
+ @user1
+ user
+ (user1@email)
+ edited message 6
+ EditedNewMsg
+ %[12]s
+
+
+
+
+
+
+Exported on %[15]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch2Perm2.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch2Perm2.tmpl
new file mode 100644
index 0000000000..5d263ed5be
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E3Batch2Perm2.tmpl
@@ -0,0 +1,63 @@
+Mattermost Compliance Export
+
+Conversation Summary
+
+
+ - TeamId:%[6]s
+ - TeamName:%[7]s
+ - TeamDisplayName:%[8]s
+ - ChannelId:%[9]s
+ - ChannelName:channel_two_name
+ - ChannelDisplayName:the Channel Two
+ - Started:%[1]s
+ - Ended:%[2]s
+ - Duration:0 seconds
+
+
+
+
+ UserId
|
+ Username
|
+ UserType
|
+ Email |
+ Joined |
+ Left |
+ Duration |
+ Messages |
+
+ =20
+
+ | %[10]s |
+ @user1 |
+ user |
+ user1@email |
+ %[3]s |
+ %[4]s |
+ 0 seconds |
+ 1 |
+
+
+
+
+Messages
+
+
+ =20
+-
+ %[13]s
+ %[5]s
+ @user1
+ %[10]s
+ @user1
+ user
+ (user1@email)
+ message 6
+ EditedOriginalMsg
+ %[12]s
+ %[14]s
+
+
+
+
+
+Exported on %[15]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch2SummaryPerm1.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch2SummaryPerm1.tmpl
new file mode 100644
index 0000000000..04b95a29da
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E3Batch2SummaryPerm1.tmpl
@@ -0,0 +1,24 @@
+--------------------
+Conversation Summary
+--------------------
+
+* TeamId: %[6]s
+* TeamName: %[7]s
+* TeamDisplayName: %[8]s
+* ChannelId: %[9]s
+* ChannelName: channel_two_name
+* ChannelDisplayName: the Channel Two
+* Started: %[1]s
+* Ended: %[2]s
+* Duration: 0 seconds
+
+UserId
+Username
+UserType
+Email Joined Left Duration Messages %[10]s @user1 user user1@email %[3]s %[4]s 0 seconds 1
+
+--------
+Messages
+--------
+
+* %[14]s %[5]s @user1 %[10]s @user1 user (user1@email) edited message 6 EditedNewMsg %[12]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch2SummaryPerm2.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch2SummaryPerm2.tmpl
new file mode 100644
index 0000000000..3f5c304d1a
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E3Batch2SummaryPerm2.tmpl
@@ -0,0 +1,24 @@
+--------------------
+Conversation Summary
+--------------------
+
+* TeamId: %[6]s
+* TeamName: %[7]s
+* TeamDisplayName: %[8]s
+* ChannelId: %[9]s
+* ChannelName: channel_two_name
+* ChannelDisplayName: the Channel Two
+* Started: %[1]s
+* Ended: %[2]s
+* Duration: 0 seconds
+
+UserId
+Username
+UserType
+Email Joined Left Duration Messages %[10]s @user1 user user1@email %[3]s %[4]s 0 seconds 1
+
+--------
+Messages
+--------
+
+* %[13]s %[5]s @user1 %[10]s @user1 user (user1@email) message 6 EditedOriginalMsg %[12]s %[14]s
diff --git a/server/enterprise/message_export/testdata/grE2E4Summary.tmpl b/server/enterprise/message_export/testdata/grE2E4Summary.tmpl
new file mode 100644
index 0000000000..5d7c87a12c
--- /dev/null
+++ b/server/enterprise/message_export/testdata/grE2E4Summary.tmpl
@@ -0,0 +1,18 @@
+--------------------
+Conversation Summary
+--------------------
+
+* TeamId: %[5]s
+* TeamName: %[6]s
+* TeamDisplayName: %[7]s
+* ChannelId: %[8]s
+* ChannelName: channel_two_name
+* ChannelDisplayName: the Channel Two
+* Started: %[1]s
+* Ended: %[2]s
+* Duration: 0 seconds
+
+UserId
+Username
+UserType
+Email Joined Left Duration Messages %[9]s @user1 user user1@email %[3]s %[4]s 0 seconds 5
diff --git a/server/enterprise/message_export/worker.go b/server/enterprise/message_export/worker.go
index 3c97073ceb..82b5c04668 100644
--- a/server/enterprise/message_export/worker.go
+++ b/server/enterprise/message_export/worker.go
@@ -4,13 +4,10 @@
package message_export
import (
- "archive/zip"
"context"
- "encoding/json"
+ "errors"
"fmt"
- "io"
"net/http"
- "os"
"path"
"strconv"
"sync"
@@ -21,28 +18,14 @@ import (
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/jobs"
"github.com/mattermost/mattermost/server/v8/channels/utils/fileutils"
- "github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
"github.com/mattermost/mattermost/server/v8/platform/shared/templates"
)
-const (
- JobDataBatchStartTimestamp = "batch_start_timestamp" // message export uses keyset pagination sorted by (posts.updateat, posts.id). batch_start_timestamp is the posts.updateat value from the previous batch.
- JobDataBatchStartId = "batch_start_id" // message export uses keyset pagination sorted by (posts.updateat, posts.id). batch_start_id is the posts.id value from the previous batch.
+const TimeBetweenBatchesMs = 100
- JobDataStartTimestamp = "start_timestamp"
- JobDataStartId = "start_id"
- JobDataExportType = "export_type"
- JOB_DATA_BatchSize = "batch_size"
- JobDataMessagesExported = "messages_exported"
- JobDataWarningCount = "warning_count"
- JobDataIsDownloadable = "is_downloadable"
- JobDirectories = "job_directories"
- TimeBetweenBatches = 100
-
- estimatedPostCount = 10_000_000
-)
-
-const exportPath = "export"
+// testEndOfBatchCb is only used for testing
+var testEndOfBatchCb func(worker *MessageExportWorker)
type MessageExportWorker struct {
name string
@@ -96,106 +79,70 @@ func (dr *MessageExportJobInterfaceImpl) MakeWorker() model.Worker {
}
}
-func (worker *MessageExportWorker) IsEnabled(cfg *model.Config) bool {
- return worker.license() != nil && *worker.license().Features.MessageExport && *cfg.MessageExportSettings.EnableExport
+func (w *MessageExportWorker) IsEnabled(cfg *model.Config) bool {
+ return w.license() != nil && *w.license().Features.MessageExport && *cfg.MessageExportSettings.EnableExport
}
-func (worker *MessageExportWorker) Run() {
- worker.stateMut.Lock()
+func (w *MessageExportWorker) Run() {
+ w.stateMut.Lock()
// We have to re-assign the stop channel again, because
// it might happen that the job was restarted due to a config change.
- if worker.stopped {
- worker.stopped = false
- worker.stopCh = make(chan struct{})
- worker.context, worker.cancel = context.WithCancel(context.Background())
+ if w.stopped {
+ w.stopped = false
+ w.stopCh = make(chan struct{})
+ w.context, w.cancel = context.WithCancel(context.Background())
} else {
- worker.stateMut.Unlock()
+ w.stateMut.Unlock()
return
}
// Run is called from a separate goroutine and doesn't return.
// So we cannot Unlock in a defer clause.
- worker.stateMut.Unlock()
+ w.stateMut.Unlock()
- worker.logger.Debug("Worker Started")
+ w.logger.Debug("Worker Started")
defer func() {
- worker.logger.Debug("Worker finished")
- worker.stoppedCh <- struct{}{}
+ w.logger.Debug("Worker finished")
+ w.stoppedCh <- struct{}{}
}()
for {
select {
- case <-worker.stopCh:
- worker.logger.Debug("Worker: Received stop signal")
+ case <-w.stopCh:
+ w.logger.Debug("Worker: Received stop signal")
return
- case job := <-worker.jobs:
- worker.DoJob(&job)
+ case job := <-w.jobs:
+ w.DoJob(&job)
}
}
}
-func (worker *MessageExportWorker) Stop() {
- worker.stateMut.Lock()
- defer worker.stateMut.Unlock()
+func (w *MessageExportWorker) Stop() {
+ w.stateMut.Lock()
+ defer w.stateMut.Unlock()
// Set to close, and if already closed before, then return.
- if worker.stopped {
+ if w.stopped {
return
}
- worker.stopped = true
+ w.stopped = true
- worker.logger.Debug("Worker: Stopping")
- worker.cancel()
- close(worker.stopCh)
- <-worker.stoppedCh
+ w.logger.Debug("Worker: Stopping")
+ w.cancel()
+ close(w.stopCh)
+ <-w.stoppedCh
}
-func (worker *MessageExportWorker) JobChannel() chan<- model.Job {
- return worker.jobs
+func (w *MessageExportWorker) JobChannel() chan<- model.Job {
+ return w.jobs
}
-// getExportBackend returns the file backend where the export will be created.
-func (worker *MessageExportWorker) getExportBackend(rctx request.CTX) (filestore.FileBackend, *model.AppError) {
- config := worker.jobServer.Config()
- insecure := config.ServiceSettings.EnableInsecureOutgoingConnections
-
- if config.FileSettings.DedicatedExportStore != nil && *config.FileSettings.DedicatedExportStore {
- rctx.Logger().Debug("Worker: using dedicated export filestore", mlog.String("driver_name", *config.FileSettings.ExportDriverName))
- backend, errFileBack := filestore.NewExportFileBackend(filestore.NewExportFileBackendSettingsFromConfig(&config.FileSettings, true, insecure != nil && *insecure))
- if errFileBack != nil {
- return nil, model.NewAppError("getFileBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError).Wrap(errFileBack)
- }
-
- return backend, nil
- }
-
- backend, err := filestore.NewFileBackend(filestore.NewFileBackendSettingsFromConfig(&config.FileSettings, true, insecure != nil && *insecure))
- if err != nil {
- return nil, model.NewAppError("getFileBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
- }
- return backend, nil
-}
-
-// getFileAttachmentBackend returns the file backend where file attachments are
-// located for messages that will be exported. This may be the same backend
-// where the export will be created.
-func (worker *MessageExportWorker) getFileAttachmentBackend(rctx request.CTX) (filestore.FileBackend, *model.AppError) {
- config := worker.jobServer.Config()
- insecure := config.ServiceSettings.EnableInsecureOutgoingConnections
-
- backend, err := filestore.NewFileBackend(filestore.NewFileBackendSettingsFromConfig(&config.FileSettings, true, insecure != nil && *insecure))
- if err != nil {
- return nil, model.NewAppError("getFileBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
- }
- return backend, nil
-}
-
-func (worker *MessageExportWorker) DoJob(job *model.Job) {
- logger := worker.logger.With(jobs.JobLoggerFields(job)...)
+func (w *MessageExportWorker) DoJob(job *model.Job) {
+ logger := w.logger.With(jobs.JobLoggerFields(job)...)
logger.Debug("Worker: Received a new candidate job.")
- defer worker.jobServer.HandleJobPanic(logger, job)
+ defer w.jobServer.HandleJobPanic(logger, job)
- claimed, appErr := worker.jobServer.ClaimJob(job)
+ claimed, appErr := w.jobServer.ClaimJob(job)
if appErr != nil {
logger.Info("Worker: Error occurred while trying to claim job", mlog.Err(appErr))
return
@@ -205,299 +152,171 @@ func (worker *MessageExportWorker) DoJob(job *model.Job) {
return
}
- var cancelContext request.CTX = request.EmptyContext(worker.logger)
+ var cancelContext request.CTX = request.EmptyContext(w.logger)
cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background())
cancelWatcherChan := make(chan struct{}, 1)
cancelContext = cancelContext.WithContext(cancelCtx)
- go worker.jobServer.CancellationWatcher(cancelContext, job.Id, cancelWatcherChan)
+ go w.jobServer.CancellationWatcher(cancelContext, job.Id, cancelWatcherChan)
defer cancelCancelWatcher()
// if job data is missing, we'll do our best to recover
- worker.initJobData(logger, job)
-
- // the initJobData call above populates the create_at timestamp of the first post that we should export
- // incase of job resumption or new job
- batchStartTime, err := strconv.ParseInt(job.Data[JobDataBatchStartTimestamp], 10, 64)
+ w.initJobData(logger, job, time.Now())
+ data, err := extractJobData(logger, job.Data)
if err != nil {
- worker.setJobError(logger, job, model.NewAppError("Job.DoJob", model.NoTranslation, nil, "", http.StatusBadRequest).Wrap((err)))
- return
- }
- batchStartId := job.Data[JobDataBatchStartId]
-
- jobStartTime, err := strconv.ParseInt(job.Data[JobDataStartTimestamp], 10, 64)
- if err != nil {
- worker.setJobError(logger, job, model.NewAppError("Job.DoJob", model.NoTranslation, nil, "", http.StatusBadRequest).Wrap((err)))
- return
- }
- jobStartId := job.Data[JobDataStartId]
-
- batchSize, err := strconv.Atoi(job.Data[JOB_DATA_BatchSize])
- if err != nil {
- worker.setJobError(logger, job, model.NewAppError("Job.DoJob", model.NoTranslation, nil, "", http.StatusBadRequest).Wrap((err)))
+ // Error in conversion. Not much we can do about that. But it shouldn't happen, unless someone edited the db.
+ w.setJobError(logger, job, model.NewAppError("Job.DoJob", "ent.message_export.job_data_conversion.app_error", nil, "", http.StatusBadRequest).Wrap(err))
return
}
- totalPostsExported, err := strconv.ParseInt(job.Data[JobDataMessagesExported], 10, 64)
+ rctx := request.EmptyContext(logger).WithContext(w.context)
+ reportProgress := func(message string) {
+ logger.Debug(message)
+ // Don't fail because we couldn't update progress.
+ w.setJobProgressMessage(0, message, rctx.Logger(), job)
+ }
+
+ jobParams := shared.BackendParams{
+ Config: w.jobServer.Config(),
+ Store: shared.NewMessageExportStore(w.jobServer.Store),
+ HtmlTemplates: w.htmlTemplateWatcher,
+ }
+ jobParams.FileAttachmentBackend, err = shared.GetFileAttachmentBackend(rctx, w.jobServer.Config())
if err != nil {
- worker.setJobError(logger, job, model.NewAppError("Job.DoJob", model.NoTranslation, nil, "", http.StatusBadRequest).Wrap((err)))
+ w.setJobError(logger, job, model.NewAppError("GetFileAttachmentBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError).Wrap(err))
+ return
+ }
+ jobParams.ExportBackend, err = shared.GetExportBackend(rctx, w.jobServer.Config())
+ if err != nil {
+ w.setJobError(logger, job, model.NewAppError("GetExportBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError).Wrap(err))
return
}
- var directories []string
- err = json.Unmarshal([]byte(job.Data[JobDirectories]), &directories)
+ data, err = shared.GetInitialExportPeriodData(rctx, jobParams.Store, data, reportProgress)
if err != nil {
- worker.setJobError(logger, job, model.NewAppError("Job.DoJob", model.NoTranslation, nil, "", http.StatusBadRequest).Wrap((err)))
+ w.setJobError(logger, job, model.NewAppError("DoJob", "ent.message_export.calculate_channel_exports.app_error", nil, "", http.StatusInternalServerError).Wrap(err))
return
}
+ job.Data[shared.JobDataTotalPostsExpected] = strconv.Itoa(data.TotalPostsExpected)
- // Counting all posts may fail or timeout when the posts table is large. If this happens, log a warning, but carry
- // on with the job anyway. The only issue is that the progress % reporting will be inaccurate.
- var totalPosts int64
- if count, err := worker.jobServer.Store.Post().AnalyticsPostCount(&model.PostCountOptions{ExcludeSystemPosts: true, SincePostID: jobStartId, SinceUpdateAt: jobStartTime}); err != nil {
- logger.Warn("Worker: Failed to fetch total post count for job. An estimated value will be used for progress reporting.", mlog.Err(err))
- totalPosts = estimatedPostCount
- } else {
- totalPosts = count
- }
-
- var totalWarningCount int64
- cursor := model.MessageExportCursor{LastPostUpdateAt: batchStartTime, LastPostId: batchStartId}
for {
select {
case <-cancelWatcherChan:
logger.Debug("Worker: Job has been canceled via CancellationWatcher")
- worker.setJobCanceled(logger, job)
+ w.setJobCanceled(logger, job)
return
- case <-worker.stopCh:
+ case <-w.stopCh:
logger.Debug("Worker: Job has been canceled via Worker Stop. Setting the job back to pending")
- worker.SetJobPending(logger, job)
+ w.SetJobPending(logger, job)
return
- case <-time.After(TimeBetweenBatches * time.Millisecond):
- logger.Debug("Starting batch export", mlog.Int("last_post_update_at", cursor.LastPostUpdateAt))
- rctx := request.EmptyContext(logger).WithContext(worker.context)
- prevPostUpdateAt := cursor.LastPostUpdateAt
+ case <-time.After(TimeBetweenBatchesMs * time.Millisecond):
+ logger.Debug("Starting batch export", mlog.Int("last_post_update_at", data.Cursor.LastPostUpdateAt))
- var postsExported []*model.MessageExport
- var nErr error
- postsExported, cursor, nErr = worker.jobServer.Store.Compliance().MessageExport(rctx, cursor, batchSize)
- if nErr != nil {
+ _, data, err = RunBatch(rctx, data, jobParams)
+ if err != nil {
// We ignore error if the job was explicitly cancelled
- // and let it
- if worker.context.Err() == context.Canceled {
+ if errors.Is(w.context.Err(), context.Canceled) {
logger.Debug("Worker: Job has been canceled via worker's context. Setting the job back to pending")
- worker.SetJobPending(logger, job)
+ w.SetJobPending(logger, job)
} else {
- worker.setJobError(logger, job, model.NewAppError("DoJob", "ent.message_export.run_export.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr))
- }
- return
- }
- logger.Debug("Found posts to export", mlog.Int("number_of_posts", len(postsExported)))
- totalPostsExported += int64(len(postsExported))
- job.Data[JobDataMessagesExported] = strconv.FormatInt(totalPostsExported, 10)
- job.Data[JobDataBatchStartTimestamp] = strconv.FormatInt(cursor.LastPostUpdateAt, 10)
- job.Data[JobDataBatchStartId] = cursor.LastPostId
-
- if len(postsExported) == 0 {
- job.Data[JobDataWarningCount] = strconv.FormatInt(totalWarningCount, 10)
- // we've exported everything up to the current time
- logger.Debug("FormatExport complete")
-
- // Create downloadable zip file of all batches.
- if job.Data[JobDataExportType] != model.ComplianceExportTypeGlobalrelay {
- exportBackend, err := worker.getExportBackend(rctx)
- if err != nil {
- worker.setJobError(logger, job, err)
- return
- }
-
- zipErr := createZipFile(rctx, exportBackend, job.Id, directories)
- if zipErr != nil {
- logger.Error("Error creating zip file for export", mlog.Err(zipErr))
- job.Data[JobDataIsDownloadable] = "false"
- } else {
- job.Data[JobDataIsDownloadable] = "true"
- }
- }
- if totalWarningCount > 0 {
- worker.setJobWarning(logger, job)
- } else {
- worker.setJobSuccess(logger, job)
+ w.setJobError(logger, job, model.NewAppError("DoJob", "ent.message_export.run_export.app_error", nil, "", http.StatusInternalServerError).Wrap(err))
}
return
}
- exportBackend, err := worker.getExportBackend(rctx)
- if err != nil {
- worker.setJobError(logger, job, err)
+ setJobDataEndOfBatch(job, data)
+
+ if data.Finished {
+ w.finishExport(rctx, logger, job, data.WarningCount)
return
}
- fileAttachmentBackend, err := worker.getFileAttachmentBackend(rctx)
- if err != nil {
- worker.setJobError(logger, job, err)
+ // also saves job.Data
+ if err := w.setJobProgress(logger, job, getJobProgress(data.MessagesExported, data.TotalPostsExpected)); err != nil {
+ // TODO: MM-59093 handle job errors (robust, recoverable)
return
}
- batchDirectory := getOutputDirectoryPath(prevPostUpdateAt, cursor.LastPostUpdateAt)
- warningCount, err := runExportByType(
- rctx,
- job.Data[JobDataExportType],
- postsExported,
- batchDirectory,
- worker.jobServer.Store,
- exportBackend,
- fileAttachmentBackend,
- worker.htmlTemplateWatcher,
- worker.jobServer.Config(),
- )
- if err != nil {
- worker.setJobError(logger, job, err)
- return
- }
-
- totalWarningCount += warningCount
-
- directories = append(directories, batchDirectory)
- directoriesBytes, e := json.Marshal(directories)
- if e != nil {
- worker.setJobError(logger, job, model.NewAppError("Job.DoJob", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap((e)))
- return
- }
- job.Data[JobDirectories] = string(directoriesBytes)
-
- // also saves the last post create time
- if err := worker.jobServer.SetJobProgress(job, getJobProgress(totalPostsExported, totalPosts)); err != nil {
- worker.setJobError(logger, job, err)
- return
+ // testEndOfBatchCb is only used by tests.
+ if testEndOfBatchCb != nil {
+ testEndOfBatchCb(w)
}
}
}
}
-func createZipFile(rctx request.CTX, fileBackend filestore.FileBackend, jobId string, directories []string) error {
- zipFileName := jobId + ".zip"
+func (w *MessageExportWorker) finishExport(rctx request.CTX, logger *mlog.Logger, job *model.Job, totalWarningCount int) {
+ job.Data[shared.JobDataWarningCount] = strconv.Itoa(totalWarningCount)
+ // we've exported everything up to the current time
+ logger.Debug("FormatExport complete")
- dest, err := os.CreateTemp("", zipFileName)
- if err != nil {
- return err
+ job.Data[shared.JobDataIsDownloadable] = "false"
+
+ if totalWarningCount > 0 {
+ w.setJobWarning(logger, job)
+ } else {
+ w.setJobSuccess(logger, job)
}
- defer os.Remove(dest.Name())
-
- // Create a new zip archive.
- w := zip.NewWriter(dest)
-
- // create a 32 KiB buffer for copying files
- buf := make([]byte, 32*1024)
-
- // Add directories to the archive.
- for _, directory := range directories {
- err = addFiles(w, fileBackend, directory, buf)
- if err != nil {
- return err
- }
- }
-
- // Make sure to check the error on Close.
- err = w.Close()
- if err != nil {
- return fmt.Errorf("error closing zip file: %s %v", dest.Name(), err)
- }
-
- _, err = dest.Seek(0, 0)
- if err != nil {
- return fmt.Errorf("error seeking zip file: %s %v", dest.Name(), err)
- }
-
- zipPath := path.Join(exportPath, zipFileName)
-
- // If the file backend allows it, we want to upload without a timeout
- _, err = filestore.TryWriteFileContext(rctx.Context(), fileBackend, dest, zipPath)
- return err
-}
-
-func addFiles(w *zip.Writer, fileBackend filestore.FileBackend, basePath string, buf []byte) error {
- // Open the Directory
- files, err := fileBackend.ListDirectoryRecursively(basePath)
- if err != nil {
- return err
- }
-
- for _, file := range files {
- err = addFile(w, fileBackend, file, basePath, buf)
- if err != nil {
- return err
- }
- }
-
- return nil
-}
-
-func addFile(w *zip.Writer, fileBackend filestore.FileBackend, file, basePath string, buf []byte) error {
- // In some storage backends like Hitachi HCP, the first entry
- // from a ListObjects API is always the dir entry itself.
- if file == basePath {
- return nil
- }
-
- size, err := fileBackend.FileSize(file)
- if err != nil {
- return fmt.Errorf("error reading file size for %s: %w", file, err)
- }
- if size == 0 {
- // skip empty files
- return nil
- }
-
- r, err := fileBackend.Reader(file)
- if err != nil {
- return fmt.Errorf("error opening file %s: %w", file, err)
- }
- defer r.Close()
-
- // Add some files to the archive.
- f, err := w.Create(file)
- if err != nil {
- return fmt.Errorf("error creating file %s in the archive: %w", file, err)
- }
- _, err = io.CopyBuffer(f, r, buf)
- if err != nil {
- return fmt.Errorf("error copying file %s into the archive: %w", file, err)
- }
-
- return nil
}
// initializes job data if it's missing, allows us to recover from failed or improperly configured jobs
-func (worker *MessageExportWorker) initJobData(logger mlog.LoggerIFace, job *model.Job) {
+func (w *MessageExportWorker) initJobData(logger mlog.LoggerIFace, job *model.Job, now time.Time) {
if job.Data == nil {
job.Data = make(map[string]string)
}
- if _, exists := job.Data[JobDataMessagesExported]; !exists {
- job.Data[JobDataMessagesExported] = "0"
+ if _, exists := job.Data[shared.JobDataMessagesExported]; !exists {
+ logger.Info("Worker: JobDataMessagesExported does not exist, starting at 0")
+ job.Data[shared.JobDataMessagesExported] = "0"
}
- if _, exists := job.Data[JobDirectories]; !exists {
- // json null value
- job.Data[JobDirectories] = "null"
+ if _, exists := job.Data[shared.JobDataExportType]; !exists {
+ exportFormat := *w.jobServer.Config().MessageExportSettings.ExportFormat
+ logger.Info("Worker: Defaulting to configured export format", mlog.String("export_format", exportFormat))
+ job.Data[shared.JobDataExportType] = exportFormat
}
- if _, exists := job.Data[JobDataExportType]; !exists {
- // for now, we'll default to Actiance. When we support multiple export types, we'll have to fetch it from config instead
- logger.Info("Worker: Defaulting to configured export format")
- job.Data[JobDataExportType] = *worker.jobServer.Config().MessageExportSettings.ExportFormat
+ if _, exists := job.Data[shared.JobDataBatchSize]; !exists {
+ batchSize := strconv.Itoa(*w.jobServer.Config().MessageExportSettings.BatchSize)
+ logger.Info("Worker: Defaulting to configured batch size", mlog.String("batch_size", batchSize))
+ job.Data[shared.JobDataBatchSize] = batchSize
}
- if _, exists := job.Data[JOB_DATA_BatchSize]; !exists {
- logger.Info("Worker: Defaulting to configured batch size")
- job.Data[JOB_DATA_BatchSize] = strconv.Itoa(*worker.jobServer.Config().MessageExportSettings.BatchSize)
+ if _, exists := job.Data[shared.JobDataChannelBatchSize]; !exists {
+ channelBatchSize := strconv.Itoa(*w.jobServer.Config().MessageExportSettings.ChannelBatchSize)
+ logger.Info("Worker: Defaulting to configured channel batch size", mlog.String("channel_batch_size", channelBatchSize))
+ job.Data[shared.JobDataChannelBatchSize] = channelBatchSize
}
- if _, exists := job.Data[JobDataBatchStartTimestamp]; !exists {
- previousJob, err := worker.jobServer.Store.Job().GetNewestJobByStatusesAndType([]string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport)
+ if _, exists := job.Data[shared.JobDataChannelHistoryBatchSize]; !exists {
+ channelHistoryBatchSize := strconv.Itoa(*w.jobServer.Config().MessageExportSettings.ChannelHistoryBatchSize)
+ logger.Info("Worker: Defaulting to configured channel history batch size", mlog.String("channel_history_batch_size", channelHistoryBatchSize))
+ job.Data[shared.JobDataChannelHistoryBatchSize] = channelHistoryBatchSize
+ }
+ if _, exists := job.Data[shared.JobDataBatchNumber]; !exists {
+ logger.Info("Worker: JobDataBatchNumber does not exist, starting at 0")
+ job.Data[shared.JobDataBatchNumber] = "0"
+ }
+
+ // If this is a new job (JobEndTime doesn't exist), set it to now, because this is when the job has first started.
+ // The logic is that a job exports messages up to the moment the job was started. If the job was picked up after
+ // gracefully stopping, then run it until that original initial endTime.
+ // However, if the job was cancelled or errored out, that job will not be picked up again, so this will be a new job
+ // starting from the last successful batchStartTimestamp up until now. This is intentional (for now) because failed
+ // jobs do not get rescheduled properly yet, and when they are run again it means that new day's worth of messages
+ // need to be exported.
+ if _, exists := job.Data[shared.JobDataJobEndTime]; !exists {
+ millis := strconv.FormatInt(model.GetMillisForTime(now), 10)
+ logger.Info("Worker: JobDataJobEndTime not found in previous job, using now", mlog.String("job_data_job_end_time", millis))
+ job.Data[shared.JobDataJobEndTime] = millis
+ }
+
+ if _, exists := job.Data[shared.JobDataBatchStartTime]; !exists {
+ previousJob, err := w.jobServer.Store.Job().GetNewestJobByStatusesAndType([]string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport)
if err != nil {
- logger.Info("Worker: No previously successful job found, falling back to configured MessageExportSettings.ExportFromTimestamp")
- job.Data[JobDataBatchStartTimestamp] = strconv.FormatInt(*worker.jobServer.Config().MessageExportSettings.ExportFromTimestamp, 10)
- job.Data[JobDataBatchStartId] = ""
- job.Data[JobDataStartTimestamp] = job.Data[JobDataBatchStartTimestamp]
- job.Data[JobDataStartId] = job.Data[JobDataBatchStartId]
+ exportFromTimestamp := strconv.FormatInt(*w.jobServer.Config().MessageExportSettings.ExportFromTimestamp, 10)
+ logger.Info("Worker: No previously successful job found, falling back to configured MessageExportSettings.ExportFromTimestamp", mlog.String("export_from_timestamp", exportFromTimestamp))
+ job.Data[shared.JobDataBatchStartTime] = exportFromTimestamp
+ job.Data[shared.JobDataJobStartTime] = exportFromTimestamp
+ job.Data[shared.JobDataBatchStartId] = ""
+ job.Data[shared.JobDataJobStartId] = job.Data[shared.JobDataBatchStartId]
+ job.Data[shared.JobDataExportDir] = getJobExportDir(logger, job.Data, exportFromTimestamp, job.Data[shared.JobDataJobEndTime])
return
}
@@ -508,72 +327,174 @@ func (worker *MessageExportWorker) initJobData(logger mlog.LoggerIFace, job *mod
if previousJob.Data == nil {
previousJob.Data = make(map[string]string)
}
- if _, prevExists := previousJob.Data[JobDataBatchStartTimestamp]; !prevExists {
- logger.Info("Worker: Previously successful job lacks job data, falling back to configured MessageExportSettings.ExportFromTimestamp")
- job.Data[JobDataBatchStartTimestamp] = strconv.FormatInt(*worker.jobServer.Config().MessageExportSettings.ExportFromTimestamp, 10)
+ if _, prevExists := previousJob.Data[shared.JobDataBatchStartTime]; !prevExists {
+ exportFromTimestamp := strconv.FormatInt(*w.jobServer.Config().MessageExportSettings.ExportFromTimestamp, 10)
+ logger.Info("Worker: Previously successful job lacks job data, falling back to configured MessageExportSettings.ExportFromTimestamp", mlog.String("export_from_timestamp", exportFromTimestamp))
+ job.Data[shared.JobDataBatchStartTime] = exportFromTimestamp
+ job.Data[shared.JobDataJobStartTime] = exportFromTimestamp
} else {
- job.Data[JobDataBatchStartTimestamp] = previousJob.Data[JobDataBatchStartTimestamp]
+ job.Data[shared.JobDataBatchStartTime] = previousJob.Data[shared.JobDataBatchStartTime]
}
- if _, prevExists := previousJob.Data[JobDataBatchStartId]; !prevExists {
+ if _, prevExists := previousJob.Data[shared.JobDataBatchStartId]; !prevExists {
logger.Info("Worker: Previously successful job lacks post ID, falling back to empty string")
- job.Data[JobDataBatchStartId] = ""
+ job.Data[shared.JobDataBatchStartId] = ""
} else {
- job.Data[JobDataBatchStartId] = previousJob.Data[JobDataBatchStartId]
+ job.Data[shared.JobDataBatchStartId] = previousJob.Data[shared.JobDataBatchStartId]
}
- job.Data[JobDataStartTimestamp] = job.Data[JobDataBatchStartTimestamp]
- job.Data[JobDataStartId] = job.Data[JobDataBatchStartId]
+ job.Data[shared.JobDataJobStartId] = job.Data[shared.JobDataBatchStartId]
} else {
- logger.Info("Worker: FormatExport start time explicitly set", mlog.String("new_start_time", job.Data[JobDataBatchStartTimestamp]))
+ logger.Info("Worker: JobDataBatchStartTime start time was already set",
+ mlog.String(shared.JobDataBatchStartTime, job.Data[shared.JobDataBatchStartTime]))
}
+
+ if _, exists := job.Data[shared.JobDataJobStartTime]; !exists {
+ // Just in case, if we don't have this (JobDataBatchStartTime was already set, but this wasn't) set it:
+ job.Data[shared.JobDataJobStartTime] = job.Data[shared.JobDataBatchStartTime]
+ logger.Info("Worker: JobDataJobStartTime start time was not set, using batch startTimestamp",
+ mlog.String(shared.JobDataJobStartTime, job.Data[shared.JobDataJobStartTime]))
+ }
+
+ job.Data[shared.JobDataExportDir] = getJobExportDir(logger, job.Data, job.Data[shared.JobDataJobStartTime], job.Data[shared.JobDataJobEndTime])
}
-func getJobProgress(totalExportedPosts, totalPosts int64) int64 {
- return totalExportedPosts * 100 / totalPosts
+func extractJobData(logger *mlog.Logger, strmap map[string]string) (shared.JobData, error) {
+ data, err := shared.StringMapToJobDataWithZeroValues(strmap)
+ if err != nil {
+ return data, err
+ }
+
+ // ExportPeriodStartTime is initialized to BatchStartTime because this is where we will start exporting. But unlike
+ // BatchStartTime, it won't change as we process the batches.
+ // If this is the first time this job has run, BatchStartTime will be the start of the entire job. If this job has
+ // been resumed, then BatchStartTime will be the start of the newest batch. This is expected--the channel activity
+ // and total posts will be calculated from ExportPeriodStartTime (anything earlier has already been exported in
+ // previous batches).
+ // Note: ExportPeriodStartTime is different from JobStartTime because JobStartTime won't change
+ // if the job processes some batches, is stopped, and picked up again.
+ data.ExportPeriodStartTime = data.BatchStartTime
+
+ logger.Info("Worker: initial job variables set",
+ mlog.String("export_type", data.ExportType),
+ mlog.String("export_dir", data.ExportDir),
+ mlog.Int("job_start_time", data.JobStartTime),
+ mlog.Int("batch_start_time", data.BatchStartTime),
+ mlog.Int("export_period_start_time", data.ExportPeriodStartTime),
+ mlog.Int("job_end_time", data.JobEndTime),
+ mlog.String("job_start_id", data.JobStartId),
+ mlog.Int("batch_size", data.BatchSize),
+ mlog.Int("channel_batch_size", data.ChannelBatchSize),
+ mlog.Int("channel_history_batch_size", data.ChannelHistoryBatchSize),
+ mlog.Int("batch_number", data.BatchNumber),
+ mlog.Int("total_posts_exported", data.MessagesExported))
+
+ return data, err
}
-func (worker *MessageExportWorker) setJobSuccess(logger mlog.LoggerIFace, job *model.Job) {
- // setting progress causes the job data to be saved, which is necessary if we want the next job to pick up where this one left off
- if err := worker.jobServer.SetJobProgress(job, 100); err != nil {
+func setJobDataEndOfBatch(job *model.Job, data shared.JobData) {
+ job.Data[shared.JobDataBatchStartTime] = strconv.FormatInt(data.BatchStartTime, 10)
+ job.Data[shared.JobDataBatchStartId] = data.Cursor.LastPostId
+ job.Data[shared.JobDataMessagesExported] = strconv.Itoa(data.MessagesExported)
+ job.Data[shared.JobDataBatchNumber] = strconv.Itoa(data.BatchNumber)
+}
+
+// getJobExportDir will use the existing JobDataExportDir if available. If it's not available, this is the first run
+// for the job, so we use the startTime and endTime passed in.
+func getJobExportDir(logger mlog.LoggerIFace, data model.StringMap, startTime string, endTime string) string {
+ exportDir, exists := data[shared.JobDataExportDir]
+ if !exists {
+ // If we don't have a jobDataExportDir, this is the first run for the job, so we use the batch startTime
+ exportDir = path.Join(model.ComplianceExportPath, fmt.Sprintf("%s-%s-%s", time.Now().Format(model.ComplianceExportDirectoryFormat), startTime, endTime))
+ logger.Info("Worker: JobDataExportDir does not exist, using current datetime", mlog.String("job_data_export_dir", exportDir))
+ }
+
+ return exportDir
+}
+
+func getJobProgress(totalExportedPosts, totalPostsExpected int) int {
+ return totalExportedPosts * 100 / totalPostsExpected
+}
+
+func (w *MessageExportWorker) setJobProgressMessage(progress int64, message string, logger mlog.LoggerIFace, job *model.Job) {
+ job.Status = model.JobStatusInProgress
+ job.Progress = progress
+ if job.Data == nil {
+ job.Data = make(map[string]string)
+ }
+ job.Data["progress_message"] = message
+
+ if _, err := w.jobServer.Store.Job().UpdateOptimistically(job, model.JobStatusInProgress); err != nil {
logger.Error("Worker: Failed to update progress for job", mlog.Err(err))
- worker.setJobError(logger, job, err)
}
- if err := worker.jobServer.SetJobSuccess(job); err != nil {
+}
+
+func (w *MessageExportWorker) setJobProgress(logger mlog.LoggerIFace, job *model.Job, progress int) error {
+ if job.Data != nil {
+ job.Data["progress_message"] = ""
+ }
+
+ if err := w.jobServer.SetJobProgress(job, int64(progress)); err != nil {
+ logger.Error("Worker: Failed to update progress for job", mlog.Err(err))
+ w.setJobError(logger, job, err)
+ return err
+ }
+
+ return nil
+}
+
+func (w *MessageExportWorker) setJobSuccess(logger mlog.LoggerIFace, job *model.Job) {
+ // setting progress causes the job data to be saved, which is necessary if we want the next job to pick up where this one left off
+ if job.Data != nil {
+ job.Data["progress_message"] = ""
+ }
+ if err := w.jobServer.SetJobProgress(job, 100); err != nil {
+ logger.Error("Worker: Failed to update progress for job", mlog.Err(err))
+ w.setJobError(logger, job, err)
+ }
+ if err := w.jobServer.SetJobSuccess(job); err != nil {
logger.Error("Worker: Failed to set success for job", mlog.Err(err))
- worker.setJobError(logger, job, err)
+ w.setJobError(logger, job, err)
}
}
-func (worker *MessageExportWorker) setJobWarning(logger mlog.LoggerIFace, job *model.Job) {
+func (w *MessageExportWorker) setJobWarning(logger mlog.LoggerIFace, job *model.Job) {
// setting progress causes the job data to be saved, which is necessary if we want the next job to pick up where this one left off
- if err := worker.jobServer.SetJobProgress(job, 100); err != nil {
+ if job.Data != nil {
+ job.Data["progress_message"] = ""
+ }
+ if err := w.jobServer.SetJobProgress(job, 100); err != nil {
logger.Error("Worker: Failed to update progress for job", mlog.Err(err))
- worker.setJobError(logger, job, err)
+ w.setJobError(logger, job, err)
}
- if err := worker.jobServer.SetJobWarning(job); err != nil {
+ if err := w.jobServer.SetJobWarning(job); err != nil {
logger.Error("Worker: Failed to set warning for job", mlog.Err(err))
- worker.setJobError(logger, job, err)
+ w.setJobError(logger, job, err)
}
}
-func (worker *MessageExportWorker) setJobError(logger mlog.LoggerIFace, job *model.Job, appError *model.AppError) {
+func (w *MessageExportWorker) setJobError(logger mlog.LoggerIFace, job *model.Job, appError *model.AppError) {
+ if job.Data != nil {
+ job.Data["progress_message"] = ""
+ }
logger.Error("Worker: Job error", mlog.Err(appError))
- if err := worker.jobServer.SetJobError(job, appError); err != nil {
- logger.Error("Worker: Failed to set job errorv", mlog.Err(err), mlog.NamedErr("set_error", appError))
+ if err := w.jobServer.SetJobError(job, appError); err != nil {
+ logger.Error("Worker: Failed to set job error", mlog.Err(err), mlog.NamedErr("set_error", appError))
}
}
-func (worker *MessageExportWorker) setJobCanceled(logger mlog.LoggerIFace, job *model.Job) {
- if err := worker.jobServer.SetJobCanceled(job); err != nil {
+func (w *MessageExportWorker) setJobCanceled(logger mlog.LoggerIFace, job *model.Job) {
+ if job.Data != nil {
+ job.Data["progress_message"] = ""
+ }
+ if err := w.jobServer.SetJobCanceled(job); err != nil {
logger.Error("Worker: Failed to mark job as canceled", mlog.Err(err))
}
}
-func (worker *MessageExportWorker) SetJobPending(logger mlog.LoggerIFace, job *model.Job) {
- if err := worker.jobServer.SetJobPending(job); err != nil {
+func (w *MessageExportWorker) SetJobPending(logger mlog.LoggerIFace, job *model.Job) {
+ if job.Data != nil {
+ job.Data["progress_message"] = ""
+ }
+ if err := w.jobServer.SetJobPending(job); err != nil {
logger.Error("Worker: Failed to mark job as pending", mlog.Err(err))
}
}
-
-func getOutputDirectoryPath(exportStartTime int64, exportEndTime int64) string {
- return path.Join(exportPath, strconv.FormatInt(exportStartTime, 10)+"-"+strconv.FormatInt(exportEndTime, 10))
-}
diff --git a/server/enterprise/message_export/worker_test.go b/server/enterprise/message_export/worker_test.go
index 68a74285c3..08baa5cf59 100644
--- a/server/enterprise/message_export/worker_test.go
+++ b/server/enterprise/message_export/worker_test.go
@@ -4,14 +4,14 @@
package message_export
import (
- "archive/zip"
- "bytes"
"context"
- "net/http"
+ "errors"
+ "fmt"
"os"
"path"
"strconv"
"testing"
+ "time"
"github.com/stretchr/testify/assert"
tmock "github.com/stretchr/testify/mock"
@@ -24,10 +24,10 @@ import (
"github.com/mattermost/mattermost/server/v8/channels/app"
"github.com/mattermost/mattermost/server/v8/channels/jobs"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
+ st "github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/utils/testutils"
"github.com/mattermost/mattermost/server/v8/einterfaces/mocks"
- "github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
- fmocks "github.com/mattermost/mattermost/server/v8/platform/shared/filestore/mocks"
+ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
)
func TestInitJobDataNoJobData(t *testing.T) {
@@ -36,14 +36,14 @@ func TestInitJobDataNoJobData(t *testing.T) {
defer mockStore.AssertExpectations(t)
job := &model.Job{
- Id: model.NewId(),
+ Id: st.NewTestID(),
CreateAt: model.GetMillis(),
Status: model.JobStatusPending,
Type: model.JobTypeMessageExport,
}
// mock job store doesn't return a previously successful job, forcing fallback to config
- mockStore.JobStore.On("GetNewestJobByStatusesAndType", []string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport).Return(nil, model.NewAppError("", "", nil, "", http.StatusBadRequest))
+ mockStore.JobStore.On("GetNewestJobByStatusesAndType", []string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport).Return(nil, errors.New("test"))
worker := &MessageExportWorker{
jobServer: &jobs.JobServer{
@@ -52,11 +52,13 @@ func TestInitJobDataNoJobData(t *testing.T) {
Cfg: &model.Config{
// mock config
MessageExportSettings: model.MessageExportSettings{
- EnableExport: model.NewPointer(true),
- ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
- DailyRunTime: model.NewPointer("01:00"),
- ExportFromTimestamp: model.NewPointer(int64(0)),
- BatchSize: model.NewPointer(10000),
+ EnableExport: model.NewPointer(true),
+ ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
+ DailyRunTime: model.NewPointer("01:00"),
+ ExportFromTimestamp: model.NewPointer(int64(0)),
+ BatchSize: model.NewPointer(10000),
+ ChannelBatchSize: model.NewPointer(100),
+ ChannelHistoryBatchSize: model.NewPointer(100),
},
},
},
@@ -64,12 +66,14 @@ func TestInitJobDataNoJobData(t *testing.T) {
logger: logger,
}
- // actually execute the code under test
- worker.initJobData(logger, job)
+ now := time.Now()
+ worker.initJobData(logger, job, now)
- assert.Equal(t, model.ComplianceExportTypeActiance, job.Data[JobDataExportType])
- assert.Equal(t, strconv.Itoa(*worker.jobServer.Config().MessageExportSettings.BatchSize), job.Data[JOB_DATA_BatchSize])
- assert.Equal(t, strconv.FormatInt(*worker.jobServer.Config().MessageExportSettings.ExportFromTimestamp, 10), job.Data[JobDataBatchStartTimestamp])
+ assert.Equal(t, model.ComplianceExportTypeActiance, job.Data[shared.JobDataExportType])
+ assert.Equal(t, strconv.Itoa(*worker.jobServer.Config().MessageExportSettings.BatchSize), job.Data[shared.JobDataBatchSize])
+ assert.Equal(t, strconv.FormatInt(*worker.jobServer.Config().MessageExportSettings.ExportFromTimestamp, 10), job.Data[shared.JobDataBatchStartTime])
+ expectedDir := path.Join(model.ComplianceExportPath, fmt.Sprintf("%s-%d-%d", now.Format(model.ComplianceExportDirectoryFormat), 0, now.UnixMilli()))
+ assert.Equal(t, expectedDir, job.Data[shared.JobDataExportDir])
}
func TestInitJobDataPreviousJobNoJobData(t *testing.T) {
@@ -78,7 +82,7 @@ func TestInitJobDataPreviousJobNoJobData(t *testing.T) {
defer mockStore.AssertExpectations(t)
previousJob := &model.Job{
- Id: model.NewId(),
+ Id: st.NewTestID(),
CreateAt: model.GetMillis(),
Status: model.JobStatusSuccess,
Type: model.JobTypeMessageExport,
@@ -87,7 +91,7 @@ func TestInitJobDataPreviousJobNoJobData(t *testing.T) {
}
job := &model.Job{
- Id: model.NewId(),
+ Id: st.NewTestID(),
CreateAt: model.GetMillis(),
Status: model.JobStatusPending,
Type: model.JobTypeMessageExport,
@@ -103,11 +107,13 @@ func TestInitJobDataPreviousJobNoJobData(t *testing.T) {
Cfg: &model.Config{
// mock config
MessageExportSettings: model.MessageExportSettings{
- EnableExport: model.NewPointer(true),
- ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
- DailyRunTime: model.NewPointer("01:00"),
- ExportFromTimestamp: model.NewPointer(int64(0)),
- BatchSize: model.NewPointer(10000),
+ EnableExport: model.NewPointer(true),
+ ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
+ DailyRunTime: model.NewPointer("01:00"),
+ ExportFromTimestamp: model.NewPointer(int64(0)),
+ BatchSize: model.NewPointer(10000),
+ ChannelBatchSize: model.NewPointer(100),
+ ChannelHistoryBatchSize: model.NewPointer(100),
},
},
},
@@ -115,12 +121,14 @@ func TestInitJobDataPreviousJobNoJobData(t *testing.T) {
logger: logger,
}
- // actually execute the code under test
- worker.initJobData(logger, job)
+ now := time.Now()
+ worker.initJobData(logger, job, now)
- assert.Equal(t, model.ComplianceExportTypeActiance, job.Data[JobDataExportType])
- assert.Equal(t, strconv.Itoa(*worker.jobServer.Config().MessageExportSettings.BatchSize), job.Data[JOB_DATA_BatchSize])
- assert.Equal(t, strconv.FormatInt(*worker.jobServer.Config().MessageExportSettings.ExportFromTimestamp, 10), job.Data[JobDataBatchStartTimestamp])
+ assert.Equal(t, model.ComplianceExportTypeActiance, job.Data[shared.JobDataExportType])
+ assert.Equal(t, strconv.Itoa(*worker.jobServer.Config().MessageExportSettings.BatchSize), job.Data[shared.JobDataBatchSize])
+ assert.Equal(t, strconv.FormatInt(*worker.jobServer.Config().MessageExportSettings.ExportFromTimestamp, 10), job.Data[shared.JobDataBatchStartTime])
+ expectedDir := path.Join(model.ComplianceExportPath, fmt.Sprintf("%s-%d-%d", now.Format(model.ComplianceExportDirectoryFormat), 0, now.UnixMilli()))
+ assert.Equal(t, expectedDir, job.Data[shared.JobDataExportDir])
}
func TestInitJobDataPreviousJobWithJobData(t *testing.T) {
@@ -129,20 +137,21 @@ func TestInitJobDataPreviousJobWithJobData(t *testing.T) {
defer mockStore.AssertExpectations(t)
previousJob := &model.Job{
- Id: model.NewId(),
+ Id: st.NewTestID(),
CreateAt: model.GetMillis(),
Status: model.JobStatusSuccess,
Type: model.JobTypeMessageExport,
StartAt: model.GetMillis() - 1000,
LastActivityAt: model.GetMillis() - 1000,
- Data: map[string]string{JobDataBatchStartTimestamp: "123"},
+ Data: map[string]string{shared.JobDataBatchStartTime: "123"},
}
job := &model.Job{
- Id: model.NewId(),
+ Id: st.NewTestID(),
CreateAt: model.GetMillis(),
Status: model.JobStatusPending,
Type: model.JobTypeMessageExport,
+ Data: map[string]string{shared.JobDataExportDir: "this-is-the-export-dir"},
}
// mock job store returns a previously successful job that has the config that we're looking for, so we use it
@@ -155,11 +164,13 @@ func TestInitJobDataPreviousJobWithJobData(t *testing.T) {
Cfg: &model.Config{
// mock config
MessageExportSettings: model.MessageExportSettings{
- EnableExport: model.NewPointer(true),
- ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
- DailyRunTime: model.NewPointer("01:00"),
- ExportFromTimestamp: model.NewPointer(int64(0)),
- BatchSize: model.NewPointer(10000),
+ EnableExport: model.NewPointer(true),
+ ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
+ DailyRunTime: model.NewPointer("01:00"),
+ ExportFromTimestamp: model.NewPointer(int64(0)),
+ BatchSize: model.NewPointer(10000),
+ ChannelBatchSize: model.NewPointer(100),
+ ChannelHistoryBatchSize: model.NewPointer(100),
},
},
},
@@ -167,12 +178,14 @@ func TestInitJobDataPreviousJobWithJobData(t *testing.T) {
logger: logger,
}
- // actually execute the code under test
- worker.initJobData(logger, job)
+ now := time.Now()
+ worker.initJobData(logger, job, now)
- assert.Equal(t, model.ComplianceExportTypeActiance, job.Data[JobDataExportType])
- assert.Equal(t, strconv.Itoa(*worker.jobServer.Config().MessageExportSettings.BatchSize), job.Data[JOB_DATA_BatchSize])
- assert.Equal(t, previousJob.Data[JobDataBatchStartTimestamp], job.Data[JobDataBatchStartTimestamp])
+ assert.Equal(t, model.ComplianceExportTypeActiance, job.Data[shared.JobDataExportType])
+ assert.Equal(t, strconv.Itoa(*worker.jobServer.Config().MessageExportSettings.BatchSize), job.Data[shared.JobDataBatchSize])
+ assert.Equal(t, previousJob.Data[shared.JobDataBatchStartTime], job.Data[shared.JobDataBatchStartTime])
+ expectedDir := "this-is-the-export-dir"
+ assert.Equal(t, expectedDir, job.Data[shared.JobDataExportDir])
}
func TestDoJobNoPostsToExport(t *testing.T) {
@@ -185,7 +198,7 @@ func TestDoJobNoPostsToExport(t *testing.T) {
defer mockMetrics.AssertExpectations(t)
job := &model.Job{
- Id: model.NewId(),
+ Id: st.NewTestID(),
CreateAt: model.GetMillis(),
Status: model.JobStatusPending,
Type: model.JobTypeMessageExport,
@@ -196,15 +209,19 @@ func TestDoJobNoPostsToExport(t *testing.T) {
mockMetrics.On("IncrementJobActive", model.JobTypeMessageExport)
// no previous job, data will be loaded from config
- mockStore.JobStore.On("GetNewestJobByStatusesAndType", []string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport).Return(nil, model.NewAppError("", "", nil, "", http.StatusBadRequest))
+ mockStore.JobStore.On("GetNewestJobByStatusesAndType", []string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport).Return(nil, errors.New("test"))
+
+ // no channels with activity
+ mockStore.ChannelMemberHistoryStore.On("GetChannelsWithActivityDuring", mock.Anything, mock.Anything).
+ Return(make([]string, 0), nil)
// no posts found to export
- mockStore.ComplianceStore.On("MessageExport", mock.Anything, mock.AnythingOfType("model.MessageExportCursor"), 10000).Return(
+ mockStore.ComplianceStore.On("MessageExport", mock.Anything, mock.AnythingOfType("model.MessageExportCursor"), 10001).Return(
make([]*model.MessageExport, 0), model.MessageExportCursor{}, nil,
)
mockStore.PostStore.On("AnalyticsPostCount", mock.Anything).Return(
- int64(estimatedPostCount), nil,
+ int64(shared.EstimatedPostCount), nil,
)
// job completed successfully
@@ -229,11 +246,13 @@ func TestDoJobNoPostsToExport(t *testing.T) {
Directory: model.NewPointer(tempDir),
},
MessageExportSettings: model.MessageExportSettings{
- EnableExport: model.NewPointer(true),
- ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
- DailyRunTime: model.NewPointer("01:00"),
- ExportFromTimestamp: model.NewPointer(int64(0)),
- BatchSize: model.NewPointer(10000),
+ EnableExport: model.NewPointer(true),
+ ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
+ DailyRunTime: model.NewPointer("01:00"),
+ ExportFromTimestamp: model.NewPointer(int64(0)),
+ BatchSize: model.NewPointer(10000),
+ ChannelBatchSize: model.NewPointer(100),
+ ChannelHistoryBatchSize: model.NewPointer(100),
},
},
},
@@ -258,7 +277,7 @@ func TestDoJobWithDedicatedExportBackend(t *testing.T) {
defer mockMetrics.AssertExpectations(t)
job := &model.Job{
- Id: model.NewId(),
+ Id: st.NewTestID(),
CreateAt: model.GetMillis(),
Status: model.JobStatusPending,
Type: model.JobTypeMessageExport,
@@ -269,15 +288,50 @@ func TestDoJobWithDedicatedExportBackend(t *testing.T) {
mockMetrics.On("IncrementJobActive", model.JobTypeMessageExport)
// no previous job, data will be loaded from config
- mockStore.JobStore.On("GetNewestJobByStatusesAndType", []string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport).Return(nil, model.NewAppError("", "", nil, "", http.StatusBadRequest))
+ mockStore.JobStore.On("GetNewestJobByStatusesAndType", []string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport).Return(nil, errors.New("test"))
- // no posts found to export
- mockStore.ComplianceStore.On("MessageExport", mock.Anything, mock.AnythingOfType("model.MessageExportCursor"), 10000).Return(
+ channelId := st.NewTestID()
+ channelName := st.NewTestID()
+ channelDisplayName := st.NewTestID()
+ channelType := model.ChannelTypeOpen
+ messages := []*model.MessageExport{
+ {
+ TeamId: model.NewPointer(st.NewTestID()),
+ ChannelId: model.NewPointer(channelId),
+ ChannelName: model.NewPointer(channelName),
+ UserId: model.NewPointer(st.NewTestID()),
+ UserEmail: model.NewPointer(st.NewTestID()),
+ Username: model.NewPointer(st.NewTestID()),
+ PostId: model.NewPointer(st.NewTestID()),
+ PostCreateAt: model.NewPointer[int64](123),
+ PostUpdateAt: model.NewPointer[int64](123),
+ PostDeleteAt: model.NewPointer[int64](123),
+ PostMessage: model.NewPointer(st.NewTestID()),
+ },
+ }
+
+ // need to export at least one post to make an export directory and file
+
+ mockStore.ChannelMemberHistoryStore.On("GetChannelsWithActivityDuring", mock.Anything, mock.Anything).
+ Return([]string{*messages[0].ChannelId}, nil)
+ mockStore.ChannelStore.On("GetMany", []string{channelId}, true).
+ Return(model.ChannelList{{
+ Id: channelId,
+ DisplayName: channelDisplayName,
+ Name: channelName,
+ Type: channelType,
+ }}, nil)
+
+ mockStore.ComplianceStore.On("MessageExport", mock.Anything, mock.AnythingOfType("model.MessageExportCursor"), 10001).Return(
+ messages, model.MessageExportCursor{}, nil,
+ ).Once()
+ mockStore.ComplianceStore.On("MessageExport", mock.Anything, mock.AnythingOfType("model.MessageExportCursor"), 10001).Return(
make([]*model.MessageExport, 0), model.MessageExportCursor{}, nil,
- )
+ ).Once()
+ mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil)
mockStore.PostStore.On("AnalyticsPostCount", mock.Anything).Return(
- int64(estimatedPostCount), nil,
+ int64(1), nil,
)
// job completed successfully
@@ -309,11 +363,13 @@ func TestDoJobWithDedicatedExportBackend(t *testing.T) {
ExportDirectory: model.NewPointer(tempDedicatedDir),
},
MessageExportSettings: model.MessageExportSettings{
- EnableExport: model.NewPointer(true),
- ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
- DailyRunTime: model.NewPointer("01:00"),
- ExportFromTimestamp: model.NewPointer(int64(0)),
- BatchSize: model.NewPointer(10000),
+ EnableExport: model.NewPointer(true),
+ ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
+ DailyRunTime: model.NewPointer("01:00"),
+ ExportFromTimestamp: model.NewPointer(int64(0)),
+ BatchSize: model.NewPointer(10000),
+ ChannelBatchSize: model.NewPointer(100),
+ ChannelHistoryBatchSize: model.NewPointer(100),
},
},
},
@@ -347,7 +403,7 @@ func TestDoJobCancel(t *testing.T) {
t.Cleanup(func() { mockMetrics.AssertExpectations(t) })
job := &model.Job{
- Id: model.NewId(),
+ Id: st.NewTestID(),
CreateAt: model.GetMillis(),
Status: model.JobStatusPending,
Type: model.JobTypeMessageExport,
@@ -368,11 +424,13 @@ func TestDoJobCancel(t *testing.T) {
Directory: model.NewPointer(tempDir),
},
MessageExportSettings: model.MessageExportSettings{
- EnableExport: model.NewPointer(true),
- ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
- DailyRunTime: model.NewPointer("01:00"),
- ExportFromTimestamp: model.NewPointer(int64(0)),
- BatchSize: model.NewPointer(10000),
+ EnableExport: model.NewPointer(true),
+ ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
+ DailyRunTime: model.NewPointer("01:00"),
+ ExportFromTimestamp: model.NewPointer(int64(0)),
+ BatchSize: model.NewPointer(10000),
+ ChannelBatchSize: model.NewPointer(100),
+ ChannelHistoryBatchSize: model.NewPointer(100),
},
},
},
@@ -390,11 +448,26 @@ func TestDoJobCancel(t *testing.T) {
mockMetrics.On("IncrementJobActive", model.JobTypeMessageExport)
// No previous job, data will be loaded from config
- mockStore.JobStore.On("GetNewestJobByStatusesAndType", []string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport).Return(nil, model.NewAppError("", "", nil, "", http.StatusBadRequest))
+ mockStore.JobStore.On("GetNewestJobByStatusesAndType", []string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport).Return(nil, errors.New("test"))
+
+ // Job updates the system console UI, once for getting channels, once for getting activity
+ mockStore.JobStore.On("UpdateOptimistically", mock.AnythingOfType("*model.Job"), model.JobStatusInProgress).Return(true, nil).Times(2)
+
+ // a few calls pass
+ mockStore.ChannelMemberHistoryStore.On("GetChannelsWithActivityDuring", mock.Anything, mock.Anything).
+ Return([]string{"channel-id"}, nil)
+ mockStore.ChannelStore.On("GetMany", []string{"channel-id"}, true).
+ Return(model.ChannelList{{
+ Id: "channel-id",
+ DisplayName: "channel-display-name",
+ Name: "channel-name",
+ Type: model.ChannelTypeDirect,
+ }}, nil)
+ mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", mock.Anything, mock.Anything, []string{"channel-id"}).Return([]*model.ChannelMemberHistoryResult{}, nil)
cancelled := make(chan struct{})
// Cancel the worker and return an error
- mockStore.ComplianceStore.On("MessageExport", mock.Anything, mock.AnythingOfType("model.MessageExportCursor"), 10000).Run(func(args tmock.Arguments) {
+ mockStore.ComplianceStore.On("MessageExport", mock.Anything, mock.AnythingOfType("model.MessageExportCursor"), 10001).Run(func(args tmock.Arguments) {
worker.cancel()
rctx, ok := args.Get(0).(request.CTX)
@@ -408,7 +481,7 @@ func TestDoJobCancel(t *testing.T) {
)
mockStore.PostStore.On("AnalyticsPostCount", mock.Anything).Return(
- int64(estimatedPostCount), nil,
+ int64(shared.EstimatedPostCount), nil,
)
// Job marked as pending
@@ -425,75 +498,3 @@ func TestDoJobCancel(t *testing.T) {
// Cleanup
worker.Stop()
}
-
-func TestCreateZipFile(t *testing.T) {
- rctx := request.TestContext(t)
-
- tempDir, ioErr := os.MkdirTemp("", "")
- require.NoError(t, ioErr)
- defer os.RemoveAll(tempDir)
-
- config := filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: tempDir,
- }
-
- fileBackend, err := filestore.NewFileBackend(config)
- assert.NoError(t, err)
- _ = fileBackend
-
- b := []byte("test")
- path1 := path.Join(exportPath, "19700101")
- path2 := path.Join(exportPath, "19800101/subdir")
-
- // We test with a mock to test the Hitachi HCP case
- // where ListDirectory returns the dir itself as the first entry.
- // Note: If the mocks fail, that means the logic in createZipFile has
- // gone wrong and needs to be verified.
- mock := &fmocks.FileBackend{}
- defer mock.AssertExpectations(t)
-
- mock.On("WriteFile", tmock.Anything, tmock.AnythingOfType("string")).Return(int64(4), nil)
- mock.On("FileSize", tmock.Anything).Return(int64(4), nil)
- mock.On("FileSize", tmock.Anything).Return(int64(4), nil)
- mock.On("Reader", path.Join(path1, "testid")).Return(mockReadSeekCloser{bytes.NewReader([]byte("test"))}, nil)
- mock.On("Reader", path.Join(path2, "testid")).Return(mockReadSeekCloser{bytes.NewReader([]byte("test"))}, nil)
- mock.On("ListDirectoryRecursively", path1).Return([]string{path1, path.Join(path1, "testid")}, nil)
- mock.On("ListDirectoryRecursively", path2).Return([]string{path2, path.Join(path2, "testid")}, nil)
-
- for i, backend := range []filestore.FileBackend{fileBackend, mock} {
- written, err := backend.WriteFile(bytes.NewReader(b), path1+"/"+model.NewId())
- assert.NoError(t, err)
- assert.Equal(t, int64(len(b)), written)
-
- written, err = backend.WriteFile(bytes.NewReader(b), path2+"/"+model.NewId())
- assert.NoError(t, err)
- assert.Equal(t, int64(len(b)), written)
-
- written, err = backend.WriteFile(bytes.NewReader(b), path2+"/"+model.NewId())
- assert.NoError(t, err)
- assert.Equal(t, int64(len(b)), written)
-
- err = createZipFile(rctx, backend, "testjob", []string{path1, path2})
- assert.NoError(t, err)
-
- // Skip checking the zip file in mock case.
- if i == 1 {
- continue
- }
- r, err := zip.OpenReader(path.Join(tempDir, exportPath) + "/testjob.zip")
- assert.NoError(t, err)
- err = r.Close()
- require.NoError(t, err)
-
- assert.Equal(t, 3, len(r.File))
- }
-}
-
-type mockReadSeekCloser struct {
- *bytes.Reader
-}
-
-func (r mockReadSeekCloser) Close() error {
- return nil
-}
diff --git a/server/i18n/en.json b/server/i18n/en.json
index 4905edc9bd..01cfb33484 100644
--- a/server/i18n/en.json
+++ b/server/i18n/en.json
@@ -7576,14 +7576,6 @@
"id": "ent.account_migration.get_saml_users_failed",
"translation": "Unable to get SAML users."
},
- {
- "id": "ent.actiance.export.marshalToXml.appError",
- "translation": "Unable to convert export to XML."
- },
- {
- "id": "ent.actiance.export.write_file.appError",
- "translation": "Unable to write the export file."
- },
{
"id": "ent.api.post.send_notifications_and_forget.push_image_only",
"translation": " attached a file."
@@ -7604,86 +7596,10 @@
"id": "ent.cluster.timeout.error",
"translation": "Timed out waiting for cluster response"
},
- {
- "id": "ent.compliance.bad_export_type.appError",
- "translation": "Unknown output format {{.ExportType}}"
- },
- {
- "id": "ent.compliance.csv.attachment.copy.appError",
- "translation": "Unable to copy the attachment into the zip file."
- },
- {
- "id": "ent.compliance.csv.attachment.export.appError",
- "translation": "Unable to add attachment to the CSV export."
- },
- {
- "id": "ent.compliance.csv.file.creation.appError",
- "translation": "Unable to create temporary CSV export file."
- },
- {
- "id": "ent.compliance.csv.header.export.appError",
- "translation": "Unable to add header to the CSV export."
- },
- {
- "id": "ent.compliance.csv.metadata.close.appError",
- "translation": "Unable to close the zip file"
- },
- {
- "id": "ent.compliance.csv.metadata.export.appError",
- "translation": "Unable to add metadata file to the zip file."
- },
- {
- "id": "ent.compliance.csv.metadata.json.marshalling.appError",
- "translation": "Unable to convert metadata to json."
- },
- {
- "id": "ent.compliance.csv.metadata.json.zipfile.appError",
- "translation": "Unable to create the zip file"
- },
- {
- "id": "ent.compliance.csv.post.export.appError",
- "translation": "Unable to export a post."
- },
- {
- "id": "ent.compliance.csv.seek.appError",
- "translation": "Unable to seek to start of export file."
- },
- {
- "id": "ent.compliance.csv.warning.appError",
- "translation": "Unable to create the warning file."
- },
- {
- "id": "ent.compliance.csv.write_file.appError",
- "translation": "Unable to write the csv file."
- },
- {
- "id": "ent.compliance.csv.zip.creation.appError",
- "translation": "Unable to create the zip export file."
- },
- {
- "id": "ent.compliance.global_relay.attachments_removed.appError",
- "translation": "Uploaded file was removed from Global Relay export because it was too large to send."
- },
- {
- "id": "ent.compliance.global_relay.open_temporary_file.appError",
- "translation": "Unable to open the temporary export file."
- },
- {
- "id": "ent.compliance.global_relay.rewind_temporary_file.appError",
- "translation": "Unable to re-read the Global Relay temporary export file."
- },
- {
- "id": "ent.compliance.global_relay.write_file.appError",
- "translation": "Unable to write the global relay file."
- },
{
"id": "ent.compliance.licence_disable.app_error",
"translation": "Compliance functionality disabled by current license. Please contact your system administrator about upgrading your enterprise license."
},
- {
- "id": "ent.compliance.run_export.template_watcher.appError",
- "translation": "Unable to load export templates. Please try again."
- },
{
"id": "ent.compliance.run_failed.error",
"translation": "Compliance export failed for job '{{.JobName}}' at '{{.FilePath}}'"
@@ -7944,10 +7860,6 @@
"id": "ent.elasticsearch.test_config.reenter_password",
"translation": "The Search Server URL or Username has changed. Please re-enter the password to test connection."
},
- {
- "id": "ent.get_users_in_channel_during",
- "translation": "Failed to get users in channel during specified time period."
- },
{
"id": "ent.id_loaded.license_disable.app_error",
"translation": "Your license does not support ID Loaded Push Notifications."
@@ -8081,72 +7993,20 @@
"translation": "unable to migrate."
},
{
- "id": "ent.message_export.actiance_export.get_attachment_error",
- "translation": "Failed to get file info for a post."
+ "id": "ent.message_export.actiance_export.calculate_channel_exports.activity_message",
+ "translation": "Calculating channel activity: {{.NumCompleted}}/{{.NumChannels}} channels completed."
},
{
- "id": "ent.message_export.csv_export.get_attachment_error",
- "translation": "Failed to get file info for a post."
+ "id": "ent.message_export.actiance_export.calculate_channel_exports.channel_message",
+ "translation": "Exporting channel information for {{.NumChannels}} channels."
},
{
- "id": "ent.message_export.global_relay.attach_file.app_error",
- "translation": "Unable to add attachment to the Global Relay export."
+ "id": "ent.message_export.calculate_channel_exports.app_error",
+ "translation": "Failed to calculate channel export data."
},
{
- "id": "ent.message_export.global_relay.close_zip_file.app_error",
- "translation": "Unable to close the zip file."
- },
- {
- "id": "ent.message_export.global_relay.create_file_in_zip.app_error",
- "translation": "Unable to create the eml file."
- },
- {
- "id": "ent.message_export.global_relay.generate_email.app_error",
- "translation": "Unable to generate eml file data."
- },
- {
- "id": "ent.message_export.global_relay_export.deliver.close.app_error",
- "translation": "Unable to deliver the email to Global Relay."
- },
- {
- "id": "ent.message_export.global_relay_export.deliver.from_address.app_error",
- "translation": "Unable to set the email From address."
- },
- {
- "id": "ent.message_export.global_relay_export.deliver.msg.app_error",
- "translation": "Unable to set the email message."
- },
- {
- "id": "ent.message_export.global_relay_export.deliver.msg_data.app_error",
- "translation": "Unable to write the email message."
- },
- {
- "id": "ent.message_export.global_relay_export.deliver.parse_mail.app_error",
- "translation": "Unable to read the email information."
- },
- {
- "id": "ent.message_export.global_relay_export.deliver.to_address.app_error",
- "translation": "Unable to set the email To address."
- },
- {
- "id": "ent.message_export.global_relay_export.deliver.unable_to_connect_smtp_server.app_error",
- "translation": "Unable to connect to the smtp server"
- },
- {
- "id": "ent.message_export.global_relay_export.deliver.unable_to_get_file_info.app_error",
- "translation": "Unable to get the information of the export temporary file."
- },
- {
- "id": "ent.message_export.global_relay_export.deliver.unable_to_open_email_file.app_error",
- "translation": "Unable to get the an email from the temporary file."
- },
- {
- "id": "ent.message_export.global_relay_export.deliver.unable_to_open_zip_file_data.app_error",
- "translation": "Unable to open the export temporary file."
- },
- {
- "id": "ent.message_export.global_relay_export.get_attachment_error",
- "translation": "Failed to get file info for a post."
+ "id": "ent.message_export.job_data_conversion.app_error",
+ "translation": "Failed to convert a value from the job's data field."
},
{
"id": "ent.message_export.run_export.app_error",
diff --git a/server/public/model/config.go b/server/public/model/config.go
index 0aa9aa878e..7b2305f51e 100644
--- a/server/public/model/config.go
+++ b/server/public/model/config.go
@@ -234,13 +234,19 @@ const (
PluginSettingsDefaultMarketplaceURL = "https://api.integrations.mattermost.com"
PluginSettingsOldMarketplaceURL = "https://marketplace.integrations.mattermost.com"
- ComplianceExportTypeCsv = "csv"
- ComplianceExportTypeActiance = "actiance"
- ComplianceExportTypeGlobalrelay = "globalrelay"
- ComplianceExportTypeGlobalrelayZip = "globalrelay-zip"
- GlobalrelayCustomerTypeA9 = "A9"
- GlobalrelayCustomerTypeA10 = "A10"
- GlobalrelayCustomerTypeCustom = "CUSTOM"
+ ComplianceExportDirectoryFormat = "compliance-export-2006-01-02-15h04m"
+ ComplianceExportPath = "export"
+ ComplianceExportPathCLI = "cli"
+ ComplianceExportTypeCsv = "csv"
+ ComplianceExportTypeActiance = "actiance"
+ ComplianceExportTypeGlobalrelay = "globalrelay"
+ ComplianceExportTypeGlobalrelayZip = "globalrelay-zip"
+ ComplianceExportChannelBatchSizeDefault = 100
+ ComplianceExportChannelHistoryBatchSizeDefault = 10
+
+ GlobalrelayCustomerTypeA9 = "A9"
+ GlobalrelayCustomerTypeA10 = "A10"
+ GlobalrelayCustomerTypeCustom = "CUSTOM"
ClientSideCertCheckPrimaryAuth = "primary"
ClientSideCertCheckSecondaryAuth = "secondary"
@@ -3403,12 +3409,14 @@ func (s *GlobalRelayMessageExportSettings) SetDefaults() {
}
type MessageExportSettings struct {
- EnableExport *bool `access:"compliance_compliance_export"`
- ExportFormat *string `access:"compliance_compliance_export"`
- DailyRunTime *string `access:"compliance_compliance_export"`
- ExportFromTimestamp *int64 `access:"compliance_compliance_export"`
- BatchSize *int `access:"compliance_compliance_export"`
- DownloadExportResults *bool `access:"compliance_compliance_export"`
+ EnableExport *bool `access:"compliance_compliance_export"`
+ ExportFormat *string `access:"compliance_compliance_export"`
+ DailyRunTime *string `access:"compliance_compliance_export"`
+ ExportFromTimestamp *int64 `access:"compliance_compliance_export"`
+ BatchSize *int `access:"compliance_compliance_export"`
+ DownloadExportResults *bool `access:"compliance_compliance_export"`
+ ChannelBatchSize *int `access:"compliance_compliance_export"`
+ ChannelHistoryBatchSize *int `access:"compliance_compliance_export"`
// formatter-specific settings - these are only expected to be non-nil if ExportFormat is set to the associated format
GlobalRelaySettings *GlobalRelayMessageExportSettings `access:"compliance_compliance_export"`
@@ -3439,6 +3447,14 @@ func (s *MessageExportSettings) SetDefaults() {
s.BatchSize = NewPointer(10000)
}
+ if s.ChannelBatchSize == nil || *s.ChannelBatchSize == 0 {
+ s.ChannelBatchSize = NewPointer(ComplianceExportChannelBatchSizeDefault)
+ }
+
+ if s.ChannelHistoryBatchSize == nil || *s.ChannelHistoryBatchSize == 0 {
+ s.ChannelHistoryBatchSize = NewPointer(ComplianceExportChannelHistoryBatchSizeDefault)
+ }
+
if s.GlobalRelaySettings == nil {
s.GlobalRelaySettings = &GlobalRelayMessageExportSettings{}
}
@@ -4434,7 +4450,7 @@ func (s *MessageExportSettings) isValid() *AppError {
return NewAppError("Config.IsValid", "model.config.is_valid.message_export.daily_runtime.app_error", nil, "", http.StatusBadRequest).Wrap(err)
} else if s.BatchSize == nil || *s.BatchSize < 0 {
return NewAppError("Config.IsValid", "model.config.is_valid.message_export.batch_size.app_error", nil, "", http.StatusBadRequest)
- } else if s.ExportFormat == nil || (*s.ExportFormat != ComplianceExportTypeActiance && *s.ExportFormat != ComplianceExportTypeGlobalrelay && *s.ExportFormat != ComplianceExportTypeCsv) {
+ } else if s.ExportFormat == nil || (*s.ExportFormat != ComplianceExportTypeActiance && *s.ExportFormat != ComplianceExportTypeGlobalrelay && *s.ExportFormat != ComplianceExportTypeCsv && *s.ExportFormat != ComplianceExportTypeGlobalrelayZip) {
return NewAppError("Config.IsValid", "model.config.is_valid.message_export.export_type.app_error", nil, "", http.StatusBadRequest)
}
diff --git a/server/public/model/job.go b/server/public/model/job.go
index 500d4959ac..a01836ffe7 100644
--- a/server/public/model/job.go
+++ b/server/public/model/job.go
@@ -10,6 +10,7 @@ import (
const (
JobTypeDataRetention = "data_retention"
JobTypeMessageExport = "message_export"
+ JobTypeCLIMessageExport = "cli_message_export"
JobTypeElasticsearchPostIndexing = "elasticsearch_post_indexing"
JobTypeElasticsearchPostAggregation = "elasticsearch_post_aggregation"
JobTypeBlevePostIndexing = "bleve_post_indexing"
diff --git a/server/public/model/message_export.go b/server/public/model/message_export.go
index d21d467b4f..192ddcfe86 100644
--- a/server/public/model/message_export.go
+++ b/server/public/model/message_export.go
@@ -24,6 +24,7 @@ type MessageExport struct {
PostCreateAt *int64
PostUpdateAt *int64
PostDeleteAt *int64
+ PostEditAt *int64
PostMessage *string
PostType *string
PostRootId *string
@@ -32,9 +33,12 @@ type MessageExport struct {
PostFileIds StringArray
}
+// MessageExportCursor retrieves posts in the inclusive range:
+// [LastPostUpdateAt + LastPostId, UntilUpdateAt]
type MessageExportCursor struct {
LastPostUpdateAt int64
LastPostId string
+ UntilUpdateAt int64
}
// PreviewID returns the value of the post's previewed_post prop, if present, or an empty string.
diff --git a/server/public/model/post.go b/server/public/model/post.go
index 1b4dc43a7b..73d3c2d1f9 100644
--- a/server/public/model/post.go
+++ b/server/public/model/post.go
@@ -399,8 +399,11 @@ type PostCountOptions struct {
UsersPostsOnly bool
// AllowFromCache looks up cache only when ExcludeDeleted and UsersPostsOnly are true and rest are falsy.
AllowFromCache bool
- SincePostID string
- SinceUpdateAt int64
+
+ // retrieves posts in the inclusive range: [SinceUpdateAt + LastPostId, UntilUpdateAt]
+ SincePostID string
+ SinceUpdateAt int64
+ UntilUpdateAt int64
}
func (o *Post) Etag() string {
diff --git a/server/public/shared/i18n/i18n.go b/server/public/shared/i18n/i18n.go
index cc3cc27b48..9f72a3d997 100644
--- a/server/public/shared/i18n/i18n.go
+++ b/server/public/shared/i18n/i18n.go
@@ -78,6 +78,28 @@ func TranslationsPreInit(translationsDir string) error {
return initTranslationsWithDir(translationsDir)
}
+// TranslationsPreInitFromFileBytes loads translations from a buffer -- useful if
+// we need to initialize i18n from an embedded i18n file (e.g., from a CLI tool)
+func TranslationsPreInitFromFileBytes(filename string, buf []byte) error {
+ if T != nil {
+ return nil
+ }
+
+ // Set T even if we fail to load the translations. Lots of shutdown handling code will
+ // segfault trying to handle the error, and the untranslated IDs are strictly better.
+ T = tfuncWithFallback(defaultLocale)
+ TDefault = tfuncWithFallback(defaultLocale)
+
+ locale := strings.Split(filename, ".")[0]
+ if !isSupportedLocale(locale) {
+ return fmt.Errorf("locale not supported: %s", locale)
+ }
+
+ locales[locale] = filename
+
+ return i18n.ParseTranslationFileBytes(filename, buf)
+}
+
// InitTranslations set the defaults configured in the server and initialize
// the T function using the server default as fallback language
func InitTranslations(serverLocale, clientLocale string) error {
diff --git a/server/templates/globalrelay_compliance_export.html b/server/templates/globalrelay_compliance_export.html
index 538d8517c2..3b9f58582a 100644
--- a/server/templates/globalrelay_compliance_export.html
+++ b/server/templates/globalrelay_compliance_export.html
@@ -62,14 +62,20 @@
Conversation Summary
- - Channel: {{.Props.ChannelName}}
- - Started: {{.Props.Started}}
- - Ended: {{.Props.Ended}}
- - Duration: {{.Props.Duration}}
+ - TeamId:{{.Props.TeamId}}
+ - TeamName:{{.Props.TeamName}}
+ - TeamDisplayName:{{.Props.TeamDisplayName}}
+ - ChannelId:{{.Props.ChannelId}}
+ - ChannelName:{{.Props.ChannelName}}
+ - ChannelDisplayName:{{.Props.ChannelDisplayName}}
+ - Started:{{.Props.Started}}
+ - Ended:{{.Props.Ended}}
+ - Duration:{{.Props.Duration}}
+ UserId
|
Username
|
UserType
|
Email |
diff --git a/server/templates/globalrelay_compliance_export_message.html b/server/templates/globalrelay_compliance_export_message.html
index 74d07bc89d..075017bd2e 100644
--- a/server/templates/globalrelay_compliance_export_message.html
+++ b/server/templates/globalrelay_compliance_export_message.html
@@ -1,11 +1,16 @@
{{define "globalrelay_compliance_export_message"}}
+ {{.Props.PostId}}
{{.Props.SentTime}}
@{{.Props.Username}}
+ {{.Props.UserId}}
{{.Props.PostUsername}}
{{.Props.UserType}}
- ({{.Props.Email}}):
- {{.Props.Message}}
- {{.Props.PreviewsPost}}
+ ({{.Props.Email}})
+ {{.Props.Message}}{{ if .Props.PreviewsPost }}
+ {{.Props.PreviewsPost}}{{ end }}{{ if .Props.UpdateType }}
+ {{.Props.UpdateType}}
+ {{.Props.UpdateTime}}
+ {{.Props.EditedNewMsgId}}{{ end }}
{{end}}
diff --git a/server/templates/globalrelay_compliance_export_participant_row.html b/server/templates/globalrelay_compliance_export_participant_row.html
index db14c56bc0..84002e114a 100644
--- a/server/templates/globalrelay_compliance_export_participant_row.html
+++ b/server/templates/globalrelay_compliance_export_participant_row.html
@@ -1,5 +1,6 @@
{{define "globalrelay_compliance_export_participant_row"}}
+ | {{.Props.UserId}} |
@{{.Props.Username}} |
{{.Props.UserType}} |
{{.Props.Email}} |
diff --git a/webapp/channels/src/components/admin_console/message_export_settings.test.tsx b/webapp/channels/src/components/admin_console/message_export_settings.test.tsx
index 58b4eb1a09..3cc94de940 100644
--- a/webapp/channels/src/components/admin_console/message_export_settings.test.tsx
+++ b/webapp/channels/src/components/admin_console/message_export_settings.test.tsx
@@ -183,13 +183,16 @@ describe('components/MessageExportSettings/getJobDetails', () => {
const wrapper = shallowWithIntl();
- function runTest(testJob: Job, expectNull: boolean, expectedCount: number) {
+ function runTest(testJob: Job, expectNull: boolean, expectedCount: number, expectedMessage = '') {
const jobDetails = (wrapper.instance() as MessageExportSettingsClass).getJobDetails(testJob);
if (expectNull) {
expect(jobDetails).toBe(null);
} else {
expect(jobDetails?.length).toBe(expectedCount);
}
+ if (expectedMessage) {
+ expect(jobDetails?.[0].props.children).toEqual(expectedMessage);
+ }
}
const job = {} as Job;
@@ -219,4 +222,13 @@ describe('components/MessageExportSettings/getJobDetails', () => {
};
runTest(job, false, 2);
});
+
+ test('test progress message', () => {
+ job.data = {
+ messages_exported: 0,
+ warning_count: 0,
+ progress_message: 'this is a custom progress message',
+ };
+ runTest(job, false, 1, 'this is a custom progress message');
+ });
});
diff --git a/webapp/channels/src/components/admin_console/message_export_settings.tsx b/webapp/channels/src/components/admin_console/message_export_settings.tsx
index 704f1dbf98..b2d3187475 100644
--- a/webapp/channels/src/components/admin_console/message_export_settings.tsx
+++ b/webapp/channels/src/components/admin_console/message_export_settings.tsx
@@ -170,6 +170,11 @@ export class MessageExportSettings extends OLDAdminSettings{job.data.progress_message},
+ );
+ }
return message;
}
return null;