MM-59966 - Compliance Export overhaul - feature branch (#29789)

* [MM-59089] Add a compliance export constant (#27919)

* add a useful constant

* i18n

* another constant

* another i18n

* [MM-60422] Add GetChannelsWithActivityDuring (#28301)

* modify GetUsersInChannelDuring to accept a slice of channelIds

* add GetChannelsWithActivityDuring

* add compliance export progress message; remove unused custom status

* linting

* tests running too fast

* add batch size config settings

* add store tests

* linting

* empty commit

* i18n changes

* fix i18n ordering

* MM-60570 - Server-side changes consolidating the export CLI with server/ent code (#28640)

* add an i18n field; add the CLI's export directory

* int64 -> int

* Add UntilUpdateAt for MessageExport and AnalyticsPostCount

to merge

* remove now-unused i18n strings

* add TranslationsPreInitFromBuffer to allow CLI to use i18n

* use GetBuilder to simplify; rename TranslationsPreInitFromFileBytes

* [MM-59089] Improve compliance export timings (#1733 - Enterprise repo)

* MM-60422 - Performance and logic fixes for Compliance Exports (#1757 - Enterprise repo)

* MM-60570 - Enterprise-side changes consolidating the export CLI with server/ent code (#1769 - Enterprise repo)

* merge conflicts; missed file from ent branch

* MM-61038 - Add an option to sqlstore.New (#28702)

remove useless comment

add test

add an option to sqlstore.New

* MM-60976: Remove RunExport command from Mattermost binary (#28805)

* remove RunExport command from mattermost binary

* remove the code it was calling

* fix i18n

* remove test (was only testing license, not functionality)

* empty commit

* fix flaky GetChannelsWithActivityDuring test

* MM-60063: Dedicated Export Filestore fix, redo of #1772 (enterprise) (#28803)

* redo filestore fix #1772 (enterprise repo) on top of MM-59966 feature

* add new e2e tests for export filestore

* golint

* ok, note to self: shadowing bad, actually (when there's a defer)

* empty commit

* MM-61137 - Message export: Support 7.8.11 era dbs (#28824)

* support 7.8.11 era dbs by wrapping the store using only what we need

* fix flaky GetChannelsWithActivityDuring test

* add a comment

* only need to define the MEFileInfoStore (the one that'll be overridden)

* blank commit

* MM-60974 - Message Export: Add performance metrics (#28836)

* support 7.8.11 era dbs by wrapping the store using only what we need

* fix flaky GetChannelsWithActivityDuring test

* add a comment

* only need to define the MEFileInfoStore (the one that'll be overridden)

* performance metrics

* cleanup unneeded named returns

* blank commit

* MM-60975 - Message export: Add startTime and endTime to export folder name (#28840)

* support 7.8.11 era dbs by wrapping the store using only what we need

* fix flaky GetChannelsWithActivityDuring test

* add a comment

* only need to define the MEFileInfoStore (the one that'll be overridden)

* performance metrics

* output startTime and endTime in export folder

* empty commit

* merge conflict

* MM-60978 - Message export: Improve xml fields; fix delete semantics (#28873)

* support 7.8.11 era dbs by wrapping the store using only what we need

* fix flaky GetChannelsWithActivityDuring test

* add a comment

* only need to define the MEFileInfoStore (the one that'll be overridden)

* performance metrics

* output startTime and endTime in export folder

* empty commit

* add xml fields, omit when empty, tests

* fix delete semantics; test (and test for update semantics)

* clarify comments

* simplify edited post detection, now there's no edge case.

* add some spacing to help fast running tests

* merge conflicts/updates needed for new deleted post semantics

* linting; fixing tests from upstream merge

* use SafeDereference

* linting

* stronger typing; better wrapped errors; better formatting

* blank commit

* goimports formatting

* fix merge mistake

* minor fixes due to changes in master

* MM-61755 - Simplifying and Support reporting to the db from the CLI (#29281)

* finally clean up JobData struct and stringMap; prep for CLI using db

* and now simplify using StringMapToJobDataWithZeroValues

* remove unused fn

* create JobDataExported; clean up errors

* MM-60176 - Message Export: Global relay cleanup (#29168)

* move global relay logic into global_relay_export

* blank commit

* blank commit

* improve errors

* MM-60693 - Refactor CSV to use same codepath as Actiance (#29191)

* move global relay logic into global_relay_export

* blank commit

* refactor (and simplify) ExportParams into shared

* blank commit

* remove unused fn

* csv now uses pre-calculated joins/leaves like actiance

* improve errors

* remove nil post check; remove ignoredPosts metric

* remove unneeded copy

* MM-61696 - Refactor GlobalRelay to use same codepath as Actiance (#29225)

* move global relay logic into global_relay_export

* blank commit

* refactor (and simplify) ExportParams into shared

* blank commit

* remove unused fn

* csv now uses pre-calculated joins/leaves like actiance

* remove newly unneeded function and its test. goodbye.

* refactor GetPostAttachments for csv + global relay to share

* refactor global_relay_export and fix tests (no changes to output)

* improve errors

* remove nil post check; remove ignoredPosts metric

* remove unneeded copy

* remove unneeded nil check

* PR comments

* MM-61715 - Generalize e2e to all export types 🤖  (#29369)

* move global relay logic into global_relay_export

* blank commit

* refactor (and simplify) ExportParams into shared

* blank commit

* remove unused fn

* csv now uses pre-calculated joins/leaves like actiance

* remove newly unneeded function and its test. goodbye.

* refactor GetPostAttachments for csv + global relay to share

* refactor global_relay_export and fix tests (no changes to output)

* improve errors

* remove nil post check; remove ignoredPosts metric

* remove unneeded copy

* remove unneeded nil check

* PR comments

* refactor isDeletedMsg for all export types

* fix start and endtime, nasty csv createAt bug; bring closer to Actiance

* align unit tests with new logic (e.g. starttime / endtime)

* refactor a TimestampConvert fn for code + tests

* bug: pass templates to global relay (hurray for e2e tests, otherwise...)

* add global relay zip to allowed list (only for tests)

* test helpers

* new templates for e2e tests

* e2e tests... phew.

* linting

* merge conflicts

* unexport PostToRow; add test helper marker

* cleanup, shortening, thanks to PR comments

* MM-61972 - Generalize export data path - Actiance (#29399)

* extract and generalize the export data generation functions

* finish moving test (bc of previous extraction)

* lift a function from common -> shared (to break an import cycle)

* actiance now takes general export data, processes it into actiance data

* bring tests in line with correct sorting rules (upadateAt, messageId)

* fixups, PR comments

* turn strings.Repeat into a more descriptive const

amended: one letter fix; bad rebase

* MM-62009 - e2e clock heisenbug (#29434)

* consolidate assertions; output debuggable diffs (keeping for future)

* refactor test output generator to generators file

* waitUntilZeroPosts + pass through until to job = fix all clock issues

* simplify messages to model.NewId(); remove unneeded waitUntilZeroPosts

* model.NewId() -> storetest.NewTestID()

* MM-61980 - Generalize export data path - CSV (#29482)

* simple refactoring

* increase sleep times for (very) rare test failures

* add extra information to the generic export for CSV

* adj Actiance to handle new generic export (no difference in its output)

* no longer need mergePosts (yay), move getJoinLeavePosts for everyone

* adjust tests for new csv semantics (detailed in summary)

* and need to add the new exported data to the export_data_tests

* rearrange csv writing to happen after data export (more logical)

* linting

* remove debug statements

* figured out what was wrong with global relay e2e test 3; solid now

* PR comments

* MM-61718 - Generalize export data path - Global Relay (#29508)

* move global relay over to using the generalized export data

* performance pass -- not much can be done

* Update server/enterprise/message_export/global_relay_export/global_relay_export.go

Co-authored-by: Claudio Costa <cstcld91@gmail.com>

---------

Co-authored-by: Claudio Costa <cstcld91@gmail.com>

* MM-62058 - Align CSV with Actiance (#29551)

* refactoring actiance files and var names for clarity

* bug found in exported attachments (we used to miss some start/ends)

* changes needed for actiance due to new generic exports

* bringing CSV up to actiance standards

* fixing global relay b/c of new semantics (adding a note on an edge case)

* aligning e2e tests, adding comments to clarify what is expected/tested

* necessary changes; 1 more test for added functionality (ignoreDeleted)

* comment style

* MM-62059 - Align Global Relay with Actiance/CSV; many fixes (#29665)

* core logic changes to general export_data and the specific export paths

* unit tests and e2e tests, covering all new edge cases and all logic

* linting

* better var naming, const value, and cleaning up functions calls

* MM-62436 - Temporarily skip cypress tests that require download link (#29772)

---------

Co-authored-by: Claudio Costa <cstcld91@gmail.com>

server/channels/app/channel_test.go

@@ -298,7 +298,7 @@ func TestJoinDefaultChannelsCreatesChannelMemberHistoryRecordTownSquare(t *testi
 	channel, err := th.App.Srv().Store().Channel().GetByName(th.BasicTeam.Id, "town-square", true)
 	require.NoError(t, err)
 	townSquareChannelID := channel.Id
-	users, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, townSquareChannelID)
+	users, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{townSquareChannelID})
 	require.NoError(t, nErr)
 	initialNumTownSquareUsers := len(users)
 
@@ -308,7 +308,7 @@ func TestJoinDefaultChannelsCreatesChannelMemberHistoryRecordTownSquare(t *testi
 	require.Nil(t, appErr)
 
 	// there should be a ChannelMemberHistory record for the user
-	histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, townSquareChannelID)
+	histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{townSquareChannelID})
 	require.NoError(t, nErr)
 	assert.Len(t, histories, initialNumTownSquareUsers+1)
 
@@ -330,7 +330,7 @@ func TestJoinDefaultChannelsCreatesChannelMemberHistoryRecordOffTopic(t *testing
 	channel, err := th.App.Srv().Store().Channel().GetByName(th.BasicTeam.Id, "off-topic", true)
 	require.NoError(t, err)
 	offTopicChannelId := channel.Id
-	users, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, offTopicChannelId)
+	users, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{offTopicChannelId})
 	require.NoError(t, nErr)
 	initialNumTownSquareUsers := len(users)
 
@@ -340,7 +340,7 @@ func TestJoinDefaultChannelsCreatesChannelMemberHistoryRecordOffTopic(t *testing
 	require.Nil(t, appError)
 
 	// there should be a ChannelMemberHistory record for the user
-	histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, offTopicChannelId)
+	histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{offTopicChannelId})
 	require.NoError(t, nErr)
 	assert.Len(t, histories, initialNumTownSquareUsers+1)
 
@@ -418,7 +418,7 @@ func TestCreateChannelPublicCreatesChannelMemberHistoryRecord(t *testing.T) {
 	publicChannel := th.createChannel(th.Context, th.BasicTeam, model.ChannelTypeOpen)
 
 	// there should be a ChannelMemberHistory record for the user
-	histories, err := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, publicChannel.Id)
+	histories, err := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{publicChannel.Id})
 	require.NoError(t, err)
 	assert.Len(t, histories, 1)
 	assert.Equal(t, th.BasicUser.Id, histories[0].UserId)
@@ -433,7 +433,7 @@ func TestCreateChannelPrivateCreatesChannelMemberHistoryRecord(t *testing.T) {
 	privateChannel := th.createChannel(th.Context, th.BasicTeam, model.ChannelTypePrivate)
 
 	// there should be a ChannelMemberHistory record for the user
-	histories, err := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, privateChannel.Id)
+	histories, err := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{privateChannel.Id})
 	require.NoError(t, err)
 	assert.Len(t, histories, 1)
 	assert.Equal(t, th.BasicUser.Id, histories[0].UserId)
@@ -557,7 +557,7 @@ func TestCreateGroupChannelCreatesChannelMemberHistoryRecord(t *testing.T) {
 	channel, appErr := th.App.CreateGroupChannel(th.Context, groupUserIds, th.BasicUser.Id)
 
 	require.Nil(t, appErr, "Failed to create group channel.")
-	histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, channel.Id)
+	histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{channel.Id})
 	require.NoError(t, nErr)
 	assert.Len(t, histories, 3)
 
@@ -582,7 +582,7 @@ func TestCreateDirectChannelCreatesChannelMemberHistoryRecord(t *testing.T) {
 	channel, appErr := th.App.GetOrCreateDirectChannel(th.Context, user1.Id, user2.Id)
 	require.Nil(t, appErr, "Failed to create direct channel.")
 
-	histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, channel.Id)
+	histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{channel.Id})
 	require.NoError(t, nErr)
 	assert.Len(t, histories, 2)
 
@@ -610,7 +610,7 @@ func TestGetDirectChannelCreatesChannelMemberHistoryRecord(t *testing.T) {
 	require.Nil(t, appErr, "Failed to create direct channel.")
 
 	// there should be a ChannelMemberHistory record for both users
-	histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, channel.Id)
+	histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{channel.Id})
 	require.NoError(t, nErr)
 	assert.Len(t, histories, 2)
 
@@ -645,7 +645,7 @@ func TestAddUserToChannelCreatesChannelMemberHistoryRecord(t *testing.T) {
 	require.Nil(t, appErr, "Failed to add user to channel.")
 
 	// there should be a ChannelMemberHistory record for the user
-	histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, channel.Id)
+	histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{channel.Id})
 	require.NoError(t, nErr)
 	assert.Len(t, histories, 2)
 	channelMemberHistoryUserIds := make([]string, 0)
@@ -656,6 +656,88 @@ func TestAddUserToChannelCreatesChannelMemberHistoryRecord(t *testing.T) {
 	assert.Equal(t, groupUserIds, channelMemberHistoryUserIds)
 }
 
+func TestUsersAndPostsCreateActivityInChannel(t *testing.T) {
+	th := Setup(t).InitBasic().DeleteBots()
+	defer th.TearDown()
+
+	user := th.CreateUser()
+	_, err := th.App.AddTeamMember(th.Context, th.BasicTeam.Id, user.Id)
+	require.Nil(t, err, "Failed to add user to team.")
+	user3 := th.CreateUser()
+	_, err = th.App.AddTeamMember(th.Context, th.BasicTeam.Id, user3.Id)
+	require.Nil(t, err, "Failed to add user to team.")
+	user4 := th.CreateUser()
+	_, err = th.App.AddTeamMember(th.Context, th.BasicTeam.Id, user4.Id)
+	require.Nil(t, err, "Failed to add user to team.")
+
+	channel1 := th.createChannel(th.Context, th.BasicTeam, model.ChannelTypeOpen)
+	channel2 := th.createChannel(th.Context, th.BasicTeam, model.ChannelTypeOpen)
+	channel3 := th.createChannel(th.Context, th.BasicTeam, model.ChannelTypeOpen)
+	channel4 := th.createChannel(th.Context, th.BasicTeam, model.ChannelTypeOpen)
+	channel5 := th.createChannel(th.Context, th.BasicTeam, model.ChannelTypeOpen)
+	channel6 := th.createChannel(th.Context, th.BasicTeam, model.ChannelTypeOpen)
+
+	// user3 is already in channel3
+	_, err = th.App.AddUserToChannel(th.Context, user3, channel3, false)
+	require.Nil(t, err, "Failed to add user to channel.")
+	// user4 is already in channel4 (for the second part of the test)
+	_, err = th.App.AddUserToChannel(th.Context, user4, channel4, false)
+	require.Nil(t, err, "Failed to add user to channel.")
+
+	// make sure we don't catch earlier posts
+	time.Sleep(10 * time.Millisecond)
+	testStart := model.GetMillis()
+
+	// Test: previous activity (user3 and 4's adds) aren't showing up:
+	channelIds, nErr := th.App.Srv().Store().ChannelMemberHistory().GetChannelsWithActivityDuring(testStart, testStart+10000)
+	require.NoError(t, nErr)
+	assert.Len(t, channelIds, 0)
+
+	// Posts, adds, and leaves should create activity
+	post := &model.Post{
+		ChannelId: channel1.Id,
+		Message:   "root post",
+		UserId:    th.BasicUser.Id,
+	}
+	_, err = th.App.CreatePost(th.Context, post, channel1, model.CreatePostFlags{})
+	require.Nil(t, err, "Failed to create post.")
+
+	_, err = th.App.AddUserToChannel(th.Context, user, channel2, false)
+	require.Nil(t, err, "Failed to add user to channel.")
+
+	err = th.App.RemoveUserFromChannel(th.Context, user3.Id, user3.Id, channel3)
+	require.Nil(t, err, "Failed to add user to channel.")
+
+	// Test: there should be a ChannelMemberHistory record for the users and the post
+	channelIds, nErr = th.App.Srv().Store().ChannelMemberHistory().GetChannelsWithActivityDuring(testStart, model.GetMillis())
+	require.NoError(t, nErr)
+	assert.Len(t, channelIds, 3)
+	assert.ElementsMatch(t, []string{channel1.Id, channel2.Id, channel3.Id}, channelIds)
+
+	testEnd := model.GetMillis()
+	// In case the tests are running very fast:
+	time.Sleep(10 * time.Millisecond)
+
+	// Now, we do not find activity for new posts, leaves, or adds after the test is over
+	post2 := &model.Post{
+		ChannelId: channel5.Id,
+		Message:   "root post",
+		UserId:    th.BasicUser.Id,
+	}
+	err = th.App.RemoveUserFromChannel(th.Context, user4.Id, user4.Id, channel4)
+	require.Nil(t, err, "Failed to create post.")
+	_, err = th.App.CreatePost(th.Context, post2, channel5, model.CreatePostFlags{})
+	require.Nil(t, err, "Failed to create post.")
+	_, err = th.App.AddUserToChannel(th.Context, user, channel6, false)
+	require.Nil(t, err, "Failed to add user to channel.")
+
+	// Test: we get the same three channels as before, not channels 4, 5, 6 which have activity after testEnd
+	channelIds, nErr = th.App.Srv().Store().ChannelMemberHistory().GetChannelsWithActivityDuring(testStart, testEnd)
+	require.NoError(t, nErr)
+	assert.Len(t, channelIds, 3)
+	assert.ElementsMatch(t, []string{channel1.Id, channel2.Id, channel3.Id}, channelIds)
+}
+
 func TestLeaveDefaultChannel(t *testing.T) {
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
@@ -811,7 +893,7 @@ func TestAddChannelMemberNoUserRequestor(t *testing.T) {
 	require.Nil(t, appErr, "Failed to add user to channel.")
 
 	// there should be a ChannelMemberHistory record for the user
-	histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, channel.Id)
+	histories, nErr := th.App.Srv().Store().ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, []string{channel.Id})
 	require.NoError(t, nErr)
 	assert.Len(t, histories, 2)
 	channelMemberHistoryUserIds := make([]string, 0)

server/channels/app/file.go

@@ -103,7 +103,7 @@ func (a *App) TestFileStoreConnectionWithConfig(cfg *model.FileSettings) *model.
 		backend, err = filestore.NewFileBackend(filestore.NewFileBackendSettingsFromConfig(cfg, complianceEnabled, insecure != nil && *insecure))
 	}
 	if err != nil {
-		return model.NewAppError("FileBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		return model.NewAppError("FileAttachmentBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
 	}
 	nErr := backend.TestConnection()
 	if nErr != nil {