[MM 12464] Include DM/GM Channels and Their Posts in the Bulk Export (#10421)

* transplant the existing PR into the working tree * start addressing review comments * move existing direct channel export code into this branch * modify channel exporter to use squirell and populate members in two steps * use squirrel to build sql queries for channel and dm/gm export methods * remove debug helpers and use Username instead of UserId * unit test for DM Channel exporter * add more unit tests for channel export * add test for DM/GM post export * checkpoint with failing test for postgres * use getQueryBuilder to make sure squirrel uses the correct formatting for each database * add a test for post export * fix shadowed vars that broke the build * address review comments and add tests to support it * address review comments and add a mlog call * s/Info/Debug/ * address review comments in post_store * address review comments in channel_store * address review comments in export * address review comment in post_store: drop GroupBy * address review comment on supplier: move getQueryBuilder to sqlstore * address review comments: explicit TearDown * address review comments: improve test coverage * address review comments: make sure public and private channels are excluded * address review comments: improve test coverage * address review comments: make sure Channels table gets truncated after each test * more cleanups and better assertions * wrap PostStore in a StoreTestWithSqlSupplier * last minute changes: improve post export test coverage and check members * address review comments: make sure all posts have their channel members set * address review comments: make sure all posts have their ChannelMembers exported correctly * gofmt fix * sort channels so it's possible to assert on index
2019-03-15 12:28:43 -03:00
--- a/store/sqlstore/channel_store.go
+++ b/store/sqlstore/channel_store.go
@@ -14,6 +14,7 @@ import (
 	"github.com/mattermost/gorp"
 	"github.com/pkg/errors"

+	sq "github.com/Masterminds/squirrel"
 	"github.com/mattermost/mattermost-server/einterfaces"
 	"github.com/mattermost/mattermost-server/mlog"
 	"github.com/mattermost/mattermost-server/model"
@@ -2502,3 +2503,66 @@ func (s SqlChannelStore) GetChannelMembersForExport(userId string, teamId string
 		result.Data = members
 	})
 }
+
+func (s SqlChannelStore) GetAllDirectChannelsForExportAfter(limit int, afterId string) store.StoreChannel {
+	return store.Do(func(result *store.StoreResult) {
+		var directChannelsForExport []*model.DirectChannelForExport
+		query := s.getQueryBuilder().
+			Select("Channels.*").
+			From("Channels").
+			Where(sq.And{
+				sq.Gt{"Channels.Id": afterId},
+				sq.Eq{"Channels.DeleteAt": int(0)},
+				sq.Eq{"Channels.Type": []string{"D", "G"}},
+			}).
+			OrderBy("Channels.Id").
+			Limit(uint64(limit))
+
+		queryString, args, err := query.ToSql()
+		if err != nil {
+			result.Err = model.NewAppError("SqlTeamStore.GetAllDirectChannelsForExportAfter", "store.sql_channel.get_all_direct.app_error", nil, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		if _, err = s.GetReplica().Select(&directChannelsForExport, queryString, args...); err != nil {
+			result.Err = model.NewAppError("SqlTeamStore.GetAllDirectChannelsForExportAfter", "store.sql_channel.get_all_direct.app_error", nil, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		var channelIds []string
+		for _, channel := range directChannelsForExport {
+			channelIds = append(channelIds, channel.Id)
+		}
+		query = s.getQueryBuilder().
+			Select("*").
+			From("ChannelMembers cm").
+			Join("Users u ON ( u.Id = cm.UserId )").
+			Where(sq.And{
+				sq.Eq{"cm.ChannelId": channelIds},
+				sq.Eq{"u.DeleteAt": int(0)},
+			})
+
+		queryString, args, err = query.ToSql()
+		if err != nil {
+			result.Err = model.NewAppError("SqlTeamStore.GetAllDirectChannelsForExportAfter", "store.sql_channel.get_all_direct.app_error", nil, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		var channelMembers []*model.ChannelMemberForExport
+		if _, err := s.GetReplica().Select(&channelMembers, queryString, args...); err != nil {
+			result.Err = model.NewAppError("SqlTeamStore.GetAllDirectChannelsForExportAfter", "store.sql_channel.get_all_direct.app_error", nil, err.Error(), http.StatusInternalServerError)
+		}
+
+		// Populate each channel with its members
+		dmChannelsMap := make(map[string]*model.DirectChannelForExport)
+		for _, channel := range directChannelsForExport {
+			channel.Members = &[]string{}
+			dmChannelsMap[channel.Id] = channel
+		}
+		for _, member := range channelMembers {
+			members := dmChannelsMap[member.ChannelId].Members
+			*members = append(*members, member.Username)
+		}
+		result.Data = directChannelsForExport
+	})
+}
--- a/store/sqlstore/post_store.go
+++ b/store/sqlstore/post_store.go
@@ -11,6 +11,7 @@ import (
 	"strings"
 	"sync"

+	sq "github.com/Masterminds/squirrel"
 	"github.com/mattermost/mattermost-server/einterfaces"
 	"github.com/mattermost/mattermost-server/mlog"
 	"github.com/mattermost/mattermost-server/model"
@@ -1366,3 +1367,77 @@ func (s *SqlPostStore) GetRepliesForExport(parentId string) store.StoreChannel {
 		}
 	})
 }
+
+func (s *SqlPostStore) GetDirectPostParentsForExportAfter(limit int, afterId string) store.StoreChannel {
+	return store.Do(func(result *store.StoreResult) {
+		query := s.getQueryBuilder().
+			Select("p.*", "Users.Username as User").
+			From("Posts p").
+			Join("Channels ON p.ChannelId = Channels.Id").
+			Join("Users ON p.UserId = Users.Id").
+			Where(sq.And{
+				sq.Gt{"p.Id": afterId},
+				sq.Eq{"p.ParentId": string("")},
+				sq.Eq{"p.DeleteAt": int(0)},
+				sq.Eq{"Channels.DeleteAt": int(0)},
+				sq.Eq{"Users.DeleteAt": int(0)},
+				sq.Eq{"Channels.Type": []string{"D", "G"}},
+			}).
+			OrderBy("p.Id").
+			Limit(uint64(limit))
+
+		queryString, args, err := query.ToSql()
+		if err != nil {
+			result.Err = model.NewAppError("SqlPostStore.GetDirectPostParentsForExportAfter", "store.sql_post.get_direct_posts.app_error", nil, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		var posts []*model.DirectPostForExport
+		if _, err = s.GetReplica().Select(&posts, queryString, args...); err != nil {
+			result.Err = model.NewAppError("SqlPostStore.GetDirectPostParentsForExportAfter", "store.sql_post.get_direct_posts.app_error", nil, err.Error(), http.StatusInternalServerError)
+		}
+		var channelIds []string
+		for _, post := range posts {
+			channelIds = append(channelIds, post.ChannelId)
+		}
+		query = s.getQueryBuilder().
+			Select("*").
+			From("ChannelMembers cm").
+			Join("Users u ON ( u.Id = cm.UserId )").
+			Where(sq.Eq{
+				"cm.ChannelId": channelIds,
+			})
+
+		queryString, args, err = query.ToSql()
+		if err != nil {
+			result.Err = model.NewAppError("SqlPostStore.GetDirectPostParentsForExportAfter", "store.sql_post.get_direct_posts.app_error", nil, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		var channelMembers []*model.ChannelMemberForExport
+		if _, err := s.GetReplica().Select(&channelMembers, queryString, args...); err != nil {
+			result.Err = model.NewAppError("SqlPostStore.GetDirectPostParentsForExportAfter", "store.sql_post.get_direct_posts.app_error", nil, err.Error(), http.StatusInternalServerError)
+		}
+
+		// Build a map of channels and their posts
+		postsChannelMap := make(map[string][]*model.DirectPostForExport)
+		for _, post := range posts {
+			post.ChannelMembers = &[]string{}
+			postsChannelMap[post.ChannelId] = append(postsChannelMap[post.ChannelId], post)
+		}
+
+		// Build a map of channels and their members
+		channelMembersMap := make(map[string][]string)
+		for _, member := range channelMembers {
+			channelMembersMap[member.ChannelId] = append(channelMembersMap[member.ChannelId], member.Username)
+		}
+
+		// Populate each post ChannelMembers extracting it from the channelMembersMap
+		for channelId := range channelMembersMap {
+			for _, post := range postsChannelMap[channelId] {
+				*post.ChannelMembers = channelMembersMap[channelId]
+			}
+		}
+		result.Data = posts
+	})
+}
--- a/store/sqlstore/post_store_test.go
+++ b/store/sqlstore/post_store_test.go
@@ -10,5 +10,5 @@ import (
 )

 func TestPostStore(t *testing.T) {
-	StoreTest(t, storetest.TestPostStore)
+	StoreTestWithSqlSupplier(t, storetest.TestPostStore)
 }
--- a/store/sqlstore/store.go
+++ b/store/sqlstore/store.go
@@ -4,10 +4,10 @@
 package sqlstore

 import (
+	sq "github.com/Masterminds/squirrel"
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/lib/pq"
 	"github.com/mattermost/gorp"
-
 	"github.com/mattermost/mattermost-server/store"
 )

@@ -98,4 +98,5 @@ type SqlStore interface {
 	TermsOfService() store.TermsOfServiceStore
 	UserTermsOfService() store.UserTermsOfServiceStore
 	LinkMetadata() store.LinkMetadataStore
+	getQueryBuilder() sq.StatementBuilderType
 }
--- a/store/sqlstore/supplier.go
+++ b/store/sqlstore/supplier.go
@@ -15,6 +15,7 @@ import (
 	"sync/atomic"
 	"time"

+	sq "github.com/Masterminds/squirrel"
 	"github.com/dyatlov/go-opengraph/opengraph"
 	"github.com/go-sql-driver/mysql"
 	"github.com/lib/pq"
@@ -1051,6 +1052,14 @@ func (ss *SqlSupplier) DropAllTables() {
 	ss.master.TruncateTables()
 }

+func (ss *SqlSupplier) getQueryBuilder() sq.StatementBuilderType {
+	builder := sq.StatementBuilder.PlaceholderFormat(sq.Question)
+	if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES {
+		builder = builder.PlaceholderFormat(sq.Dollar)
+	}
+	return builder
+}
+
 type mattermConverter struct{}

 func (me mattermConverter) ToDb(val interface{}) (interface{}, error) {
--- a/store/sqlstore/user_store.go
+++ b/store/sqlstore/user_store.go
@@ -67,15 +67,11 @@ func NewSqlUserStore(sqlStore SqlStore, metrics einterfaces.MetricsInterface) st
 		metrics:  metrics,
 	}

-	us.usersQuery = sq.
+	us.usersQuery = us.getQueryBuilder().
 		Select("u.*", "b.UserId IS NOT NULL AS IsBot").
 		From("Users u").
 		LeftJoin("Bots b ON ( b.UserId = u.Id )")

-	if us.DriverName() == model.DATABASE_DRIVER_POSTGRES {
-		us.usersQuery = us.usersQuery.PlaceholderFormat(sq.Dollar)
-	}
-
 	for _, db := range sqlStore.GetAllConns() {
 		table := db.AddTableWithName(model.User{}, "Users").SetKeys(false, "Id")
 		table.ColMap("Id").SetMaxSize(26)
--- a/store/store.go
+++ b/store/store.go
@@ -187,6 +187,7 @@ type ChannelStore interface {
 	ClearAllCustomRoleAssignments() StoreChannel
 	MigratePublicChannels() error
 	GetAllChannelsForExportAfter(limit int, afterId string) StoreChannel
+	GetAllDirectChannelsForExportAfter(limit int, afterId string) StoreChannel
 	GetChannelMembersForExport(userId string, teamId string) StoreChannel
 	RemoveAllDeactivatedMembers(channelId string) StoreChannel
 }
@@ -229,6 +230,7 @@ type PostStore interface {
 	GetMaxPostSize() StoreChannel
 	GetParentsForExportAfter(limit int, afterId string) StoreChannel
 	GetRepliesForExport(parentId string) StoreChannel
+	GetDirectPostParentsForExportAfter(limit int, afterId string) StoreChannel
 }

 type UserStore interface {
--- a/store/storetest/channel_store.go
+++ b/store/storetest/channel_store.go
@@ -36,11 +36,11 @@ func TestChannelStore(t *testing.T, ss store.Store, s SqlSupplier) {
 	createDefaultRoles(t, ss)

 	t.Run("Save", func(t *testing.T) { testChannelStoreSave(t, ss) })
-	t.Run("SaveDirectChannel", func(t *testing.T) { testChannelStoreSaveDirectChannel(t, ss) })
+	t.Run("SaveDirectChannel", func(t *testing.T) { testChannelStoreSaveDirectChannel(t, ss, s) })
 	t.Run("CreateDirectChannel", func(t *testing.T) { testChannelStoreCreateDirectChannel(t, ss) })
 	t.Run("Update", func(t *testing.T) { testChannelStoreUpdate(t, ss) })
 	t.Run("GetChannelUnread", func(t *testing.T) { testGetChannelUnread(t, ss) })
-	t.Run("Get", func(t *testing.T) { testChannelStoreGet(t, ss) })
+	t.Run("Get", func(t *testing.T) { testChannelStoreGet(t, ss, s) })
 	t.Run("GetForPost", func(t *testing.T) { testChannelStoreGetForPost(t, ss) })
 	t.Run("Restore", func(t *testing.T) { testChannelStoreRestore(t, ss) })
 	t.Run("Delete", func(t *testing.T) { testChannelStoreDelete(t, ss) })
@@ -51,7 +51,7 @@ func TestChannelStore(t *testing.T, ss store.Store, s SqlSupplier) {
 	t.Run("ChannelMemberStore", func(t *testing.T) { testChannelMemberStore(t, ss) })
 	t.Run("ChannelDeleteMemberStore", func(t *testing.T) { testChannelDeleteMemberStore(t, ss) })
 	t.Run("GetChannels", func(t *testing.T) { testChannelStoreGetChannels(t, ss) })
-	t.Run("GetAllChannels", func(t *testing.T) { testChannelStoreGetAllChannels(t, ss) })
+	t.Run("GetAllChannels", func(t *testing.T) { testChannelStoreGetAllChannels(t, ss, s) })
 	t.Run("GetMoreChannels", func(t *testing.T) { testChannelStoreGetMoreChannels(t, ss) })
 	t.Run("GetPublicChannelsForTeam", func(t *testing.T) { testChannelStoreGetPublicChannelsForTeam(t, ss) })
 	t.Run("GetPublicChannelsByIdsForTeam", func(t *testing.T) { testChannelStoreGetPublicChannelsByIdsForTeam(t, ss) })
@@ -67,7 +67,7 @@ func TestChannelStore(t *testing.T, ss store.Store, s SqlSupplier) {
 	t.Run("SearchMore", func(t *testing.T) { testChannelStoreSearchMore(t, ss) })
 	t.Run("SearchInTeam", func(t *testing.T) { testChannelStoreSearchInTeam(t, ss) })
 	t.Run("SearchAllChannels", func(t *testing.T) { testChannelStoreSearchAllChannels(t, ss) })
-	t.Run("AutocompleteInTeamForSearch", func(t *testing.T) { testChannelStoreAutocompleteInTeamForSearch(t, ss) })
+	t.Run("AutocompleteInTeamForSearch", func(t *testing.T) { testChannelStoreAutocompleteInTeamForSearch(t, ss, s) })
 	t.Run("GetMembersByIds", func(t *testing.T) { testChannelStoreGetMembersByIds(t, ss) })
 	t.Run("AnalyticsDeletedTypeCount", func(t *testing.T) { testChannelStoreAnalyticsDeletedTypeCount(t, ss) })
 	t.Run("GetPinnedPosts", func(t *testing.T) { testChannelStoreGetPinnedPosts(t, ss) })
@@ -80,6 +80,9 @@ func TestChannelStore(t *testing.T, ss store.Store, s SqlSupplier) {
 	t.Run("GetAllChannelsForExportAfter", func(t *testing.T) { testChannelStoreGetAllChannelsForExportAfter(t, ss) })
 	t.Run("GetChannelMembersForExport", func(t *testing.T) { testChannelStoreGetChannelMembersForExport(t, ss) })
 	t.Run("RemoveAllDeactivatedMembers", func(t *testing.T) { testChannelStoreRemoveAllDeactivatedMembers(t, ss) })
+	t.Run("ExportAllDirectChannels", func(t *testing.T) { testChannelStoreExportAllDirectChannels(t, ss, s) })
+	t.Run("ExportAllDirectChannelsExcludePrivateAndPublic", func(t *testing.T) { testChannelStoreExportAllDirectChannelsExcludePrivateAndPublic(t, ss, s) })
+	t.Run("ExportAllDirectChannelsDeletedChannel", func(t *testing.T) { testChannelStoreExportAllDirectChannelsDeletedChannel(t, ss, s) })
 }

 func testChannelStoreSave(t *testing.T, ss store.Store) {
@@ -112,7 +115,7 @@ func testChannelStoreSave(t *testing.T, ss store.Store) {
 	}
 }

-func testChannelStoreSaveDirectChannel(t *testing.T, ss store.Store) {
+func testChannelStoreSaveDirectChannel(t *testing.T, ss store.Store, s SqlSupplier) {
 	teamId := model.NewId()

 	o1 := model.Channel{}
@@ -194,6 +197,8 @@ func testChannelStoreSaveDirectChannel(t *testing.T, ss store.Store) {
 		t.Fatal("should have saved just 1 member")
 	}

+	// Manually truncate Channels table until testlib can handle cleanups
+	s.GetMaster().Exec("TRUNCATE Channels")
 }

 func testChannelStoreCreateDirectChannel(t *testing.T, ss store.Store) {
@@ -340,7 +345,7 @@ func testGetChannelUnread(t *testing.T, ss store.Store) {
 	}
 }

-func testChannelStoreGet(t *testing.T, ss store.Store) {
+func testChannelStoreGet(t *testing.T, ss store.Store, s SqlSupplier) {
 	o1 := model.Channel{}
 	o1.TeamId = model.NewId()
 	o1.DisplayName = "Name"
@@ -423,6 +428,8 @@ func testChannelStoreGet(t *testing.T, ss store.Store) {
 			t.Fatal("too little")
 		}
 	}
+	// Manually truncate Channels table until testlib can handle cleanups
+	s.GetMaster().Exec("TRUNCATE Channels")
 }

 func testChannelStoreGetForPost(t *testing.T, ss store.Store) {
@@ -962,7 +969,7 @@ func testChannelStoreGetChannels(t *testing.T, ss store.Store) {
 	ss.Channel().InvalidateAllChannelMembersForUser(m1.UserId)
 }

-func testChannelStoreGetAllChannels(t *testing.T, ss store.Store) {
+func testChannelStoreGetAllChannels(t *testing.T, ss store.Store, s SqlSupplier) {
 	cleanupChannels(t, ss)

 	t1 := model.Team{}
@@ -1035,6 +1042,9 @@ func testChannelStoreGetAllChannels(t *testing.T, ss store.Store) {
 	assert.Len(t, *list, 1)
 	assert.Equal(t, (*list)[0].Id, c1.Id)
 	assert.Equal(t, (*list)[0].TeamDisplayName, "Name")
+
+	// Manually truncate Channels table until testlib can handle cleanups
+	s.GetMaster().Exec("TRUNCATE Channels")
 }

 func testChannelStoreGetMoreChannels(t *testing.T, ss store.Store) {
@@ -2306,7 +2316,7 @@ func testChannelStoreSearchAllChannels(t *testing.T, ss store.Store) {
 	}
 }

-func testChannelStoreAutocompleteInTeamForSearch(t *testing.T, ss store.Store) {
+func testChannelStoreAutocompleteInTeamForSearch(t *testing.T, ss store.Store, s SqlSupplier) {
 	u1 := &model.User{}
 	u1.Email = MakeEmail()
 	u1.Username = "user1" + model.NewId()
@@ -2419,6 +2429,9 @@ func testChannelStoreAutocompleteInTeamForSearch(t *testing.T, ss store.Store) {
 			require.Len(t, *channels, 2)
 		})
 	}
+
+	// Manually truncate Channels table until testlib can handle cleanups
+	s.GetMaster().Exec("TRUNCATE Channels")
 }

 func testChannelStoreGetMembersByIds(t *testing.T, ss store.Store) {
@@ -3158,3 +3171,160 @@ func testChannelStoreRemoveAllDeactivatedMembers(t *testing.T, ss store.Store) {
 	assert.Len(t, *d2, 1)
 	assert.Equal(t, (*d2)[0].UserId, u3.Id)
 }
+
+func testChannelStoreExportAllDirectChannels(t *testing.T, ss store.Store, s SqlSupplier) {
+	teamId := model.NewId()
+
+	o1 := model.Channel{}
+	o1.TeamId = teamId
+	o1.DisplayName = "Name" + model.NewId()
+	o1.Name = "zz" + model.NewId() + "b"
+	o1.Type = model.CHANNEL_DIRECT
+
+	userIds := []string{model.NewId(), model.NewId(), model.NewId()}
+
+	o2 := model.Channel{}
+	o2.Name = model.GetGroupNameFromUserIds(userIds)
+	o2.DisplayName = "GroupChannel" + model.NewId()
+	o2.Name = "zz" + model.NewId() + "b"
+	o2.Type = model.CHANNEL_GROUP
+	store.Must(ss.Channel().Save(&o2, -1))
+
+	u1 := &model.User{}
+	u1.Email = MakeEmail()
+	u1.Nickname = model.NewId()
+	store.Must(ss.User().Save(u1))
+	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1))
+
+	u2 := &model.User{}
+	u2.Email = MakeEmail()
+	u2.Nickname = model.NewId()
+	store.Must(ss.User().Save(u2))
+	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u2.Id}, -1))
+
+	m1 := model.ChannelMember{}
+	m1.ChannelId = o1.Id
+	m1.UserId = u1.Id
+	m1.NotifyProps = model.GetDefaultChannelNotifyProps()
+
+	m2 := model.ChannelMember{}
+	m2.ChannelId = o1.Id
+	m2.UserId = u2.Id
+	m2.NotifyProps = model.GetDefaultChannelNotifyProps()
+
+	<-ss.Channel().SaveDirectChannel(&o1, &m1, &m2)
+
+	r1 := <-ss.Channel().GetAllDirectChannelsForExportAfter(10000, strings.Repeat("0", 26))
+	assert.Nil(t, r1.Err)
+	d1 := r1.Data.([]*model.DirectChannelForExport)
+
+	assert.Equal(t, 2, len(d1))
+	assert.ElementsMatch(t, []string{o1.DisplayName, o2.DisplayName}, []string{d1[0].DisplayName, d1[1].DisplayName})
+
+	// Manually truncate Channels table until testlib can handle cleanups
+	s.GetMaster().Exec("TRUNCATE Channels")
+}
+
+func testChannelStoreExportAllDirectChannelsExcludePrivateAndPublic(t *testing.T, ss store.Store, s SqlSupplier) {
+	teamId := model.NewId()
+
+	o1 := model.Channel{}
+	o1.TeamId = teamId
+	o1.DisplayName = "The Direct Channel" + model.NewId()
+	o1.Name = "zz" + model.NewId() + "b"
+	o1.Type = model.CHANNEL_DIRECT
+
+	o2 := model.Channel{}
+	o2.TeamId = teamId
+	o2.DisplayName = "Channel2" + model.NewId()
+	o2.Name = "zz" + model.NewId() + "b"
+	o2.Type = model.CHANNEL_OPEN
+	store.Must(ss.Channel().Save(&o2, -1))
+
+	o3 := model.Channel{}
+	o3.TeamId = teamId
+	o3.DisplayName = "Channel3" + model.NewId()
+	o3.Name = "zz" + model.NewId() + "b"
+	o3.Type = model.CHANNEL_PRIVATE
+	store.Must(ss.Channel().Save(&o3, -1))
+
+	u1 := &model.User{}
+	u1.Email = MakeEmail()
+	u1.Nickname = model.NewId()
+	store.Must(ss.User().Save(u1))
+	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1))
+
+	u2 := &model.User{}
+	u2.Email = MakeEmail()
+	u2.Nickname = model.NewId()
+	store.Must(ss.User().Save(u2))
+	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u2.Id}, -1))
+
+	m1 := model.ChannelMember{}
+	m1.ChannelId = o1.Id
+	m1.UserId = u1.Id
+	m1.NotifyProps = model.GetDefaultChannelNotifyProps()
+
+	m2 := model.ChannelMember{}
+	m2.ChannelId = o1.Id
+	m2.UserId = u2.Id
+	m2.NotifyProps = model.GetDefaultChannelNotifyProps()
+
+	<-ss.Channel().SaveDirectChannel(&o1, &m1, &m2)
+
+	r1 := <-ss.Channel().GetAllDirectChannelsForExportAfter(10000, strings.Repeat("0", 26))
+	assert.Nil(t, r1.Err)
+	d1 := r1.Data.([]*model.DirectChannelForExport)
+	assert.Equal(t, 1, len(d1))
+	assert.Equal(t, o1.DisplayName, d1[0].DisplayName)
+
+	// Manually truncate Channels table until testlib can handle cleanups
+	s.GetMaster().Exec("TRUNCATE Channels")
+}
+
+func testChannelStoreExportAllDirectChannelsDeletedChannel(t *testing.T, ss store.Store, s SqlSupplier) {
+	teamId := model.NewId()
+
+	o1 := model.Channel{}
+	o1.TeamId = teamId
+	o1.DisplayName = "Different Name" + model.NewId()
+	o1.Name = "zz" + model.NewId() + "b"
+	o1.Type = model.CHANNEL_DIRECT
+
+	u1 := &model.User{}
+	u1.Email = MakeEmail()
+	u1.Nickname = model.NewId()
+	store.Must(ss.User().Save(u1))
+	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1))
+
+	u2 := &model.User{}
+	u2.Email = MakeEmail()
+	u2.Nickname = model.NewId()
+	store.Must(ss.User().Save(u2))
+	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u2.Id}, -1))
+
+	m1 := model.ChannelMember{}
+	m1.ChannelId = o1.Id
+	m1.UserId = u1.Id
+	m1.NotifyProps = model.GetDefaultChannelNotifyProps()
+
+	m2 := model.ChannelMember{}
+	m2.ChannelId = o1.Id
+	m2.UserId = u2.Id
+	m2.NotifyProps = model.GetDefaultChannelNotifyProps()
+
+	result := <-ss.Channel().SaveDirectChannel(&o1, &m1, &m2)
+
+	o1.DeleteAt = 1
+	result = <-ss.Channel().SetDeleteAt(o1.Id, 1, 1)
+	assert.Nil(t, result.Err)
+
+	r1 := <-ss.Channel().GetAllDirectChannelsForExportAfter(10000, strings.Repeat("0", 26))
+	assert.Nil(t, r1.Err)
+	d1 := r1.Data.([]*model.DirectChannelForExport)
+
+	assert.Equal(t, 0, len(d1))
+
+	// Manually truncate Channels table until testlib can handle cleanups
+	s.GetMaster().Exec("TRUNCATE Channels")
+}
--- a/store/storetest/mocks/ChannelStore.go
+++ b/store/storetest/mocks/ChannelStore.go
@@ -226,6 +226,22 @@ func (_m *ChannelStore) GetAllChannelsForExportAfter(limit int, afterId string)
 	return r0
 }

+// GetAllDirectChannelsForExportAfter provides a mock function with given fields: limit, afterId
+func (_m *ChannelStore) GetAllDirectChannelsForExportAfter(limit int, afterId string) store.StoreChannel {
+	ret := _m.Called(limit, afterId)
+
+	var r0 store.StoreChannel
+	if rf, ok := ret.Get(0).(func(int, string) store.StoreChannel); ok {
+		r0 = rf(limit, afterId)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(store.StoreChannel)
+		}
+	}
+
+	return r0
+}
+
 // GetByName provides a mock function with given fields: team_id, name, allowFromCache
 func (_m *ChannelStore) GetByName(team_id string, name string, allowFromCache bool) store.StoreChannel {
 	ret := _m.Called(team_id, name, allowFromCache)
--- a/store/storetest/mocks/PostStore.go
+++ b/store/storetest/mocks/PostStore.go
@@ -470,3 +470,18 @@ func (_m *PostStore) Update(newPost *model.Post, oldPost *model.Post) store.Stor

 	return r0
 }
+
+func (_m *PostStore) GetDirectPostParentsForExportAfter(limit int, afterId string) store.StoreChannel {
+	ret := _m.Called(limit, afterId)
+
+	var r0 store.StoreChannel
+	if rf, ok := ret.Get(0).(func(int, string) store.StoreChannel); ok {
+		r0 = rf(limit, afterId)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(store.StoreChannel)
+		}
+	}
+
+	return r0
+}
--- a/store/storetest/post_store.go
+++ b/store/storetest/post_store.go
@@ -5,6 +5,7 @@ package storetest

 import (
 	"fmt"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -16,7 +17,7 @@ import (
 	"github.com/stretchr/testify/require"
 )

-func TestPostStore(t *testing.T, ss store.Store) {
+func TestPostStore(t *testing.T, ss store.Store, s SqlSupplier) {
 	t.Run("Save", func(t *testing.T) { testPostStoreSave(t, ss) })
 	t.Run("SaveAndUpdateChannelMsgCounts", func(t *testing.T) { testPostStoreSaveChannelMsgCounts(t, ss) })
 	t.Run("Get", func(t *testing.T) { testPostStoreGet(t, ss) })
@@ -35,7 +36,7 @@ func TestPostStore(t *testing.T, ss store.Store) {
 	t.Run("Search", func(t *testing.T) { testPostStoreSearch(t, ss) })
 	t.Run("UserCountsWithPostsByDay", func(t *testing.T) { testUserCountsWithPostsByDay(t, ss) })
 	t.Run("PostCountsByDay", func(t *testing.T) { testPostCountsByDay(t, ss) })
-	t.Run("GetFlaggedPostsForTeam", func(t *testing.T) { testPostStoreGetFlaggedPostsForTeam(t, ss) })
+	t.Run("GetFlaggedPostsForTeam", func(t *testing.T) { testPostStoreGetFlaggedPostsForTeam(t, ss, s) })
 	t.Run("GetFlaggedPosts", func(t *testing.T) { testPostStoreGetFlaggedPosts(t, ss) })
 	t.Run("GetFlaggedPostsForChannel", func(t *testing.T) { testPostStoreGetFlaggedPostsForChannel(t, ss) })
 	t.Run("GetPostsCreatedAt", func(t *testing.T) { testPostStoreGetPostsCreatedAt(t, ss) })
@@ -47,6 +48,9 @@ func TestPostStore(t *testing.T, ss store.Store) {
 	t.Run("TestGetMaxPostSize", func(t *testing.T) { testGetMaxPostSize(t, ss) })
 	t.Run("GetParentsForExportAfter", func(t *testing.T) { testPostStoreGetParentsForExportAfter(t, ss) })
 	t.Run("GetRepliesForExport", func(t *testing.T) { testPostStoreGetRepliesForExport(t, ss) })
+	t.Run("GetDirectPostParentsForExportAfter", func(t *testing.T) { testPostStoreGetDirectPostParentsForExportAfter(t, ss, s) })
+	t.Run("GetDirectPostParentsForExportAfterDeleted", func(t *testing.T) { testPostStoreGetDirectPostParentsForExportAfterDeleted(t, ss, s) })
+	t.Run("GetDirectPostParentsForExportAfterBatched", func(t *testing.T) { testPostStoreGetDirectPostParentsForExportAfterBatched(t, ss, s) })
 }

 func testPostStoreSave(t *testing.T, ss store.Store) {
@@ -1192,7 +1196,7 @@ func testPostCountsByDay(t *testing.T, ss store.Store) {
 	}
 }

-func testPostStoreGetFlaggedPostsForTeam(t *testing.T, ss store.Store) {
+func testPostStoreGetFlaggedPostsForTeam(t *testing.T, ss store.Store, s SqlSupplier) {
 	c1 := &model.Channel{}
 	c1.TeamId = model.NewId()
 	c1.DisplayName = "Channel1"
@@ -1365,6 +1369,9 @@ func testPostStoreGetFlaggedPostsForTeam(t *testing.T, ss store.Store) {
 	if len(r4.Order) != 3 {
 		t.Fatal("should have 3 posts")
 	}
+
+	// Manually truncate Channels table until testlib can handle cleanups
+	s.GetMaster().Exec("TRUNCATE Channels")
 }

 func testPostStoreGetFlaggedPosts(t *testing.T, ss store.Store) {
@@ -1972,3 +1979,190 @@ func testPostStoreGetRepliesForExport(t *testing.T, ss store.Store) {
 	assert.Equal(t, reply1.Username, u1.Username)

 }
+
+func testPostStoreGetDirectPostParentsForExportAfter(t *testing.T, ss store.Store, s SqlSupplier) {
+	teamId := model.NewId()
+
+	o1 := model.Channel{}
+	o1.TeamId = teamId
+	o1.DisplayName = "Name"
+	o1.Name = "zz" + model.NewId() + "b"
+	o1.Type = model.CHANNEL_DIRECT
+
+	u1 := &model.User{}
+	u1.Email = MakeEmail()
+	u1.Nickname = model.NewId()
+	store.Must(ss.User().Save(u1))
+	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1))
+
+	u2 := &model.User{}
+	u2.Email = MakeEmail()
+	u2.Nickname = model.NewId()
+	store.Must(ss.User().Save(u2))
+	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u2.Id}, -1))
+
+	m1 := model.ChannelMember{}
+	m1.ChannelId = o1.Id
+	m1.UserId = u1.Id
+	m1.NotifyProps = model.GetDefaultChannelNotifyProps()
+
+	m2 := model.ChannelMember{}
+	m2.ChannelId = o1.Id
+	m2.UserId = u2.Id
+	m2.NotifyProps = model.GetDefaultChannelNotifyProps()
+
+	<-ss.Channel().SaveDirectChannel(&o1, &m1, &m2)
+
+	p1 := &model.Post{}
+	p1.ChannelId = o1.Id
+	p1.UserId = u1.Id
+	p1.Message = "zz" + model.NewId() + "AAAAAAAAAAA"
+	p1.CreateAt = 1000
+	p1 = (<-ss.Post().Save(p1)).Data.(*model.Post)
+
+	r1 := <-ss.Post().GetDirectPostParentsForExportAfter(10000, strings.Repeat("0", 26))
+	assert.Nil(t, r1.Err)
+	d1 := r1.Data.([]*model.DirectPostForExport)
+
+	assert.Equal(t, p1.Message, d1[0].Message)
+
+	// Manually truncate Channels table until testlib can handle cleanups
+	s.GetMaster().Exec("TRUNCATE Channels")
+}
+
+func testPostStoreGetDirectPostParentsForExportAfterDeleted(t *testing.T, ss store.Store, s SqlSupplier) {
+	teamId := model.NewId()
+
+	o1 := model.Channel{}
+	o1.TeamId = teamId
+	o1.DisplayName = "Name"
+	o1.Name = "zz" + model.NewId() + "b"
+	o1.Type = model.CHANNEL_DIRECT
+
+	u1 := &model.User{}
+	u1.DeleteAt = 1
+	u1.Email = MakeEmail()
+	u1.Nickname = model.NewId()
+	store.Must(ss.User().Save(u1))
+	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1))
+
+	u2 := &model.User{}
+	u2.DeleteAt = 1
+	u2.Email = MakeEmail()
+	u2.Nickname = model.NewId()
+	store.Must(ss.User().Save(u2))
+	store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u2.Id}, -1))
+
+	m1 := model.ChannelMember{}
+	m1.ChannelId = o1.Id
+	m1.UserId = u1.Id
+	m1.NotifyProps = model.GetDefaultChannelNotifyProps()
+
+	m2 := model.ChannelMember{}
+	m2.ChannelId = o1.Id
+	m2.UserId = u2.Id
+	m2.NotifyProps = model.GetDefaultChannelNotifyProps()
+
+	<-ss.Channel().SaveDirectChannel(&o1, &m1, &m2)
+
+	o1.DeleteAt = 1
+	result := <-ss.Channel().SetDeleteAt(o1.Id, 1, 1)
+	assert.Nil(t, result.Err)
+
+	p1 := &model.Post{}
+	p1.ChannelId = o1.Id
+	p1.UserId = u1.Id
+	p1.Message = "zz" + model.NewId() + "BBBBBBBBBBBB"
+	p1.CreateAt = 1000
+	p1 = (<-ss.Post().Save(p1)).Data.(*model.Post)
+
+	o1a := &model.Post{}
+	*o1a = *p1
+	o1a.DeleteAt = 1
+	o1a.Message = p1.Message + "BBBBBBBBBB"
+	if result := <-ss.Post().Update(o1a, p1); result.Err != nil {
+		t.Fatal(result.Err)
+	}
+
+	r1 := <-ss.Post().GetDirectPostParentsForExportAfter(10000, strings.Repeat("0", 26))
+	assert.Nil(t, r1.Err)
+	d1 := r1.Data.([]*model.DirectPostForExport)
+
+	assert.Equal(t, 0, len(d1))
+
+	// Manually truncate Channels table until testlib can handle cleanups
+	s.GetMaster().Exec("TRUNCATE Channels")
+}
+
+func testPostStoreGetDirectPostParentsForExportAfterBatched(t *testing.T, ss store.Store, s SqlSupplier) {
+	teamId := model.NewId()
+
+	o1 := model.Channel{}
+	o1.TeamId = teamId
+	o1.DisplayName = "Name"
+	o1.Name = "zz" + model.NewId() + "b"
+	o1.Type = model.CHANNEL_DIRECT
+
+	var postIds []string
+	for i := 0; i < 150; i++ {
+		u1 := &model.User{}
+		u1.Email = MakeEmail()
+		u1.Nickname = model.NewId()
+		store.Must(ss.User().Save(u1))
+		store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1))
+
+		u2 := &model.User{}
+		u2.Email = MakeEmail()
+		u2.Nickname = model.NewId()
+		store.Must(ss.User().Save(u2))
+		store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u2.Id}, -1))
+
+		m1 := model.ChannelMember{}
+		m1.ChannelId = o1.Id
+		m1.UserId = u1.Id
+		m1.NotifyProps = model.GetDefaultChannelNotifyProps()
+
+		m2 := model.ChannelMember{}
+		m2.ChannelId = o1.Id
+		m2.UserId = u2.Id
+		m2.NotifyProps = model.GetDefaultChannelNotifyProps()
+
+		<-ss.Channel().SaveDirectChannel(&o1, &m1, &m2)
+
+		p1 := &model.Post{}
+		p1.ChannelId = o1.Id
+		p1.UserId = u1.Id
+		p1.Message = "zz" + model.NewId() + "AAAAAAAAAAA"
+		p1.CreateAt = 1000
+		p1 = (<-ss.Post().Save(p1)).Data.(*model.Post)
+		postIds = append(postIds, p1.Id)
+	}
+	sort.Slice(postIds, func(i, j int) bool { return postIds[i] < postIds[j] })
+
+	// Get all posts
+	r1 := <-ss.Post().GetDirectPostParentsForExportAfter(10000, strings.Repeat("0", 26))
+	assert.Nil(t, r1.Err)
+	d1 := r1.Data.([]*model.DirectPostForExport)
+	assert.Equal(t, len(postIds), len(d1))
+	var exportedPostIds []string
+	for i := range d1 {
+		exportedPostIds = append(exportedPostIds, d1[i].Id)
+	}
+	sort.Slice(exportedPostIds, func(i, j int) bool { return exportedPostIds[i] < exportedPostIds[j] })
+	assert.ElementsMatch(t, postIds, exportedPostIds)
+
+	// Get 100
+	r1 = <-ss.Post().GetDirectPostParentsForExportAfter(100, strings.Repeat("0", 26))
+	assert.Nil(t, r1.Err)
+	d1 = r1.Data.([]*model.DirectPostForExport)
+	assert.Equal(t, 100, len(d1))
+	exportedPostIds = []string{}
+	for i := range d1 {
+		exportedPostIds = append(exportedPostIds, d1[i].Id)
+	}
+	sort.Slice(exportedPostIds, func(i, j int) bool { return exportedPostIds[i] < exportedPostIds[j] })
+	assert.ElementsMatch(t, postIds[:100], exportedPostIds)
+
+	// Manually truncate Channels table until testlib can handle cleanups
+	s.GetMaster().Exec("TRUNCATE Channels")
+}