[MM 12464] Include DM/GM Channels and Their Posts in the Bulk Export (#10421)

* transplant the existing PR into the working tree

* start addressing review comments

* move existing direct channel export code into this branch

* modify channel exporter to use squirell and populate members in two steps

* use squirrel to build sql queries for channel and dm/gm export methods

* remove debug helpers and use Username instead of UserId

* unit test for DM Channel exporter

* add more unit tests for channel export

* add test for DM/GM post export

* checkpoint with failing test for postgres

* use getQueryBuilder to make sure squirrel uses the correct formatting
for each database

* add a test for post export

* fix shadowed vars that broke the build

* address review comments and add tests to support it

* address review comments and add a mlog call

* s/Info/Debug/

* address review comments in post_store

* address review comments in channel_store

* address review comments in export

* address review comment in post_store: drop GroupBy

* address review comment on supplier: move getQueryBuilder to sqlstore

* address review comments: explicit TearDown

* address review comments: improve test coverage

* address review comments: make sure public and private channels are excluded

* address review comments: improve test coverage

* address review comments: make sure Channels table gets truncated after
each test

* more cleanups and better assertions

* wrap PostStore in a StoreTestWithSqlSupplier

* last minute changes: improve post export test coverage and check members

* address review comments: make sure all posts have their channel
members set

* address review comments: make sure all posts have their ChannelMembers
exported correctly

* gofmt fix

* sort channels so it's possible to assert on index

app/export.go

@@ -11,6 +11,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/mattermost/mattermost-server/mlog"
 	"github.com/mattermost/mattermost-server/model"
 	"github.com/pkg/errors"
 )
@@ -70,10 +71,19 @@ func (a *App) BulkExport(writer io.Writer, file string, pathToEmojiDir string, d
 	if err := a.ExportAllPosts(writer); err != nil {
 		return err
 	}
+
 	if err := a.ExportCustomEmoji(writer, file, pathToEmojiDir, dirNameToExportEmoji); err != nil {
 		return err
 	}
 
+	if err := a.ExportAllDirectChannels(writer); err != nil {
+		return err
+	}
+
+	if err := a.ExportAllDirectPosts(writer); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -503,3 +513,83 @@ func (a *App) copyEmojiImages(emojiId string, emojiImagePath string, pathToDir s
 
 	return nil
 }
+
+func (a *App) ExportAllDirectChannels(writer io.Writer) *model.AppError {
+	afterId := strings.Repeat("0", 26)
+	for {
+		result := <-a.Srv.Store.Channel().GetAllDirectChannelsForExportAfter(1000, afterId)
+		if result.Err != nil {
+			return result.Err
+		}
+
+		channels := result.Data.([]*model.DirectChannelForExport)
+		if len(channels) == 0 {
+			break
+		}
+
+		for _, channel := range channels {
+			afterId = channel.Id
+
+			// Skip deleted.
+			if channel.DeleteAt != 0 {
+				continue
+			}
+
+			// There's no import support for single member channels yet.
+			if len(*channel.Members) == 1 {
+				mlog.Debug("Bulk export for direct channels containing a single member is not supported.")
+				continue
+			}
+
+			channelLine := ImportLineFromDirectChannel(channel)
+			if err := a.ExportWriteLine(writer, channelLine); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (a *App) ExportAllDirectPosts(writer io.Writer) *model.AppError {
+	afterId := strings.Repeat("0", 26)
+	for {
+		result := <-a.Srv.Store.Post().GetDirectPostParentsForExportAfter(1000, afterId)
+		if result.Err != nil {
+			return result.Err
+		}
+
+		posts := result.Data.([]*model.DirectPostForExport)
+		if len(posts) == 0 {
+			break
+		}
+
+		for _, post := range posts {
+			afterId = post.Id
+
+			// Skip deleted.
+			if post.DeleteAt != 0 {
+				continue
+			}
+
+			// There's no import support for single member channels yet.
+			if len(*post.ChannelMembers) == 1 {
+				mlog.Debug("Bulk export for posts containing a single member is not supported.")
+				continue
+			}
+
+			// Do the Replies.
+			replies, err := a.buildPostReplies(post.Id)
+			if err != nil {
+				return err
+			}
+
+			postLine := ImportLineForDirectPost(post)
+			postLine.DirectPost.Replies = replies
+			if err := a.ExportWriteLine(writer, postLine); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}