[MM 12464] Include DM/GM Channels and Their Posts in the Bulk Export (#10421)

* transplant the existing PR into the working tree

* start addressing review comments

* move existing direct channel export code into this branch

* modify channel exporter to use squirell and populate members in two steps

* use squirrel to build sql queries for channel and dm/gm export methods

* remove debug helpers and use Username instead of UserId

* unit test for DM Channel exporter

* add more unit tests for channel export

* add test for DM/GM post export

* checkpoint with failing test for postgres

* use getQueryBuilder to make sure squirrel uses the correct formatting
for each database

* add a test for post export

* fix shadowed vars that broke the build

* address review comments and add tests to support it

* address review comments and add a mlog call

* s/Info/Debug/

* address review comments in post_store

* address review comments in channel_store

* address review comments in export

* address review comment in post_store: drop GroupBy

* address review comment on supplier: move getQueryBuilder to sqlstore

* address review comments: explicit TearDown

* address review comments: improve test coverage

* address review comments: make sure public and private channels are excluded

* address review comments: improve test coverage

* address review comments: make sure Channels table gets truncated after
each test

* more cleanups and better assertions

* wrap PostStore in a StoreTestWithSqlSupplier

* last minute changes: improve post export test coverage and check members

* address review comments: make sure all posts have their channel
members set

* address review comments: make sure all posts have their ChannelMembers
exported correctly

* gofmt fix

* sort channels so it's possible to assert on index

store/sqlstore/post_store.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"sync"
 
+	sq "github.com/Masterminds/squirrel"
 	"github.com/mattermost/mattermost-server/einterfaces"
 	"github.com/mattermost/mattermost-server/mlog"
 	"github.com/mattermost/mattermost-server/model"
@@ -1366,3 +1367,77 @@ func (s *SqlPostStore) GetRepliesForExport(parentId string) store.StoreChannel {
 		}
 	})
 }
+
+func (s *SqlPostStore) GetDirectPostParentsForExportAfter(limit int, afterId string) store.StoreChannel {
+	return store.Do(func(result *store.StoreResult) {
+		query := s.getQueryBuilder().
+			Select("p.*", "Users.Username as User").
+			From("Posts p").
+			Join("Channels ON p.ChannelId = Channels.Id").
+			Join("Users ON p.UserId = Users.Id").
+			Where(sq.And{
+				sq.Gt{"p.Id": afterId},
+				sq.Eq{"p.ParentId": string("")},
+				sq.Eq{"p.DeleteAt": int(0)},
+				sq.Eq{"Channels.DeleteAt": int(0)},
+				sq.Eq{"Users.DeleteAt": int(0)},
+				sq.Eq{"Channels.Type": []string{"D", "G"}},
+			}).
+			OrderBy("p.Id").
+			Limit(uint64(limit))
+
+		queryString, args, err := query.ToSql()
+		if err != nil {
+			result.Err = model.NewAppError("SqlPostStore.GetDirectPostParentsForExportAfter", "store.sql_post.get_direct_posts.app_error", nil, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		var posts []*model.DirectPostForExport
+		if _, err = s.GetReplica().Select(&posts, queryString, args...); err != nil {
+			result.Err = model.NewAppError("SqlPostStore.GetDirectPostParentsForExportAfter", "store.sql_post.get_direct_posts.app_error", nil, err.Error(), http.StatusInternalServerError)
+		}
+		var channelIds []string
+		for _, post := range posts {
+			channelIds = append(channelIds, post.ChannelId)
+		}
+		query = s.getQueryBuilder().
+			Select("*").
+			From("ChannelMembers cm").
+			Join("Users u ON ( u.Id = cm.UserId )").
+			Where(sq.Eq{
+				"cm.ChannelId": channelIds,
+			})
+
+		queryString, args, err = query.ToSql()
+		if err != nil {
+			result.Err = model.NewAppError("SqlPostStore.GetDirectPostParentsForExportAfter", "store.sql_post.get_direct_posts.app_error", nil, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		var channelMembers []*model.ChannelMemberForExport
+		if _, err := s.GetReplica().Select(&channelMembers, queryString, args...); err != nil {
+			result.Err = model.NewAppError("SqlPostStore.GetDirectPostParentsForExportAfter", "store.sql_post.get_direct_posts.app_error", nil, err.Error(), http.StatusInternalServerError)
+		}
+
+		// Build a map of channels and their posts
+		postsChannelMap := make(map[string][]*model.DirectPostForExport)
+		for _, post := range posts {
+			post.ChannelMembers = &[]string{}
+			postsChannelMap[post.ChannelId] = append(postsChannelMap[post.ChannelId], post)
+		}
+
+		// Build a map of channels and their members
+		channelMembersMap := make(map[string][]string)
+		for _, member := range channelMembers {
+			channelMembersMap[member.ChannelId] = append(channelMembersMap[member.ChannelId], member.Username)
+		}
+
+		// Populate each post ChannelMembers extracting it from the channelMembersMap
+		for channelId := range channelMembersMap {
+			for _, post := range postsChannelMap[channelId] {
+				*post.ChannelMembers = channelMembersMap[channelId]
+			}
+		}
+		result.Data = posts
+	})
+}