Improve bulk export of posts (#11702)

This change modifies some of the logic for processing posts during bulk export in order to improve performance.
2019-07-25 12:20:49 -04:00
--- a/app/export.go
+++ b/app/export.go
@@ -52,34 +52,42 @@ var exportablePreferences = map[ComparablePreference]string{{
 }

 func (a *App) BulkExport(writer io.Writer, file string, pathToEmojiDir string, dirNameToExportEmoji string) *model.AppError {
+	mlog.Info("Bulk export: exporting version")
 	if err := a.ExportVersion(writer); err != nil {
 		return err
 	}

+	mlog.Info("Bulk export: exporting teams")
 	if err := a.ExportAllTeams(writer); err != nil {
 		return err
 	}

+	mlog.Info("Bulk export: exporting channels")
 	if err := a.ExportAllChannels(writer); err != nil {
 		return err
 	}

+	mlog.Info("Bulk export: exporting users")
 	if err := a.ExportAllUsers(writer); err != nil {
 		return err
 	}

+	mlog.Info("Bulk export: exporting posts")
 	if err := a.ExportAllPosts(writer); err != nil {
 		return err
 	}

+	mlog.Info("Bulk export: exporting emoji")
 	if err := a.ExportCustomEmoji(writer, file, pathToEmojiDir, dirNameToExportEmoji); err != nil {
 		return err
 	}

+	mlog.Info("Bulk export: exporting direct channels")
 	if err := a.ExportAllDirectChannels(writer); err != nil {
 		return err
 	}

+	mlog.Info("Bulk export: exporting direct posts")
 	if err := a.ExportAllDirectPosts(writer); err != nil {
 		return err
 	}
@@ -327,15 +335,15 @@ func (a *App) buildUserNotifyProps(notifyProps model.StringMap) *UserNotifyProps

 func (a *App) ExportAllPosts(writer io.Writer) *model.AppError {
 	afterId := strings.Repeat("0", 26)
+
 	for {
 		posts, err := a.Srv.Store.Post().GetParentsForExportAfter(1000, afterId)
-
 		if err != nil {
 			return err
 		}

 		if len(posts) == 0 {
-			break
+			return nil
 		}

 		for _, post := range posts {
@@ -348,47 +356,41 @@ func (a *App) ExportAllPosts(writer io.Writer) *model.AppError {

 			postLine := ImportLineForPost(post)

-			// Do the Replies.
-			replies, err := a.buildPostReplies(post.Id)
+			postLine.Post.Replies, err = a.buildPostReplies(post.Id)
 			if err != nil {
 				return err
 			}

-			reactions, err := a.BuildPostReactions(post.Id)
-			if err != nil {
-				return err
+			postLine.Post.Reactions = &[]ReactionImportData{}
+			if post.HasReactions {
+				postLine.Post.Reactions, err = a.BuildPostReactions(post.Id)
+				if err != nil {
+					return err
+				}
 			}

-			postLine.Post.Replies = replies
-
-			postLine.Post.Reactions = reactions
-
 			if err := a.ExportWriteLine(writer, postLine); err != nil {
 				return err
 			}
 		}
 	}
-
-	return nil
 }

 func (a *App) buildPostReplies(postId string) (*[]ReplyImportData, *model.AppError) {
 	var replies []ReplyImportData

 	replyPosts, err := a.Srv.Store.Post().GetRepliesForExport(postId)
-
 	if err != nil {
 		return nil, err
 	}

 	for _, reply := range replyPosts {
 		replyImportObject := ImportReplyFromPost(reply)
-		if reply.HasReactions == true {
-			reactionsOfReply, err := a.BuildPostReactions(reply.Id)
+		if reply.HasReactions {
+			replyImportObject.Reactions, err = a.BuildPostReactions(reply.Id)
 			if err != nil {
 				return nil, err
 			}
-			replyImportObject.Reactions = reactionsOfReply
 		}
 		replies = append(replies, *replyImportObject)
 	}
--- a/cmd/mattermost/commands/export.go
+++ b/cmd/mattermost/commands/export.go
@@ -4,14 +4,12 @@
 package commands

 import (
-	"errors"
-	"os"
-
 	"context"
-
+	"os"
 	"time"

 	"github.com/mattermost/mattermost-server/model"
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )

@@ -57,7 +55,7 @@ var BulkExportCmd = &cobra.Command{
 	Use:     "bulk [file]",
 	Short:   "Export bulk data.",
 	Long:    "Export data to a file compatible with the Mattermost Bulk Import format.",
-	Example: "  export bulk bulk_data.json",
+	Example: "export bulk bulk_data.json",
 	RunE:    bulkExportCmdF,
 	Args:    cobra.ExactArgs(1),
 }
@@ -72,7 +70,7 @@ func init() {
 	ActianceExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
 	GlobalRelayZipExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")

-	BulkExportCmd.Flags().Bool("all-teams", false, "Export all teams from the server.")
+	BulkExportCmd.Flags().Bool("all-teams", true, "Export all teams from the server.")

 	ExportCmd.AddCommand(ScheduleExportCmd)
 	ExportCmd.AddCommand(CsvExportCmd)
@@ -177,9 +175,8 @@ func bulkExportCmdF(command *cobra.Command, args []string) error {

 	allTeams, err := command.Flags().GetBool("all-teams")
 	if err != nil {
-		return errors.New("Apply flag error")
+		return errors.Wrap(err, "all-teams flag error")
 	}
-
 	if !allTeams {
 		return errors.New("Nothing to export. Please specify the --all-teams flag to export all teams.")
 	}
--- a/store/sqlstore/post_store.go
+++ b/store/sqlstore/post_store.go
@@ -1312,42 +1312,68 @@ func (s *SqlPostStore) GetMaxPostSize() int {
 }

 func (s *SqlPostStore) GetParentsForExportAfter(limit int, afterId string) ([]*model.PostForExport, *model.AppError) {
-	var posts []*model.PostForExport
-	_, err := s.GetSearchReplica().Select(&posts, `
-                SELECT
-                    p1.*,
-                    Users.Username as Username,
-                    Teams.Name as TeamName,
-                    Channels.Name as ChannelName
-                FROM
-                    Posts p1
-                INNER JOIN
-                    Channels ON p1.ChannelId = Channels.Id
-                INNER JOIN
-                    Teams ON Channels.TeamId = Teams.Id
-                INNER JOIN
-                    Users ON p1.UserId = Users.Id
-                WHERE
-                    p1.Id > :AfterId
-                    AND p1.ParentId = ''
-                    AND p1.DeleteAt = 0
-					AND Channels.DeleteAt = 0
-					AND Teams.DeleteAt = 0
-                ORDER BY
-                    p1.Id
-                LIMIT
-                    :Limit`,
-		map[string]interface{}{"Limit": limit, "AfterId": afterId})
+	for {
+		var rootIds []string
+		_, err := s.GetReplica().Select(&rootIds,
+			`SELECT
+				Id
+			FROM
+				Posts
+			WHERE
+				Id > :AfterId
+				AND RootId = ''
+				AND DeleteAt = 0
+			ORDER BY Id
+			LIMIT :Limit`,
+			map[string]interface{}{"Limit": limit, "AfterId": afterId})
+		if err != nil {
+			return nil, model.NewAppError("SqlPostStore.GetAllAfterForExport", "store.sql_post.get_posts.app_error",
+				nil, err.Error(), http.StatusInternalServerError)
+		}

-	if err != nil {
-		return nil, model.NewAppError("SqlPostStore.GetAllAfterForExport", "store.sql_post.get_posts.app_error",
-			nil, err.Error(), http.StatusInternalServerError)
+		var postsForExport []*model.PostForExport
+		if len(rootIds) == 0 {
+			return postsForExport, nil
+		}
+
+		keys, params := MapStringsToQueryParams(rootIds, "PostId")
+		_, err = s.GetSearchReplica().Select(&postsForExport, `
+			SELECT
+				p1.*,
+				Users.Username as Username,
+				Teams.Name as TeamName,
+				Channels.Name as ChannelName
+			FROM
+				(Select * FROM Posts WHERE Id IN `+keys+`) p1
+			INNER JOIN
+				Channels ON p1.ChannelId = Channels.Id
+			INNER JOIN
+				Teams ON Channels.TeamId = Teams.Id
+			INNER JOIN
+				Users ON p1.UserId = Users.Id
+			WHERE
+				Channels.DeleteAt = 0
+				AND Teams.DeleteAt = 0
+			ORDER BY
+				p1.Id`,
+			params)
+		if err != nil {
+			return nil, model.NewAppError("SqlPostStore.GetAllAfterForExport", "store.sql_post.get_posts.app_error",
+				nil, err.Error(), http.StatusInternalServerError)
+		}
+
+		if len(postsForExport) == 0 {
+			// All of the posts were in channels or teams that were deleted.
+			// Update the afterId and try again.
+			afterId = rootIds[len(rootIds)-1]
+			continue
+		}
+
+		return postsForExport, nil
 	}
-	return posts, nil
 }

-func (s *SqlPostStore) GetRepliesForExport(parentId string) ([]*model.ReplyForExport, *model.AppError) {
-
+func (s *SqlPostStore) GetRepliesForExport(rootId string) ([]*model.ReplyForExport, *model.AppError) {
 	var posts []*model.ReplyForExport
 	_, err := s.GetSearchReplica().Select(&posts, `
 			SELECT
@@ -1358,11 +1384,11 @@ func (s *SqlPostStore) GetRepliesForExport(parentId string) ([]*model.ReplyForEx
 			INNER JOIN
 				Users ON Posts.UserId = Users.Id
 			WHERE
-				Posts.ParentId = :ParentId
+				Posts.RootId = :RootId
 				AND Posts.DeleteAt = 0
 			ORDER BY
 				Posts.Id`,
-		map[string]interface{}{"ParentId": parentId})
+		map[string]interface{}{"RootId": rootId})

 	if err != nil {
 		return nil, model.NewAppError("SqlPostStore.GetAllAfterForExport", "store.sql_post.get_posts.app_error", nil, err.Error(), http.StatusInternalServerError)