diff --git a/app/export.go b/app/export.go index 907d50df30..9939bb6fb9 100644 --- a/app/export.go +++ b/app/export.go @@ -52,34 +52,42 @@ var exportablePreferences = map[ComparablePreference]string{{ } func (a *App) BulkExport(writer io.Writer, file string, pathToEmojiDir string, dirNameToExportEmoji string) *model.AppError { + mlog.Info("Bulk export: exporting version") if err := a.ExportVersion(writer); err != nil { return err } + mlog.Info("Bulk export: exporting teams") if err := a.ExportAllTeams(writer); err != nil { return err } + mlog.Info("Bulk export: exporting channels") if err := a.ExportAllChannels(writer); err != nil { return err } + mlog.Info("Bulk export: exporting users") if err := a.ExportAllUsers(writer); err != nil { return err } + mlog.Info("Bulk export: exporting posts") if err := a.ExportAllPosts(writer); err != nil { return err } + mlog.Info("Bulk export: exporting emoji") if err := a.ExportCustomEmoji(writer, file, pathToEmojiDir, dirNameToExportEmoji); err != nil { return err } + mlog.Info("Bulk export: exporting direct channels") if err := a.ExportAllDirectChannels(writer); err != nil { return err } + mlog.Info("Bulk export: exporting direct posts") if err := a.ExportAllDirectPosts(writer); err != nil { return err } @@ -327,15 +335,15 @@ func (a *App) buildUserNotifyProps(notifyProps model.StringMap) *UserNotifyProps func (a *App) ExportAllPosts(writer io.Writer) *model.AppError { afterId := strings.Repeat("0", 26) + for { posts, err := a.Srv.Store.Post().GetParentsForExportAfter(1000, afterId) - if err != nil { return err } if len(posts) == 0 { - break + return nil } for _, post := range posts { @@ -348,47 +356,41 @@ func (a *App) ExportAllPosts(writer io.Writer) *model.AppError { postLine := ImportLineForPost(post) - // Do the Replies. - replies, err := a.buildPostReplies(post.Id) + postLine.Post.Replies, err = a.buildPostReplies(post.Id) if err != nil { return err } - reactions, err := a.BuildPostReactions(post.Id) - if err != nil { - return err + postLine.Post.Reactions = &[]ReactionImportData{} + if post.HasReactions { + postLine.Post.Reactions, err = a.BuildPostReactions(post.Id) + if err != nil { + return err + } } - postLine.Post.Replies = replies - - postLine.Post.Reactions = reactions - if err := a.ExportWriteLine(writer, postLine); err != nil { return err } } } - - return nil } func (a *App) buildPostReplies(postId string) (*[]ReplyImportData, *model.AppError) { var replies []ReplyImportData replyPosts, err := a.Srv.Store.Post().GetRepliesForExport(postId) - if err != nil { return nil, err } for _, reply := range replyPosts { replyImportObject := ImportReplyFromPost(reply) - if reply.HasReactions == true { - reactionsOfReply, err := a.BuildPostReactions(reply.Id) + if reply.HasReactions { + replyImportObject.Reactions, err = a.BuildPostReactions(reply.Id) if err != nil { return nil, err } - replyImportObject.Reactions = reactionsOfReply } replies = append(replies, *replyImportObject) } diff --git a/cmd/mattermost/commands/export.go b/cmd/mattermost/commands/export.go index 6f8a5ddd67..75202c937a 100644 --- a/cmd/mattermost/commands/export.go +++ b/cmd/mattermost/commands/export.go @@ -4,14 +4,12 @@ package commands import ( - "errors" - "os" - "context" - + "os" "time" "github.com/mattermost/mattermost-server/model" + "github.com/pkg/errors" "github.com/spf13/cobra" ) @@ -57,7 +55,7 @@ var BulkExportCmd = &cobra.Command{ Use: "bulk [file]", Short: "Export bulk data.", Long: "Export data to a file compatible with the Mattermost Bulk Import format.", - Example: " export bulk bulk_data.json", + Example: "export bulk bulk_data.json", RunE: bulkExportCmdF, Args: cobra.ExactArgs(1), } @@ -72,7 +70,7 @@ func init() { ActianceExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.") GlobalRelayZipExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.") - BulkExportCmd.Flags().Bool("all-teams", false, "Export all teams from the server.") + BulkExportCmd.Flags().Bool("all-teams", true, "Export all teams from the server.") ExportCmd.AddCommand(ScheduleExportCmd) ExportCmd.AddCommand(CsvExportCmd) @@ -177,9 +175,8 @@ func bulkExportCmdF(command *cobra.Command, args []string) error { allTeams, err := command.Flags().GetBool("all-teams") if err != nil { - return errors.New("Apply flag error") + return errors.Wrap(err, "all-teams flag error") } - if !allTeams { return errors.New("Nothing to export. Please specify the --all-teams flag to export all teams.") } diff --git a/store/sqlstore/post_store.go b/store/sqlstore/post_store.go index 219ad62e6b..bac4488e5b 100644 --- a/store/sqlstore/post_store.go +++ b/store/sqlstore/post_store.go @@ -1312,42 +1312,68 @@ func (s *SqlPostStore) GetMaxPostSize() int { } func (s *SqlPostStore) GetParentsForExportAfter(limit int, afterId string) ([]*model.PostForExport, *model.AppError) { - var posts []*model.PostForExport - _, err := s.GetSearchReplica().Select(&posts, ` - SELECT - p1.*, - Users.Username as Username, - Teams.Name as TeamName, - Channels.Name as ChannelName - FROM - Posts p1 - INNER JOIN - Channels ON p1.ChannelId = Channels.Id - INNER JOIN - Teams ON Channels.TeamId = Teams.Id - INNER JOIN - Users ON p1.UserId = Users.Id - WHERE - p1.Id > :AfterId - AND p1.ParentId = '' - AND p1.DeleteAt = 0 - AND Channels.DeleteAt = 0 - AND Teams.DeleteAt = 0 - ORDER BY - p1.Id - LIMIT - :Limit`, - map[string]interface{}{"Limit": limit, "AfterId": afterId}) + for { + var rootIds []string + _, err := s.GetReplica().Select(&rootIds, + `SELECT + Id + FROM + Posts + WHERE + Id > :AfterId + AND RootId = '' + AND DeleteAt = 0 + ORDER BY Id + LIMIT :Limit`, + map[string]interface{}{"Limit": limit, "AfterId": afterId}) + if err != nil { + return nil, model.NewAppError("SqlPostStore.GetAllAfterForExport", "store.sql_post.get_posts.app_error", + nil, err.Error(), http.StatusInternalServerError) + } - if err != nil { - return nil, model.NewAppError("SqlPostStore.GetAllAfterForExport", "store.sql_post.get_posts.app_error", - nil, err.Error(), http.StatusInternalServerError) + var postsForExport []*model.PostForExport + if len(rootIds) == 0 { + return postsForExport, nil + } + + keys, params := MapStringsToQueryParams(rootIds, "PostId") + _, err = s.GetSearchReplica().Select(&postsForExport, ` + SELECT + p1.*, + Users.Username as Username, + Teams.Name as TeamName, + Channels.Name as ChannelName + FROM + (Select * FROM Posts WHERE Id IN `+keys+`) p1 + INNER JOIN + Channels ON p1.ChannelId = Channels.Id + INNER JOIN + Teams ON Channels.TeamId = Teams.Id + INNER JOIN + Users ON p1.UserId = Users.Id + WHERE + Channels.DeleteAt = 0 + AND Teams.DeleteAt = 0 + ORDER BY + p1.Id`, + params) + if err != nil { + return nil, model.NewAppError("SqlPostStore.GetAllAfterForExport", "store.sql_post.get_posts.app_error", + nil, err.Error(), http.StatusInternalServerError) + } + + if len(postsForExport) == 0 { + // All of the posts were in channels or teams that were deleted. + // Update the afterId and try again. + afterId = rootIds[len(rootIds)-1] + continue + } + + return postsForExport, nil } - return posts, nil } -func (s *SqlPostStore) GetRepliesForExport(parentId string) ([]*model.ReplyForExport, *model.AppError) { - +func (s *SqlPostStore) GetRepliesForExport(rootId string) ([]*model.ReplyForExport, *model.AppError) { var posts []*model.ReplyForExport _, err := s.GetSearchReplica().Select(&posts, ` SELECT @@ -1358,11 +1384,11 @@ func (s *SqlPostStore) GetRepliesForExport(parentId string) ([]*model.ReplyForEx INNER JOIN Users ON Posts.UserId = Users.Id WHERE - Posts.ParentId = :ParentId + Posts.RootId = :RootId AND Posts.DeleteAt = 0 ORDER BY Posts.Id`, - map[string]interface{}{"ParentId": parentId}) + map[string]interface{}{"RootId": rootId}) if err != nil { return nil, model.NewAppError("SqlPostStore.GetAllAfterForExport", "store.sql_post.get_posts.app_error", nil, err.Error(), http.StatusInternalServerError)