MM-11781: Basic Data Export Command Line. (#9296)

* MM-11781: Basic Data Export Command Line.

* ChannelStore new unit tests.

* TeamStore new unit tests.

* Unit test for new UserStore function.

* Unit tests for post store new methods.

* Review fixes.

* Fix duplicate command name.

app/export.go

@@ -0,0 +1,279 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package app
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/model"
+)
+
+func (a *App) BulkExport(writer io.Writer) *model.AppError {
+	if err := a.ExportVersion(writer); err != nil {
+		return err
+	}
+
+	if err := a.ExportAllTeams(writer); err != nil {
+		return err
+	}
+
+	if err := a.ExportAllChannels(writer); err != nil {
+		return err
+	}
+
+	if err := a.ExportAllUsers(writer); err != nil {
+		return err
+	}
+
+	if err := a.ExportAllPosts(writer); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (a *App) ExportWriteLine(writer io.Writer, line *LineImportData) *model.AppError {
+	b, err := json.Marshal(line)
+	if err != nil {
+		return model.NewAppError("BulkExport", "app.export.export_write_line.json_marshall.error", nil, "err="+err.Error(), http.StatusBadRequest)
+	}
+
+	if _, err := writer.Write(append(b, '\n')); err != nil {
+		return model.NewAppError("BulkExport", "app.export.export_write_line.io_writer.error", nil, "err="+err.Error(), http.StatusBadRequest)
+	}
+
+	return nil
+}
+
+func (a *App) ExportVersion(writer io.Writer) *model.AppError {
+	version := 1
+	versionLine := &LineImportData{
+		Type:    "version",
+		Version: &version,
+	}
+
+	return a.ExportWriteLine(writer, versionLine)
+}
+
+func (a *App) ExportAllTeams(writer io.Writer) *model.AppError {
+	afterId := strings.Repeat("0", 26)
+	for {
+		result := <-a.Srv.Store.Team().GetAllForExportAfter(1000, afterId)
+
+		if result.Err != nil {
+			return result.Err
+		}
+
+		teams := result.Data.([]*model.TeamForExport)
+
+		if len(teams) == 0 {
+			break
+		}
+
+		for _, team := range teams {
+			afterId = team.Id
+
+			// Skip deleted.
+			if team.DeleteAt != 0 {
+				continue
+			}
+
+			teamLine := ImportLineFromTeam(team)
+			if err := a.ExportWriteLine(writer, teamLine); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (a *App) ExportAllChannels(writer io.Writer) *model.AppError {
+	afterId := strings.Repeat("0", 26)
+	for {
+		result := <-a.Srv.Store.Channel().GetAllChannelsForExportAfter(1000, afterId)
+
+		if result.Err != nil {
+			return result.Err
+		}
+
+		channels := result.Data.([]*model.ChannelForExport)
+
+		if len(channels) == 0 {
+			break
+		}
+
+		for _, channel := range channels {
+			afterId = channel.Id
+
+			// Skip deleted.
+			if channel.DeleteAt != 0 {
+				continue
+			}
+
+			channelLine := ImportLineFromChannel(channel)
+			if err := a.ExportWriteLine(writer, channelLine); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (a *App) ExportAllUsers(writer io.Writer) *model.AppError {
+	afterId := strings.Repeat("0", 26)
+	for {
+		result := <-a.Srv.Store.User().GetAllAfter(1000, afterId)
+
+		if result.Err != nil {
+			return result.Err
+		}
+
+		users := result.Data.([]*model.User)
+
+		if len(users) == 0 {
+			break
+		}
+
+		for _, user := range users {
+			afterId = user.Id
+
+			// Skip deleted.
+			if user.DeleteAt != 0 {
+				continue
+			}
+
+			userLine := ImportLineFromUser(user)
+
+			// Do the Team Memberships.
+			members, err := a.buildUserTeamAndChannelMemberships(user.Id)
+			if err != nil {
+				return err
+			}
+
+			userLine.User.Teams = members
+
+			if err := a.ExportWriteLine(writer, userLine); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (a *App) buildUserTeamAndChannelMemberships(userId string) (*[]UserTeamImportData, *model.AppError) {
+	var memberships []UserTeamImportData
+
+	result := <-a.Srv.Store.Team().GetTeamMembersForExport(userId)
+
+	if result.Err != nil {
+		return nil, result.Err
+	}
+
+	members := result.Data.([]*model.TeamMemberForExport)
+
+	for _, member := range members {
+		// Skip deleted.
+		if member.DeleteAt != 0 {
+			continue
+		}
+
+		memberData := ImportUserTeamDataFromTeamMember(member)
+
+		// Do the Channel Memberships.
+		channelMembers, err := a.buildUserChannelMemberships(userId, member.TeamId)
+		if err != nil {
+			return nil, err
+		}
+
+		memberData.Channels = channelMembers
+
+		memberships = append(memberships, *memberData)
+	}
+
+	return &memberships, nil
+}
+
+func (a *App) buildUserChannelMemberships(userId string, teamId string) (*[]UserChannelImportData, *model.AppError) {
+	var memberships []UserChannelImportData
+
+	result := <-a.Srv.Store.Channel().GetChannelMembersForExport(userId, teamId)
+
+	if result.Err != nil {
+		return nil, result.Err
+	}
+
+	members := result.Data.([]*model.ChannelMemberForExport)
+
+	for _, member := range members {
+		memberships = append(memberships, *ImportUserChannelDataFromChannelMember(member))
+	}
+
+	return &memberships, nil
+}
+
+func (a *App) ExportAllPosts(writer io.Writer) *model.AppError {
+	afterId := strings.Repeat("0", 26)
+	for {
+		result := <-a.Srv.Store.Post().GetParentsForExportAfter(1000, afterId)
+
+		if result.Err != nil {
+			return result.Err
+		}
+
+		posts := result.Data.([]*model.PostForExport)
+
+		if len(posts) == 0 {
+			break
+		}
+
+		for _, post := range posts {
+			afterId = post.Id
+
+			// Skip deleted.
+			if post.DeleteAt != 0 {
+				continue
+			}
+
+			postLine := ImportLineForPost(post)
+
+			// Do the Replies.
+			replies, err := a.buildPostReplies(post.Id)
+			if err != nil {
+				return err
+			}
+
+			postLine.Post.Replies = replies
+
+			if err := a.ExportWriteLine(writer, postLine); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (a *App) buildPostReplies(postId string) (*[]ReplyImportData, *model.AppError) {
+	var replies []ReplyImportData
+
+	result := <-a.Srv.Store.Post().GetRepliesForExport(postId)
+
+	if result.Err != nil {
+		return nil, result.Err
+	}
+
+	replyPosts := result.Data.([]*model.ReplyForExport)
+
+	for _, reply := range replyPosts {
+		replies = append(replies, *ImportReplyFromPost(reply))
+	}
+
+	return &replies, nil
+}

app/export_converters.go

@@ -0,0 +1,113 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package app
+
+import (
+	"github.com/mattermost/mattermost-server/model"
+	"strings"
+)
+
+func ImportLineFromTeam(team *model.TeamForExport) *LineImportData {
+	return &LineImportData{
+		Type: "team",
+		Team: &TeamImportData{
+			Name:            &team.Name,
+			DisplayName:     &team.DisplayName,
+			Type:            &team.Type,
+			Description:     &team.Description,
+			AllowOpenInvite: &team.AllowOpenInvite,
+			Scheme:          team.SchemeName,
+		},
+	}
+}
+
+func ImportLineFromChannel(channel *model.ChannelForExport) *LineImportData {
+	return &LineImportData{
+		Type: "channel",
+		Channel: &ChannelImportData{
+			Team:        &channel.TeamName,
+			Name:        &channel.Name,
+			DisplayName: &channel.DisplayName,
+			Type:        &channel.Type,
+			Header:      &channel.Header,
+			Purpose:     &channel.Purpose,
+			Scheme:      channel.SchemeName,
+		},
+	}
+}
+
+func ImportLineFromUser(user *model.User) *LineImportData {
+	// Bulk Importer doesn't accept "empty string" for AuthService.
+	var authService *string
+	if user.AuthService != "" {
+		authService = &user.AuthService
+	}
+
+	return &LineImportData{
+		Type: "user",
+		User: &UserImportData{
+			Username:    &user.Username,
+			Email:       &user.Email,
+			AuthService: authService,
+			AuthData:    user.AuthData,
+			Nickname:    &user.Nickname,
+			FirstName:   &user.FirstName,
+			LastName:    &user.LastName,
+			Position:    &user.Position,
+			Roles:       &user.Roles,
+			Locale:      &user.Locale,
+		},
+	}
+}
+
+func ImportUserTeamDataFromTeamMember(member *model.TeamMemberForExport) *UserTeamImportData {
+	rolesList := strings.Fields(member.Roles)
+	if member.SchemeAdmin {
+		rolesList = append(rolesList, model.TEAM_ADMIN_ROLE_ID)
+	}
+	if member.SchemeUser {
+		rolesList = append(rolesList, model.TEAM_USER_ROLE_ID)
+	}
+	roles := strings.Join(rolesList, " ")
+	return &UserTeamImportData{
+		Name:  &member.TeamName,
+		Roles: &roles,
+	}
+}
+
+func ImportUserChannelDataFromChannelMember(member *model.ChannelMemberForExport) *UserChannelImportData {
+	rolesList := strings.Fields(member.Roles)
+	if member.SchemeAdmin {
+		rolesList = append(rolesList, model.CHANNEL_ADMIN_ROLE_ID)
+	}
+	if member.SchemeUser {
+		rolesList = append(rolesList, model.CHANNEL_USER_ROLE_ID)
+	}
+	roles := strings.Join(rolesList, " ")
+	return &UserChannelImportData{
+		Name:  &member.ChannelName,
+		Roles: &roles,
+	}
+}
+
+func ImportLineForPost(post *model.PostForExport) *LineImportData {
+	return &LineImportData{
+		Type: "post",
+		Post: &PostImportData{
+			Team:     &post.TeamName,
+			Channel:  &post.ChannelName,
+			User:     &post.Username,
+			Message:  &post.Message,
+			CreateAt: &post.CreateAt,
+		},
+	}
+}
+
+func ImportReplyFromPost(post *model.ReplyForExport) *ReplyImportData {
+	return &ReplyImportData{
+		User:     &post.Username,
+		Message:  &post.Message,
+		CreateAt: &post.CreateAt,
+	}
+}

app/import_types.go

@@ -9,24 +9,24 @@ import "github.com/mattermost/mattermost-server/model"
 
 type LineImportData struct {
 	Type          string                   `json:"type"`
-	Scheme        *SchemeImportData        `json:"scheme"`
-	Team          *TeamImportData          `json:"team"`
-	Channel       *ChannelImportData       `json:"channel"`
-	User          *UserImportData          `json:"user"`
-	Post          *PostImportData          `json:"post"`
-	DirectChannel *DirectChannelImportData `json:"direct_channel"`
-	DirectPost    *DirectPostImportData    `json:"direct_post"`
-	Emoji         *EmojiImportData         `json:"emoji"`
-	Version       *int                     `json:"version"`
+	Scheme        *SchemeImportData        `json:"scheme,omitempty"`
+	Team          *TeamImportData          `json:"team,omitempty"`
+	Channel       *ChannelImportData       `json:"channel,omitempty"`
+	User          *UserImportData          `json:"user,omitempty"`
+	Post          *PostImportData          `json:"post,omitempty"`
+	DirectChannel *DirectChannelImportData `json:"direct_channel,omitempty"`
+	DirectPost    *DirectPostImportData    `json:"direct_post,omitempty"`
+	Emoji         *EmojiImportData         `json:"emoji,omitempty"`
+	Version       *int                     `json:"version,omitempty"`
 }
 
 type TeamImportData struct {
 	Name            *string `json:"name"`
 	DisplayName     *string `json:"display_name"`
 	Type            *string `json:"type"`
-	Description     *string `json:"description"`
-	AllowOpenInvite *bool   `json:"allow_open_invite"`
-	Scheme          *string `json:"scheme"`
+	Description     *string `json:"description,omitempty"`
+	AllowOpenInvite *bool   `json:"allow_open_invite,omitempty"`
+	Scheme          *string `json:"scheme,omitempty"`
 }
 
 type ChannelImportData struct {
@@ -34,38 +34,38 @@ type ChannelImportData struct {
 	Name        *string `json:"name"`
 	DisplayName *string `json:"display_name"`
 	Type        *string `json:"type"`
-	Header      *string `json:"header"`
-	Purpose     *string `json:"purpose"`
-	Scheme      *string `json:"scheme"`
+	Header      *string `json:"header,omitempty"`
+	Purpose     *string `json:"purpose,omitempty"`
+	Scheme      *string `json:"scheme,omitempty"`
 }
 
 type UserImportData struct {
-	ProfileImage       *string `json:"profile_image"`
+	ProfileImage       *string `json:"profile_image,omitempty"`
 	Username           *string `json:"username"`
 	Email              *string `json:"email"`
 	AuthService        *string `json:"auth_service"`
-	AuthData           *string `json:"auth_data"`
-	Password           *string `json:"password"`
+	AuthData           *string `json:"auth_data,omitempty"`
+	Password           *string `json:"password,omitempty"`
 	Nickname           *string `json:"nickname"`
 	FirstName          *string `json:"first_name"`
 	LastName           *string `json:"last_name"`
 	Position           *string `json:"position"`
 	Roles              *string `json:"roles"`
 	Locale             *string `json:"locale"`
-	UseMarkdownPreview *string `json:"feature_enabled_markdown_preview"`
-	UseFormatting      *string `json:"formatting"`
-	ShowUnreadSection  *string `json:"show_unread_section"`
+	UseMarkdownPreview *string `json:"feature_enabled_markdown_preview,omitempty"`
+	UseFormatting      *string `json:"formatting,omitempty"`
+	ShowUnreadSection  *string `json:"show_unread_section,omitempty"`
 
-	Teams *[]UserTeamImportData `json:"teams"`
+	Teams *[]UserTeamImportData `json:"teams,omitempty"`
 
-	Theme              *string `json:"theme"`
-	UseMilitaryTime    *string `json:"military_time"`
-	CollapsePreviews   *string `json:"link_previews"`
-	MessageDisplay     *string `json:"message_display"`
-	ChannelDisplayMode *string `json:"channel_display_mode"`
-	TutorialStep       *string `json:"tutorial_step"`
+	Theme              *string `json:"theme,omitempty"`
+	UseMilitaryTime    *string `json:"military_time,omitempty"`
+	CollapsePreviews   *string `json:"link_previews,omitempty"`
+	MessageDisplay     *string `json:"message_display,omitempty"`
+	ChannelDisplayMode *string `json:"channel_display_mode,omitempty"`
+	TutorialStep       *string `json:"tutorial_step,omitempty"`
 
-	NotifyProps *UserNotifyPropsImportData `json:"notify_props"`
+	NotifyProps *UserNotifyPropsImportData `json:"notify_props,omitempty"`
 }
 
 type UserNotifyPropsImportData struct {
@@ -85,15 +85,15 @@ type UserNotifyPropsImportData struct {
 type UserTeamImportData struct {
 	Name     *string                  `json:"name"`
 	Roles    *string                  `json:"roles"`
-	Theme    *string                  `json:"theme"`
-	Channels *[]UserChannelImportData `json:"channels"`
+	Theme    *string                  `json:"theme,omitempty"`
+	Channels *[]UserChannelImportData `json:"channels,omitempty"`
 }
 
 type UserChannelImportData struct {
 	Name        *string                           `json:"name"`
 	Roles       *string                           `json:"roles"`
-	NotifyProps *UserChannelNotifyPropsImportData `json:"notify_props"`
-	Favorite    *bool                             `json:"favorite"`
+	NotifyProps *UserChannelNotifyPropsImportData `json:"notify_props,omitempty"`
+	Favorite    *bool                             `json:"favorite,omitempty"`
 }
 
 type UserChannelNotifyPropsImportData struct {
@@ -119,9 +119,9 @@ type ReplyImportData struct {
 	Message  *string `json:"message"`
 	CreateAt *int64  `json:"create_at"`
 
-	FlaggedBy   *[]string               `json:"flagged_by"`
-	Reactions   *[]ReactionImportData   `json:"reactions"`
-	Attachments *[]AttachmentImportData `json:"attachments"`
+	FlaggedBy   *[]string               `json:"flagged_by,omitempty"`
+	Reactions   *[]ReactionImportData   `json:"reactions,omitempty"`
+	Attachments *[]AttachmentImportData `json:"attachments,omitempty"`
 }
 
 type PostImportData struct {
@@ -132,10 +132,10 @@ type PostImportData struct {
 	Message  *string `json:"message"`
 	CreateAt *int64  `json:"create_at"`
 
-	FlaggedBy   *[]string               `json:"flagged_by"`
-	Reactions   *[]ReactionImportData   `json:"reactions"`
-	Replies     *[]ReplyImportData      `json:"replies"`
-	Attachments *[]AttachmentImportData `json:"attachments"`
+	FlaggedBy   *[]string               `json:"flagged_by,omitempty"`
+	Reactions   *[]ReactionImportData   `json:"reactions,omitempty"`
+	Replies     *[]ReplyImportData      `json:"replies,omitempty"`
+	Attachments *[]AttachmentImportData `json:"attachments,omitempty"`
 }
 
 type DirectChannelImportData struct {