MM-21552: Adding SaveMultiple to posts (#13766)

* Adding SaveMultiple to posts

* Improving tests

* fixing i18n

* Fixing tests

* Improving testing on top of Save and SaveMultiple

* Fixing shadow variables

* Addressing some PR comments

* More clear update post test

* Addressing some PR comments

* Addressing some PR comments and simplifying the code

* Improting replies in bulk too

* Fixing reply count and processing last imported replies

* Adding OverwriteMultiple to posts aggregating everything in the same transaction

* Adding 2 pending tests to implement

* Adding tests for overwrite multiple posts

* Adding tests for TeamStore.GetByNames method

* Fixing shadow variables

* Addressing PR comments

* Extracting i18n strings

* Fixing tests

* Fixing tests

* Adding more test cases

* Using a variable instead of a fake timestamp

app/import.go

@@ -16,7 +16,10 @@ import (
 	"github.com/mattermost/mattermost-server/v5/model"
 )
 
-const maxScanTokenSize = 16 * 1024 * 1024 // Need to set a higher limit than default because some customers cross the limit. See MM-22314
+const (
+	importMultiplePostsThreshold = 1000
+	maxScanTokenSize             = 16 * 1024 * 1024 // Need to set a higher limit than default because some customers cross the limit. See MM-22314
+)
 
 func stopOnError(err LineImportWorkerError) bool {
 	if err.Error.Id == "api.file.upload_file.large_image.app_error" {
@@ -27,11 +30,41 @@ func stopOnError(err LineImportWorkerError) bool {
 }
 
 func (a *App) bulkImportWorker(dryRun bool, wg *sync.WaitGroup, lines <-chan LineImportWorkerData, errors chan<- LineImportWorkerError) {
+	posts := []*PostImportData{}
+	directPosts := []*DirectPostImportData{}
 	for line := range lines {
-		if err := a.importLine(line.LineImportData, dryRun); err != nil {
-			errors <- LineImportWorkerError{err, line.LineNumber}
+		switch {
+		case line.LineImportData.Type == "post":
+			posts = append(posts, line.Post)
+			if line.Post == nil {
+				errors <- LineImportWorkerError{model.NewAppError("BulkImport", "app.import.import_line.null_post.error", nil, "", http.StatusBadRequest), line.LineNumber}
+			}
+			if len(posts) >= importMultiplePostsThreshold {
+				a.importMultiplePosts(posts, dryRun)
+				posts = []*PostImportData{}
+			}
+		case line.LineImportData.Type == "direct_post":
+			directPosts = append(directPosts, line.DirectPost)
+			if line.DirectPost == nil {
+				errors <- LineImportWorkerError{model.NewAppError("BulkImport", "app.import.import_line.null_direct_post.error", nil, "", http.StatusBadRequest), line.LineNumber}
+			}
+			if len(directPosts) >= importMultiplePostsThreshold {
+				a.importMultipleDirectPosts(directPosts, dryRun)
+				directPosts = []*DirectPostImportData{}
+			}
+		default:
+			if err := a.importLine(line.LineImportData, dryRun); err != nil {
+				errors <- LineImportWorkerError{err, line.LineNumber}
+			}
 		}
 	}
+
+	if len(posts) > 0 {
+		a.importMultiplePosts(posts, dryRun)
+	}
+	if len(directPosts) > 0 {
+		a.importMultipleDirectPosts(directPosts, dryRun)
+	}
 	wg.Done()
 }
 
@@ -159,21 +192,11 @@ func (a *App) importLine(line LineImportData, dryRun bool) *model.AppError {
 			return model.NewAppError("BulkImport", "app.import.import_line.null_user.error", nil, "", http.StatusBadRequest)
 		}
 		return a.importUser(line.User, dryRun)
-	case line.Type == "post":
-		if line.Post == nil {
-			return model.NewAppError("BulkImport", "app.import.import_line.null_post.error", nil, "", http.StatusBadRequest)
-		}
-		return a.importPost(line.Post, dryRun)
 	case line.Type == "direct_channel":
 		if line.DirectChannel == nil {
 			return model.NewAppError("BulkImport", "app.import.import_line.null_direct_channel.error", nil, "", http.StatusBadRequest)
 		}
 		return a.importDirectChannel(line.DirectChannel, dryRun)
-	case line.Type == "direct_post":
-		if line.DirectPost == nil {
-			return model.NewAppError("BulkImport", "app.import.import_line.null_direct_post.error", nil, "", http.StatusBadRequest)
-		}
-		return a.importDirectPost(line.DirectPost, dryRun)
 	case line.Type == "emoji":
 		if line.Emoji == nil {
 			return model.NewAppError("BulkImport", "app.import.import_line.null_emoji.error", nil, "", http.StatusBadRequest)

app/import_functions.go

@@ -851,67 +851,87 @@ func (a *App) importReaction(data *ReactionImportData, post *model.Post, dryRun
 	return nil
 }
 
-func (a *App) importReply(data *ReplyImportData, post *model.Post, teamId string, dryRun bool) *model.AppError {
+func (a *App) importReplies(data []ReplyImportData, post *model.Post, teamId string, dryRun bool) *model.AppError {
 	var err *model.AppError
-	if err = validateReplyImportData(data, post.CreateAt, a.MaxPostSize()); err != nil {
-		return err
-	}
-
-	var user *model.User
-	user, err = a.Srv().Store.User().GetByUsername(*data.User)
-	if err != nil {
-		return model.NewAppError("BulkImport", "app.import.import_post.user_not_found.error", map[string]interface{}{"Username": data.User}, err.Error(), http.StatusBadRequest)
-	}
-
-	// Check if this post already exists.
-	replies, err := a.Srv().Store.Post().GetPostsCreatedAt(post.ChannelId, *data.CreateAt)
-	if err != nil {
-		return err
-	}
-
-	var reply *model.Post
-	for _, r := range replies {
-		if r.Message == *data.Message && r.RootId == post.Id {
-			reply = r
-			break
-		}
-	}
-
-	if reply == nil {
-		reply = &model.Post{}
-	}
-	reply.UserId = user.Id
-	reply.ChannelId = post.ChannelId
-	reply.ParentId = post.Id
-	reply.RootId = post.Id
-	reply.Message = *data.Message
-	reply.CreateAt = *data.CreateAt
-
-	fileIds, err := a.uploadAttachments(data.Attachments, reply, teamId, dryRun)
-	if err != nil {
-		return err
-	}
-	for _, fileID := range reply.FileIds {
-		if _, ok := fileIds[fileID]; !ok {
-			a.Srv().Store.FileInfo().PermanentDelete(fileID)
-		}
-	}
-	reply.FileIds = make([]string, 0)
-	for fileID := range fileIds {
-		reply.FileIds = append(reply.FileIds, fileID)
-	}
-
-	if reply.Id == "" {
-		if _, err := a.Srv().Store.Post().Save(reply); err != nil {
+	usernames := []string{}
+	for _, replyData := range data {
+		if err = validateReplyImportData(&replyData, post.CreateAt, a.MaxPostSize()); err != nil {
 			return err
 		}
-	} else {
-		if _, err := a.Srv().Store.Post().Overwrite(reply); err != nil {
+		usernames = append(usernames, *replyData.User)
+	}
+
+	users, err := a.getUsersByUsernames(usernames)
+	if err != nil {
+		return err
+	}
+
+	postsWithData := []postAndData{}
+	postsForCreateList := []*model.Post{}
+	postsForOverwriteList := []*model.Post{}
+
+	for _, replyData := range data {
+		user := users[*replyData.User]
+
+		// Check if this post already exists.
+		replies, err := a.Srv().Store.Post().GetPostsCreatedAt(post.ChannelId, *replyData.CreateAt)
+		if err != nil {
+			return err
+		}
+
+		var reply *model.Post
+		for _, r := range replies {
+			if r.Message == *replyData.Message && r.RootId == post.Id {
+				reply = r
+				break
+			}
+		}
+
+		if reply == nil {
+			reply = &model.Post{}
+		}
+		reply.UserId = user.Id
+		reply.ChannelId = post.ChannelId
+		reply.ParentId = post.Id
+		reply.RootId = post.Id
+		reply.Message = *replyData.Message
+		reply.CreateAt = *replyData.CreateAt
+
+		fileIds, err := a.uploadAttachments(replyData.Attachments, reply, teamId, dryRun)
+		if err != nil {
+			return err
+		}
+		for _, fileID := range reply.FileIds {
+			if _, ok := fileIds[fileID]; !ok {
+				a.Srv().Store.FileInfo().PermanentDelete(fileID)
+			}
+		}
+		reply.FileIds = make([]string, 0)
+		for fileID := range fileIds {
+			reply.FileIds = append(reply.FileIds, fileID)
+		}
+
+		if len(reply.Id) == 0 {
+			postsForCreateList = append(postsForCreateList, reply)
+		} else {
+			postsForOverwriteList = append(postsForOverwriteList, reply)
+		}
+		postsWithData = append(postsWithData, postAndData{post: reply, replyData: &replyData})
+	}
+
+	if len(postsForCreateList) > 0 {
+		if _, err := a.Srv().Store.Post().SaveMultiple(postsForCreateList); err != nil {
 			return err
 		}
 	}
 
-	a.updateFileInfoWithPostId(reply)
+	if _, err := a.Srv().Store.Post().OverwriteMultiple(postsForOverwriteList); err != nil {
+		return err
+	}
+
+	for _, postWithData := range postsWithData {
+		a.updateFileInfoWithPostId(postWithData.post)
+	}
 
 	return nil
 }
@@ -961,9 +981,70 @@ func (a *App) importAttachment(data *AttachmentImportData, post *model.Post, tea
 	return fileInfo, nil
 }
 
-func (a *App) importPost(data *PostImportData, dryRun bool) *model.AppError {
-	if err := validatePostImportData(data, a.MaxPostSize()); err != nil {
-		return err
+type postAndData struct {
+	post           *model.Post
+	postData       *PostImportData
+	directPostData *DirectPostImportData
+	replyData      *ReplyImportData
+	team           *model.Team
+}
+
+func (a *App) getUsersByUsernames(usernames []string) (map[string]*model.User, *model.AppError) {
+	uniqueUsernames := utils.RemoveDuplicatesFromStringArray(usernames)
+	allUsers, err := a.Srv().Store.User().GetProfilesByUsernames(uniqueUsernames, nil)
+	if err != nil {
+		return nil, model.NewAppError("BulkImport", "app.import.get_users_by_username.some_users_not_found.error", nil, err.Error(), http.StatusBadRequest)
+	}
+
+	if len(allUsers) != len(uniqueUsernames) {
+		return nil, model.NewAppError("BulkImport", "app.import.get_users_by_username.some_users_not_found.error", nil, "", http.StatusBadRequest)
+	}
+
+	users := make(map[string]*model.User)
+	for _, user := range allUsers {
+		users[user.Username] = user
+	}
+	return users, nil
+}
+
+func (a *App) getTeamsByNames(names []string) (map[string]*model.Team, *model.AppError) {
+	allTeams, err := a.Srv().Store.Team().GetByNames(names)
+	if err != nil {
+		return nil, model.NewAppError("BulkImport", "app.import.get_teams_by_names.some_teams_not_found.error", nil, err.Error(), http.StatusBadRequest)
+	}
+
+	teams := make(map[string]*model.Team)
+	for _, team := range allTeams {
+		teams[team.Name] = team
+	}
+	return teams, nil
+}
+
+func (a *App) getChannelsForPosts(teams map[string]*model.Team, data []*PostImportData) (map[string]*model.Channel, *model.AppError) {
+	channels := make(map[string]*model.Channel)
+	for _, postData := range data {
+		team := teams[*postData.Team]
+		if channel, ok := channels[*postData.Channel]; !ok || channel == nil {
+			var err *model.AppError
+			channel, err = a.Srv().Store.Channel().GetByName(team.Id, *postData.Channel, true)
+			if err != nil {
+				return nil, model.NewAppError("BulkImport", "app.import.import_post.channel_not_found.error", map[string]interface{}{"ChannelName": *postData.Channel}, err.Error(), http.StatusBadRequest)
+			}
+			channels[*postData.Channel] = channel
+		}
+	}
+	return channels, nil
+}
+
+func (a *App) importMultiplePosts(data []*PostImportData, dryRun bool) *model.AppError {
+	if len(data) == 0 {
+		return nil
+	}
+
+	for _, postData := range data {
+		if err := validatePostImportData(postData, a.MaxPostSize()); err != nil {
+			return err
+		}
 	}
 
 	// If this is a Dry Run, do not continue any further.
@@ -971,113 +1052,148 @@ func (a *App) importPost(data *PostImportData, dryRun bool) *model.AppError {
 		return nil
 	}
 
-	team, err := a.Srv().Store.Team().GetByName(*data.Team)
-	if err != nil {
-		return model.NewAppError("BulkImport", "app.import.import_post.team_not_found.error", map[string]interface{}{"TeamName": *data.Team}, err.Error(), http.StatusBadRequest)
+	usernames := []string{}
+	teamNames := []string{}
+	for _, postData := range data {
+		usernames = append(usernames, *postData.User)
+		if postData.FlaggedBy != nil {
+			usernames = append(usernames, *postData.FlaggedBy...)
+		}
+		teamNames = append(teamNames, *postData.Team)
 	}
 
-	channel, err := a.Srv().Store.Channel().GetByName(team.Id, *data.Channel, false)
-	if err != nil {
-		return model.NewAppError("BulkImport", "app.import.import_post.channel_not_found.error", map[string]interface{}{"ChannelName": *data.Channel}, err.Error(), http.StatusBadRequest)
-	}
-
-	var user *model.User
-	user, err = a.Srv().Store.User().GetByUsername(*data.User)
-	if err != nil {
-		return model.NewAppError("BulkImport", "app.import.import_post.user_not_found.error", map[string]interface{}{"Username": *data.User}, err.Error(), http.StatusBadRequest)
-	}
-
-	// Check if this post already exists.
-	posts, err := a.Srv().Store.Post().GetPostsCreatedAt(channel.Id, *data.CreateAt)
+	users, err := a.getUsersByUsernames(usernames)
 	if err != nil {
 		return err
 	}
 
-	var post *model.Post
-	for _, p := range posts {
-		if p.Message == *data.Message {
-			post = p
-			break
-		}
-	}
-
-	if post == nil {
-		post = &model.Post{}
-	}
-
-	post.ChannelId = channel.Id
-	post.Message = *data.Message
-	post.UserId = user.Id
-	post.CreateAt = *data.CreateAt
-
-	post.Hashtags, _ = model.ParseHashtags(post.Message)
-
-	fileIds, err := a.uploadAttachments(data.Attachments, post, team.Id, dryRun)
+	teams, err := a.getTeamsByNames(teamNames)
 	if err != nil {
 		return err
 	}
-	for _, fileID := range post.FileIds {
-		if _, ok := fileIds[fileID]; !ok {
-			a.Srv().Store.FileInfo().PermanentDelete(fileID)
-		}
-	}
-	post.FileIds = make([]string, 0)
-	for fileID := range fileIds {
-		post.FileIds = append(post.FileIds, fileID)
-	}
 
-	if post.Id == "" {
-		if _, err = a.Srv().Store.Post().Save(post); err != nil {
+	channels, err := a.getChannelsForPosts(teams, data)
+	if err != nil {
+		return err
+	}
+	postsWithData := []postAndData{}
+	postsForCreateList := []*model.Post{}
+	postsForOverwriteList := []*model.Post{}
+
+	for _, postData := range data {
+		team := teams[*postData.Team]
+		channel := channels[*postData.Channel]
+		user := users[*postData.User]
+
+		// Check if this post already exists.
+		posts, err := a.Srv().Store.Post().GetPostsCreatedAt(channel.Id, *postData.CreateAt)
+		if err != nil {
 			return err
 		}
-	} else {
-		if _, err = a.Srv().Store.Post().Overwrite(post); err != nil {
+
+		var post *model.Post
+		for _, p := range posts {
+			if p.Message == *postData.Message {
+				post = p
+				break
+			}
+		}
+
+		if post == nil {
+			post = &model.Post{}
+		}
+
+		post.ChannelId = channel.Id
+		post.Message = *postData.Message
+		post.UserId = user.Id
+		post.CreateAt = *postData.CreateAt
+		post.Hashtags, _ = model.ParseHashtags(post.Message)
+
+		fileIds, err := a.uploadAttachments(postData.Attachments, post, team.Id, dryRun)
+		if err != nil {
+			return err
+		}
+		for _, fileID := range post.FileIds {
+			if _, ok := fileIds[fileID]; !ok {
+				a.Srv().Store.FileInfo().PermanentDelete(fileID)
+			}
+		}
+		post.FileIds = make([]string, 0)
+		for fileID := range fileIds {
+			post.FileIds = append(post.FileIds, fileID)
+		}
+
+		if len(post.Id) == 0 {
+			postsForCreateList = append(postsForCreateList, post)
+		} else {
+			postsForOverwriteList = append(postsForOverwriteList, post)
+		}
+		postsWithData = append(postsWithData, postAndData{post: post, postData: postData, team: team})
+	}
+
+	if len(postsForCreateList) > 0 {
+		if _, err := a.Srv().Store.Post().SaveMultiple(postsForCreateList); err != nil {
 			return err
 		}
 	}
 
-	if data.FlaggedBy != nil {
-		var preferences model.Preferences
+	if _, err := a.Srv().Store.Post().OverwriteMultiple(postsForOverwriteList); err != nil {
+		return err
+	}
 
-		for _, username := range *data.FlaggedBy {
-			var user *model.User
-			user, err = a.Srv().Store.User().GetByUsername(username)
-			if err != nil {
-				return model.NewAppError("BulkImport", "app.import.import_post.user_not_found.error", map[string]interface{}{"Username": username}, err.Error(), http.StatusBadRequest)
+	var lastPostWithData *postAndData
+	repliesBulk := []ReplyImportData{}
+	for _, postWithData := range postsWithData {
+		if postWithData.postData.FlaggedBy != nil {
+			var preferences model.Preferences
+
+			for _, username := range *postWithData.postData.FlaggedBy {
+				user := users[username]
+
+				preferences = append(preferences, model.Preference{
+					UserId:   user.Id,
+					Category: model.PREFERENCE_CATEGORY_FLAGGED_POST,
+					Name:     postWithData.post.Id,
+					Value:    "true",
+				})
 			}
 
-			preferences = append(preferences, model.Preference{
-				UserId:   user.Id,
-				Category: model.PREFERENCE_CATEGORY_FLAGGED_POST,
-				Name:     post.Id,
-				Value:    "true",
-			})
+			if len(preferences) > 0 {
+				if err := a.Srv().Store.Preference().Save(&preferences); err != nil {
+					return model.NewAppError("BulkImport", "app.import.import_post.save_preferences.error", nil, err.Error(), http.StatusInternalServerError)
+				}
+			}
 		}
 
-		if len(preferences) > 0 {
-			if err := a.Srv().Store.Preference().Save(&preferences); err != nil {
-				return model.NewAppError("BulkImport", "app.import.import_post.save_preferences.error", nil, err.Error(), http.StatusInternalServerError)
+		if postWithData.postData.Reactions != nil {
+			for _, reaction := range *postWithData.postData.Reactions {
+				if err := a.importReaction(&reaction, postWithData.post, dryRun); err != nil {
+					return err
+				}
 			}
 		}
+
+		if postWithData.postData.Replies != nil {
+			repliesBulk = append(repliesBulk, *postWithData.postData.Replies...)
+			if len(repliesBulk) >= importMultiplePostsThreshold {
+				err := a.importReplies(repliesBulk, postWithData.post, postWithData.team.Id, dryRun)
+				if err != nil {
+					return err
+				}
+				repliesBulk = []ReplyImportData{}
+			}
+		}
+		a.updateFileInfoWithPostId(postWithData.post)
+		lastPostWithData = &postWithData
+	}
+
+	if len(repliesBulk) >= 0 && lastPostWithData != nil {
+		err := a.importReplies(repliesBulk, lastPostWithData.post, lastPostWithData.team.Id, dryRun)
+		if err != nil {
+			return err
+		}
 	}
 
-	if data.Reactions != nil {
-		for _, reaction := range *data.Reactions {
-			if err := a.importReaction(&reaction, post, dryRun); err != nil {
-				return err
-			}
-		}
-	}
-
-	if data.Replies != nil {
-		for _, reply := range *data.Replies {
-			if err := a.importReply(&reply, post, team.Id, dryRun); err != nil {
-				return err
-			}
-		}
-	}
-
-	a.updateFileInfoWithPostId(post)
 	return nil
 }
 
@@ -1116,15 +1232,12 @@ func (a *App) importDirectChannel(data *DirectChannelImportData, dryRun bool) *m
 	}
 
 	var userIds []string
-	userMap := make(map[string]string)
-	for _, username := range *data.Members {
-		var user *model.User
-		user, err = a.Srv().Store.User().GetByUsername(username)
-		if err != nil {
-			return model.NewAppError("BulkImport", "app.import.import_direct_channel.member_not_found.error", nil, err.Error(), http.StatusBadRequest)
-		}
-		userIds = append(userIds, user.Id)
-		userMap[username] = user.Id
+	userMap, err := a.getUsersByUsernames(*data.Members)
+	if err != nil {
+		return err
+	}
+	for _, user := range *data.Members {
+		userIds = append(userIds, userMap[user].Id)
 	}
 
 	var channel *model.Channel
@@ -1157,7 +1270,7 @@ func (a *App) importDirectChannel(data *DirectChannelImportData, dryRun bool) *m
 	if data.FavoritedBy != nil {
 		for _, favoriter := range *data.FavoritedBy {
 			preferences = append(preferences, model.Preference{
-				UserId:   userMap[favoriter],
+				UserId:   userMap[favoriter].Id,
 				Category: model.PREFERENCE_CATEGORY_FAVORITE_CHANNEL,
 				Name:     channel.Id,
 				Value:    "true",
@@ -1180,10 +1293,15 @@ func (a *App) importDirectChannel(data *DirectChannelImportData, dryRun bool) *m
 	return nil
 }
 
-func (a *App) importDirectPost(data *DirectPostImportData, dryRun bool) *model.AppError {
-	var err *model.AppError
-	if err = validateDirectPostImportData(data, a.MaxPostSize()); err != nil {
-		return err
+func (a *App) importMultipleDirectPosts(data []*DirectPostImportData, dryRun bool) *model.AppError {
+	if len(data) == 0 {
+		return nil
+	}
+
+	for _, postData := range data {
+		if err := validateDirectPostImportData(postData, a.MaxPostSize()); err != nil {
+			return err
+		}
 	}
 
 	// If this is a Dry Run, do not continue any further.
@@ -1191,129 +1309,143 @@ func (a *App) importDirectPost(data *DirectPostImportData, dryRun bool) *model.A
 		return nil
 	}
 
-	var userIds []string
-	for _, username := range *data.ChannelMembers {
-		var user *model.User
-		user, err = a.Srv().Store.User().GetByUsername(username)
+	usernames := []string{}
+	for _, postData := range data {
+		usernames = append(usernames, *postData.User)
+		if postData.FlaggedBy != nil {
+			usernames = append(usernames, *postData.FlaggedBy...)
+		}
+		usernames = append(usernames, *postData.ChannelMembers...)
+	}
+
+	users, err := a.getUsersByUsernames(usernames)
+	if err != nil {
+		return err
+	}
+
+	postsWithData := []postAndData{}
+	postsForCreateList := []*model.Post{}
+	postsForOverwriteList := []*model.Post{}
+
+	for _, postData := range data {
+		var userIds []string
+		var err *model.AppError
+		for _, username := range *postData.ChannelMembers {
+			user := users[username]
+			userIds = append(userIds, user.Id)
+		}
+
+		var channel *model.Channel
+		var ch *model.Channel
+		if len(userIds) == 2 {
+			ch, err = a.GetOrCreateDirectChannel(userIds[0], userIds[1])
+			if err != nil && err.Id != store.CHANNEL_EXISTS_ERROR {
+				return model.NewAppError("BulkImport", "app.import.import_direct_post.create_direct_channel.error", nil, err.Error(), http.StatusBadRequest)
+			}
+			channel = ch
+		} else {
+			ch, err = a.createGroupChannel(userIds, userIds[0])
+			if err != nil && err.Id != store.CHANNEL_EXISTS_ERROR {
+				return model.NewAppError("BulkImport", "app.import.import_direct_post.create_group_channel.error", nil, err.Error(), http.StatusBadRequest)
+			}
+			channel = ch
+		}
+
+		user := users[*postData.User]
+
+		// Check if this post already exists.
+		posts, err := a.Srv().Store.Post().GetPostsCreatedAt(channel.Id, *postData.CreateAt)
 		if err != nil {
-			return model.NewAppError("BulkImport", "app.import.import_direct_post.channel_member_not_found.error", nil, err.Error(), http.StatusBadRequest)
-		}
-		userIds = append(userIds, user.Id)
-	}
-
-	var channel *model.Channel
-	var ch *model.Channel
-	if len(userIds) == 2 {
-		ch, err = a.createDirectChannel(userIds[0], userIds[1])
-		if err != nil && err.Id != store.CHANNEL_EXISTS_ERROR {
-			return model.NewAppError("BulkImport", "app.import.import_direct_post.create_direct_channel.error", nil, err.Error(), http.StatusBadRequest)
-		}
-		channel = ch
-	} else {
-		ch, err = a.createGroupChannel(userIds, userIds[0])
-		if err != nil && err.Id != store.CHANNEL_EXISTS_ERROR {
-			return model.NewAppError("BulkImport", "app.import.import_direct_post.create_group_channel.error", nil, err.Error(), http.StatusBadRequest)
-		}
-		channel = ch
-	}
-
-	var user *model.User
-	user, err = a.Srv().Store.User().GetByUsername(*data.User)
-	if err != nil {
-		return model.NewAppError("BulkImport", "app.import.import_direct_post.user_not_found.error", map[string]interface{}{"Username": *data.User}, "", http.StatusBadRequest)
-	}
-
-	// Check if this post already exists.
-	posts, err := a.Srv().Store.Post().GetPostsCreatedAt(channel.Id, *data.CreateAt)
-	if err != nil {
-		return err
-	}
-
-	var post *model.Post
-	for _, p := range posts {
-		if p.Message == *data.Message {
-			post = p
-			break
-		}
-	}
-
-	if post == nil {
-		post = &model.Post{}
-	}
-
-	post.ChannelId = channel.Id
-	post.Message = *data.Message
-	post.UserId = user.Id
-	post.CreateAt = *data.CreateAt
-
-	post.Hashtags, _ = model.ParseHashtags(post.Message)
-
-	fileIds, err := a.uploadAttachments(data.Attachments, post, "noteam", dryRun)
-	if err != nil {
-		return err
-	}
-	for _, fileID := range post.FileIds {
-		if _, ok := fileIds[fileID]; !ok {
-			a.Srv().Store.FileInfo().PermanentDelete(fileID)
-		}
-	}
-	post.FileIds = make([]string, 0)
-	for fileID := range fileIds {
-		post.FileIds = append(post.FileIds, fileID)
-	}
-
-	if post.Id == "" {
-		if _, err = a.Srv().Store.Post().Save(post); err != nil {
 			return err
 		}
-	} else {
-		if _, err = a.Srv().Store.Post().Overwrite(post); err != nil {
+
+		var post *model.Post
+		for _, p := range posts {
+			if p.Message == *postData.Message {
+				post = p
+				break
+			}
+		}
+
+		if post == nil {
+			post = &model.Post{}
+		}
+
+		post.ChannelId = channel.Id
+		post.Message = *postData.Message
+		post.UserId = user.Id
+		post.CreateAt = *postData.CreateAt
+		post.Hashtags, _ = model.ParseHashtags(post.Message)
+
+		fileIds, err := a.uploadAttachments(postData.Attachments, post, "noteam", dryRun)
+		if err != nil {
+			return err
+		}
+		for _, fileID := range post.FileIds {
+			if _, ok := fileIds[fileID]; !ok {
+				a.Srv().Store.FileInfo().PermanentDelete(fileID)
+			}
+		}
+		post.FileIds = make([]string, 0)
+		for fileID := range fileIds {
+			post.FileIds = append(post.FileIds, fileID)
+		}
+
+		if len(post.Id) == 0 {
+			postsForCreateList = append(postsForCreateList, post)
+		} else {
+			postsForOverwriteList = append(postsForOverwriteList, post)
+		}
+		postsWithData = append(postsWithData, postAndData{post: post, directPostData: postData})
+	}
+
+	if len(postsForCreateList) > 0 {
+		if _, err := a.Srv().Store.Post().SaveMultiple(postsForCreateList); err != nil {
 			return err
 		}
 	}
-
-	if data.FlaggedBy != nil {
-		var preferences model.Preferences
-
-		for _, username := range *data.FlaggedBy {
-			var user *model.User
-			user, err = a.Srv().Store.User().GetByUsername(username)
-			if err != nil {
-				return model.NewAppError("BulkImport", "app.import.import_direct_post.user_not_found.error", map[string]interface{}{"Username": username}, "", http.StatusBadRequest)
-			}
-
-			preferences = append(preferences, model.Preference{
-				UserId:   user.Id,
-				Category: model.PREFERENCE_CATEGORY_FLAGGED_POST,
-				Name:     post.Id,
-				Value:    "true",
-			})
-		}
-
-		if len(preferences) > 0 {
-			if err := a.Srv().Store.Preference().Save(&preferences); err != nil {
-				return model.NewAppError("BulkImport", "app.import.import_direct_post.save_preferences.error", nil, err.Error(), http.StatusInternalServerError)
-			}
-		}
+	if _, err := a.Srv().Store.Post().OverwriteMultiple(postsForOverwriteList); err != nil {
+		return err
 	}
 
-	if data.Reactions != nil {
-		for _, reaction := range *data.Reactions {
-			if err := a.importReaction(&reaction, post, dryRun); err != nil {
+	for _, postWithData := range postsWithData {
+		if postWithData.directPostData.FlaggedBy != nil {
+			var preferences model.Preferences
+
+			for _, username := range *postWithData.directPostData.FlaggedBy {
+				user := users[username]
+
+				preferences = append(preferences, model.Preference{
+					UserId:   user.Id,
+					Category: model.PREFERENCE_CATEGORY_FLAGGED_POST,
+					Name:     postWithData.post.Id,
+					Value:    "true",
+				})
+			}
+
+			if len(preferences) > 0 {
+				if err := a.Srv().Store.Preference().Save(&preferences); err != nil {
+					return model.NewAppError("BulkImport", "app.import.import_post.save_preferences.error", nil, err.Error(), http.StatusInternalServerError)
+				}
+			}
+		}
+
+		if postWithData.directPostData.Reactions != nil {
+			for _, reaction := range *postWithData.directPostData.Reactions {
+				if err := a.importReaction(&reaction, postWithData.post, dryRun); err != nil {
+					return err
+				}
+			}
+		}
+
+		if postWithData.directPostData.Replies != nil {
+			if err := a.importReplies(*postWithData.directPostData.Replies, postWithData.post, "noteam", dryRun); err != nil {
 				return err
 			}
 		}
-	}
 
-	if data.Replies != nil {
-		for _, reply := range *data.Replies {
-			if err := a.importReply(&reply, post, "noteam", dryRun); err != nil {
-				return err
-			}
-		}
+		a.updateFileInfoWithPostId(postWithData.post)
 	}
-
-	a.updateFileInfoWithPostId(post)
 	return nil
 }
 

app/import_test.go

@@ -239,12 +239,12 @@ func GetAttachments(userId string, th *TestHelper, t *testing.T) []*model.FileIn
 
 func AssertFileIdsInPost(files []*model.FileInfo, th *TestHelper, t *testing.T) {
 	postId := files[0].PostId
-	assert.NotNil(t, postId)
+	require.NotNil(t, postId)
 
 	posts, err := th.App.Srv().Store.Post().GetPostsByIds([]string{postId})
 	require.Nil(t, err)
 
-	assert.Equal(t, len(posts), 1)
+	require.Len(t, posts, 1)
 	for _, file := range files {
 		assert.Contains(t, posts[0].FileIds, file.Id)
 	}