diff --git a/app/import.go b/app/import.go index c3e5cb3e25..42c9180b70 100644 --- a/app/import.go +++ b/app/import.go @@ -30,31 +30,31 @@ func stopOnError(err LineImportWorkerError) bool { } func (a *App) bulkImportWorker(dryRun bool, wg *sync.WaitGroup, lines <-chan LineImportWorkerData, errors chan<- LineImportWorkerError) { - posts := []*PostImportData{} - directPosts := []*DirectPostImportData{} + postLines := []LineImportWorkerData{} + directPostLines := []LineImportWorkerData{} for line := range lines { switch { case line.LineImportData.Type == "post": - posts = append(posts, line.Post) + postLines = append(postLines, line) if line.Post == nil { errors <- LineImportWorkerError{model.NewAppError("BulkImport", "app.import.import_line.null_post.error", nil, "", http.StatusBadRequest), line.LineNumber} } - if len(posts) >= importMultiplePostsThreshold { - if err := a.importMultiplePosts(posts, dryRun); err != nil { - errors <- LineImportWorkerError{err, line.LineNumber} + if len(postLines) >= importMultiplePostsThreshold { + if errLine, err := a.importMultiplePostLines(postLines, dryRun); err != nil { + errors <- LineImportWorkerError{err, errLine} } - posts = []*PostImportData{} + postLines = []LineImportWorkerData{} } case line.LineImportData.Type == "direct_post": - directPosts = append(directPosts, line.DirectPost) + directPostLines = append(directPostLines, line) if line.DirectPost == nil { errors <- LineImportWorkerError{model.NewAppError("BulkImport", "app.import.import_line.null_direct_post.error", nil, "", http.StatusBadRequest), line.LineNumber} } - if len(directPosts) >= importMultiplePostsThreshold { - if err := a.importMultipleDirectPosts(directPosts, dryRun); err != nil { - errors <- LineImportWorkerError{err, line.LineNumber} + if len(directPostLines) >= importMultiplePostsThreshold { + if errLine, err := a.importMultipleDirectPostLines(directPostLines, dryRun); err != nil { + errors <- LineImportWorkerError{err, errLine} } - directPosts = []*DirectPostImportData{} + directPostLines = []LineImportWorkerData{} } default: if err := a.importLine(line.LineImportData, dryRun); err != nil { @@ -63,14 +63,14 @@ func (a *App) bulkImportWorker(dryRun bool, wg *sync.WaitGroup, lines <-chan Lin } } - if len(posts) > 0 { - if err := a.importMultiplePosts(posts, dryRun); err != nil { - errors <- LineImportWorkerError{err, 0} + if len(postLines) > 0 { + if errLine, err := a.importMultiplePostLines(postLines, dryRun); err != nil { + errors <- LineImportWorkerError{err, errLine} } } - if len(directPosts) > 0 { - if err := a.importMultipleDirectPosts(directPosts, dryRun); err != nil { - errors <- LineImportWorkerError{err, 0} + if len(directPostLines) > 0 { + if errLine, err := a.importMultipleDirectPostLines(directPostLines, dryRun); err != nil { + errors <- LineImportWorkerError{err, errLine} } } wg.Done() diff --git a/app/import_functions.go b/app/import_functions.go index cd84105bd6..d14761f412 100644 --- a/app/import_functions.go +++ b/app/import_functions.go @@ -7,6 +7,7 @@ import ( "bytes" "crypto/sha1" "errors" + "fmt" "io" "net/http" "os" @@ -1030,12 +1031,12 @@ func (a *App) importReplies(data []ReplyImportData, post *model.Post, teamId str } if len(postsForCreateList) > 0 { - if _, err := a.Srv().Store.Post().SaveMultiple(postsForCreateList); err != nil { + if _, _, err := a.Srv().Store.Post().SaveMultiple(postsForCreateList); err != nil { return err } } - if _, err := a.Srv().Store.Post().OverwriteMultiple(postsForOverwriteList); err != nil { + if _, _, err := a.Srv().Store.Post().OverwriteMultiple(postsForOverwriteList); err != nil { return err } @@ -1097,6 +1098,7 @@ type postAndData struct { directPostData *DirectPostImportData replyData *ReplyImportData team *model.Team + lineNumber int } func (a *App) getUsersByUsernames(usernames []string) (map[string]*model.User, *model.AppError) { @@ -1159,64 +1161,76 @@ func (a *App) getChannelsForPosts(teams map[string]*model.Team, data []*PostImpo return channels, nil } -func (a *App) importMultiplePosts(data []*PostImportData, dryRun bool) *model.AppError { - if len(data) == 0 { - return nil +// getPostStrID returns a string ID composed of several post fields to +// uniquely identify a post before it's imported, so it has no ID yet +func getPostStrID(post *model.Post) string { + return fmt.Sprintf("%d%s%s", post.CreateAt, post.ChannelId, post.Message) +} + +// importMultiplePostLines will return an error and the line that +// caused it whenever possible +func (a *App) importMultiplePostLines(lines []LineImportWorkerData, dryRun bool) (int, *model.AppError) { + if len(lines) == 0 { + return 0, nil } - for _, postData := range data { - if err := validatePostImportData(postData, a.MaxPostSize()); err != nil { - return err + for _, line := range lines { + if err := validatePostImportData(line.Post, a.MaxPostSize()); err != nil { + return line.LineNumber, err } } // If this is a Dry Run, do not continue any further. if dryRun { - return nil + return 0, nil } usernames := []string{} - teamNames := []string{} - for _, postData := range data { - usernames = append(usernames, *postData.User) - if postData.FlaggedBy != nil { - usernames = append(usernames, *postData.FlaggedBy...) + teamNames := make([]string, len(lines)) + postsData := make([]*PostImportData, len(lines)) + for i, line := range lines { + usernames = append(usernames, *line.Post.User) + if line.Post.FlaggedBy != nil { + usernames = append(usernames, *line.Post.FlaggedBy...) } - teamNames = append(teamNames, *postData.Team) + teamNames[i] = *line.Post.Team + postsData[i] = line.Post } users, err := a.getUsersByUsernames(usernames) if err != nil { - return err + return 0, err } teams, err := a.getTeamsByNames(teamNames) if err != nil { - return err + return 0, err } - channels, err := a.getChannelsForPosts(teams, data) + channels, err := a.getChannelsForPosts(teams, postsData) if err != nil { - return err + return 0, err } postsWithData := []postAndData{} postsForCreateList := []*model.Post{} + postsForCreateMap := map[string]int{} postsForOverwriteList := []*model.Post{} + postsForOverwriteMap := map[string]int{} - for _, postData := range data { - team := teams[*postData.Team] - channel := channels[*postData.Channel] - user := users[*postData.User] + for _, line := range lines { + team := teams[*line.Post.Team] + channel := channels[*line.Post.Channel] + user := users[*line.Post.User] // Check if this post already exists. - posts, err := a.Srv().Store.Post().GetPostsCreatedAt(channel.Id, *postData.CreateAt) + posts, err := a.Srv().Store.Post().GetPostsCreatedAt(channel.Id, *line.Post.CreateAt) if err != nil { - return err + return line.LineNumber, err } var post *model.Post for _, p := range posts { - if p.Message == *postData.Message { + if p.Message == *line.Post.Message { post = p break } @@ -1227,18 +1241,18 @@ func (a *App) importMultiplePosts(data []*PostImportData, dryRun bool) *model.Ap } post.ChannelId = channel.Id - post.Message = *postData.Message + post.Message = *line.Post.Message post.UserId = user.Id - post.CreateAt = *postData.CreateAt + post.CreateAt = *line.Post.CreateAt post.Hashtags, _ = model.ParseHashtags(post.Message) - if postData.Props != nil { - post.Props = *postData.Props + if line.Post.Props != nil { + post.Props = *line.Post.Props } - fileIds, err := a.uploadAttachments(postData.Attachments, post, team.Id, dryRun) + fileIds, err := a.uploadAttachments(line.Post.Attachments, post, team.Id, dryRun) if err != nil { - return err + return line.LineNumber, err } for _, fileID := range post.FileIds { if _, ok := fileIds[fileID]; !ok { @@ -1252,20 +1266,34 @@ func (a *App) importMultiplePosts(data []*PostImportData, dryRun bool) *model.Ap if len(post.Id) == 0 { postsForCreateList = append(postsForCreateList, post) + postsForCreateMap[getPostStrID(post)] = line.LineNumber } else { postsForOverwriteList = append(postsForOverwriteList, post) + postsForOverwriteMap[getPostStrID(post)] = line.LineNumber } - postsWithData = append(postsWithData, postAndData{post: post, postData: postData, team: team}) + postsWithData = append(postsWithData, postAndData{post: post, postData: line.Post, team: team, lineNumber: line.LineNumber}) } if len(postsForCreateList) > 0 { - if _, err := a.Srv().Store.Post().SaveMultiple(postsForCreateList); err != nil { - return err + if _, idx, err := a.Srv().Store.Post().SaveMultiple(postsForCreateList); err != nil { + if idx != -1 && idx < len(postsForCreateList) { + post := postsForCreateList[idx] + if lineNumber, ok := postsForCreateMap[getPostStrID(post)]; ok { + return lineNumber, err + } + } + return 0, err } } - if _, err := a.Srv().Store.Post().OverwriteMultiple(postsForOverwriteList); err != nil { - return err + if _, idx, err := a.Srv().Store.Post().OverwriteMultiple(postsForOverwriteList); err != nil { + if idx != -1 && idx < len(postsForOverwriteList) { + post := postsForOverwriteList[idx] + if lineNumber, ok := postsForOverwriteMap[getPostStrID(post)]; ok { + return lineNumber, err + } + } + return 0, err } for _, postWithData := range postsWithData { @@ -1286,7 +1314,7 @@ func (a *App) importMultiplePosts(data []*PostImportData, dryRun bool) *model.Ap if len(preferences) > 0 { if err := a.Srv().Store.Preference().Save(&preferences); err != nil { - return model.NewAppError("BulkImport", "app.import.import_post.save_preferences.error", nil, err.Error(), http.StatusInternalServerError) + return postWithData.lineNumber, model.NewAppError("BulkImport", "app.import.import_post.save_preferences.error", nil, err.Error(), http.StatusInternalServerError) } } } @@ -1295,7 +1323,7 @@ func (a *App) importMultiplePosts(data []*PostImportData, dryRun bool) *model.Ap for _, reaction := range *postWithData.postData.Reactions { reaction := reaction if err := a.importReaction(&reaction, postWithData.post, dryRun); err != nil { - return err + return postWithData.lineNumber, err } } } @@ -1303,12 +1331,12 @@ func (a *App) importMultiplePosts(data []*PostImportData, dryRun bool) *model.Ap if postWithData.postData.Replies != nil && len(*postWithData.postData.Replies) > 0 { err := a.importReplies(*postWithData.postData.Replies, postWithData.post, postWithData.team.Id, dryRun) if err != nil { - return err + return postWithData.lineNumber, err } } a.updateFileInfoWithPostId(postWithData.post) } - return nil + return 0, nil } // uploadAttachments imports new attachments and returns current attachments of the post as a map @@ -1408,44 +1436,48 @@ func (a *App) importDirectChannel(data *DirectChannelImportData, dryRun bool) *m return nil } -func (a *App) importMultipleDirectPosts(data []*DirectPostImportData, dryRun bool) *model.AppError { - if len(data) == 0 { - return nil +// importMultipleDirectPostLines will return an error and the line +// that caused it whenever possible +func (a *App) importMultipleDirectPostLines(lines []LineImportWorkerData, dryRun bool) (int, *model.AppError) { + if len(lines) == 0 { + return 0, nil } - for _, postData := range data { - if err := validateDirectPostImportData(postData, a.MaxPostSize()); err != nil { - return err + for _, line := range lines { + if err := validateDirectPostImportData(line.DirectPost, a.MaxPostSize()); err != nil { + return line.LineNumber, err } } // If this is a Dry Run, do not continue any further. if dryRun { - return nil + return 0, nil } usernames := []string{} - for _, postData := range data { - usernames = append(usernames, *postData.User) - if postData.FlaggedBy != nil { - usernames = append(usernames, *postData.FlaggedBy...) + for _, line := range lines { + usernames = append(usernames, *line.DirectPost.User) + if line.DirectPost.FlaggedBy != nil { + usernames = append(usernames, *line.DirectPost.FlaggedBy...) } - usernames = append(usernames, *postData.ChannelMembers...) + usernames = append(usernames, *line.DirectPost.ChannelMembers...) } users, err := a.getUsersByUsernames(usernames) if err != nil { - return err + return 0, err } postsWithData := []postAndData{} postsForCreateList := []*model.Post{} + postsForCreateMap := map[string]int{} postsForOverwriteList := []*model.Post{} + postsForOverwriteMap := map[string]int{} - for _, postData := range data { + for _, line := range lines { var userIds []string var err *model.AppError - for _, username := range *postData.ChannelMembers { + for _, username := range *line.DirectPost.ChannelMembers { user := users[username] userIds = append(userIds, user.Id) } @@ -1455,28 +1487,28 @@ func (a *App) importMultipleDirectPosts(data []*DirectPostImportData, dryRun boo if len(userIds) == 2 { ch, err = a.GetOrCreateDirectChannel(userIds[0], userIds[1]) if err != nil && err.Id != store.CHANNEL_EXISTS_ERROR { - return model.NewAppError("BulkImport", "app.import.import_direct_post.create_direct_channel.error", nil, err.Error(), http.StatusBadRequest) + return line.LineNumber, model.NewAppError("BulkImport", "app.import.import_direct_post.create_direct_channel.error", nil, err.Error(), http.StatusBadRequest) } channel = ch } else { ch, err = a.createGroupChannel(userIds, userIds[0]) if err != nil && err.Id != store.CHANNEL_EXISTS_ERROR { - return model.NewAppError("BulkImport", "app.import.import_direct_post.create_group_channel.error", nil, err.Error(), http.StatusBadRequest) + return line.LineNumber, model.NewAppError("BulkImport", "app.import.import_direct_post.create_group_channel.error", nil, err.Error(), http.StatusBadRequest) } channel = ch } - user := users[*postData.User] + user := users[*line.DirectPost.User] // Check if this post already exists. - posts, err := a.Srv().Store.Post().GetPostsCreatedAt(channel.Id, *postData.CreateAt) + posts, err := a.Srv().Store.Post().GetPostsCreatedAt(channel.Id, *line.DirectPost.CreateAt) if err != nil { - return err + return line.LineNumber, err } var post *model.Post for _, p := range posts { - if p.Message == *postData.Message { + if p.Message == *line.DirectPost.Message { post = p break } @@ -1487,18 +1519,18 @@ func (a *App) importMultipleDirectPosts(data []*DirectPostImportData, dryRun boo } post.ChannelId = channel.Id - post.Message = *postData.Message + post.Message = *line.DirectPost.Message post.UserId = user.Id - post.CreateAt = *postData.CreateAt + post.CreateAt = *line.DirectPost.CreateAt post.Hashtags, _ = model.ParseHashtags(post.Message) - if postData.Props != nil { - post.Props = *postData.Props + if line.DirectPost.Props != nil { + post.Props = *line.DirectPost.Props } - fileIds, err := a.uploadAttachments(postData.Attachments, post, "noteam", dryRun) + fileIds, err := a.uploadAttachments(line.DirectPost.Attachments, post, "noteam", dryRun) if err != nil { - return err + return line.LineNumber, err } for _, fileID := range post.FileIds { if _, ok := fileIds[fileID]; !ok { @@ -1512,19 +1544,33 @@ func (a *App) importMultipleDirectPosts(data []*DirectPostImportData, dryRun boo if len(post.Id) == 0 { postsForCreateList = append(postsForCreateList, post) + postsForCreateMap[getPostStrID(post)] = line.LineNumber } else { postsForOverwriteList = append(postsForOverwriteList, post) + postsForOverwriteMap[getPostStrID(post)] = line.LineNumber } - postsWithData = append(postsWithData, postAndData{post: post, directPostData: postData}) + postsWithData = append(postsWithData, postAndData{post: post, directPostData: line.DirectPost, lineNumber: line.LineNumber}) } if len(postsForCreateList) > 0 { - if _, err := a.Srv().Store.Post().SaveMultiple(postsForCreateList); err != nil { - return err + if _, idx, err := a.Srv().Store.Post().SaveMultiple(postsForCreateList); err != nil { + if idx != -1 && idx < len(postsForCreateList) { + post := postsForCreateList[idx] + if lineNumber, ok := postsForCreateMap[getPostStrID(post)]; ok { + return lineNumber, err + } + } + return 0, err } } - if _, err := a.Srv().Store.Post().OverwriteMultiple(postsForOverwriteList); err != nil { - return err + if _, idx, err := a.Srv().Store.Post().OverwriteMultiple(postsForOverwriteList); err != nil { + if idx != -1 && idx < len(postsForOverwriteList) { + post := postsForOverwriteList[idx] + if lineNumber, ok := postsForOverwriteMap[getPostStrID(post)]; ok { + return lineNumber, err + } + } + return 0, err } for _, postWithData := range postsWithData { @@ -1544,7 +1590,7 @@ func (a *App) importMultipleDirectPosts(data []*DirectPostImportData, dryRun boo if len(preferences) > 0 { if err := a.Srv().Store.Preference().Save(&preferences); err != nil { - return model.NewAppError("BulkImport", "app.import.import_post.save_preferences.error", nil, err.Error(), http.StatusInternalServerError) + return postWithData.lineNumber, model.NewAppError("BulkImport", "app.import.import_post.save_preferences.error", nil, err.Error(), http.StatusInternalServerError) } } } @@ -1553,20 +1599,20 @@ func (a *App) importMultipleDirectPosts(data []*DirectPostImportData, dryRun boo for _, reaction := range *postWithData.directPostData.Reactions { reaction := reaction if err := a.importReaction(&reaction, postWithData.post, dryRun); err != nil { - return err + return postWithData.lineNumber, err } } } if postWithData.directPostData.Replies != nil { if err := a.importReplies(*postWithData.directPostData.Replies, postWithData.post, "noteam", dryRun); err != nil { - return err + return postWithData.lineNumber, err } } a.updateFileInfoWithPostId(postWithData.post) } - return nil + return 0, nil } func (a *App) importEmoji(data *EmojiImportData, dryRun bool) *model.AppError { diff --git a/app/import_functions_test.go b/app/import_functions_test.go index 969e4966b2..9e0f5aa2ee 100644 --- a/app/import_functions_test.go +++ b/app/import_functions_test.go @@ -1923,7 +1923,7 @@ func TestImportUserDefaultNotifyProps(t *testing.T) { } } -func TestImportimportMultiplePosts(t *testing.T) { +func TestImportimportMultiplePostLines(t *testing.T) { th := Setup(t) defer th.TearDown() @@ -1962,85 +1962,133 @@ func TestImportimportMultiplePosts(t *testing.T) { require.Nil(t, err) // Try adding an invalid post in dry run mode. - data := &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + }, + }, + 25, } - err = th.App.importMultiplePosts([]*PostImportData{data}, true) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, true) assert.NotNil(t, err) + assert.Equal(t, data.LineNumber, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) // Try adding a valid post in dry run mode. - data = &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - Message: ptrStr("Hello"), - CreateAt: ptrInt64(model.GetMillis()), + data = LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Hello"), + CreateAt: ptrInt64(model.GetMillis()), + }, + }, + 1, } - err = th.App.importMultiplePosts([]*PostImportData{data}, true) + errLine, err = th.App.importMultiplePostLines([]LineImportWorkerData{data}, true) assert.Nil(t, err) + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) // Try adding an invalid post in apply mode. - data = &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - CreateAt: ptrInt64(model.GetMillis()), + data = LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + CreateAt: ptrInt64(model.GetMillis()), + }, + }, + 35, } - err = th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err = th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.NotNil(t, err) + assert.Equal(t, data.LineNumber, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) // Try adding a valid post with invalid team in apply mode. - data = &PostImportData{ - Team: ptrStr(model.NewId()), - Channel: &channelName, - User: &username, - Message: ptrStr("Message"), - CreateAt: ptrInt64(model.GetMillis()), + data = LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: ptrStr(model.NewId()), + Channel: &channelName, + User: &username, + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + }, + }, + 10, } - err = th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err = th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.NotNil(t, err) + // Batch will fail when searching for teams, so no specific line + // is associated with the error + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) // Try adding a valid post with invalid channel in apply mode. - data = &PostImportData{ - Team: &teamName, - Channel: ptrStr(model.NewId()), - User: &username, - Message: ptrStr("Message"), - CreateAt: ptrInt64(model.GetMillis()), + data = LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: ptrStr(model.NewId()), + User: &username, + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + }, + }, + 7, } - err = th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err = th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.NotNil(t, err) + // Batch will fail when searching for channels, so no specific + // line is associated with the error + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) // Try adding a valid post with invalid user in apply mode. - data = &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: ptrStr(model.NewId()), - Message: ptrStr("Message"), - CreateAt: ptrInt64(model.GetMillis()), + data = LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: ptrStr(model.NewId()), + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + }, + }, + 2, } - err = th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err = th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.NotNil(t, err) + // Batch will fail when searching for users, so no specific line + // is associated with the error + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) // Try adding a valid post in apply mode. time := model.GetMillis() - data = &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - Message: ptrStr("Message"), - CreateAt: &time, + data = LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message"), + CreateAt: &time, + }, + }, + 1, } - err = th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err = th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.Nil(t, err) + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 1, team.Id) // Check the post values. @@ -2050,19 +2098,25 @@ func TestImportimportMultiplePosts(t *testing.T) { require.Len(t, posts, 1, "Unexpected number of posts found.") post := posts[0] - postBool := post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id + postBool := post.Message != *data.Post.Message || post.CreateAt != *data.Post.CreateAt || post.UserId != user.Id require.False(t, postBool, "Post properties not as expected") // Update the post. - data = &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - Message: ptrStr("Message"), - CreateAt: &time, + data = LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message"), + CreateAt: &time, + }, + }, + 1, } - err = th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err = th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.Nil(t, err) + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 1, team.Id) // Check the post values. @@ -2072,45 +2126,63 @@ func TestImportimportMultiplePosts(t *testing.T) { require.Len(t, posts, 1, "Unexpected number of posts found.") post = posts[0] - postBool = post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id + postBool = post.Message != *data.Post.Message || post.CreateAt != *data.Post.CreateAt || post.UserId != user.Id require.False(t, postBool, "Post properties not as expected") // Save the post with a different time. newTime := time + 1 - data = &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - Message: ptrStr("Message"), - CreateAt: &newTime, + data = LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message"), + CreateAt: &newTime, + }, + }, + 1, } - err = th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err = th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.Nil(t, err) + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 2, team.Id) // Save the post with a different message. - data = &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - Message: ptrStr("Message 2"), - CreateAt: &time, + data = LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message 2"), + CreateAt: &time, + }, + }, + 1, } - err = th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err = th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.Nil(t, err) + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 3, team.Id) // Test with hashtags hashtagTime := time + 2 - data = &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - Message: ptrStr("Message 2 #hashtagmashupcity"), - CreateAt: &hashtagTime, + data = LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message 2 #hashtagmashupcity"), + CreateAt: &hashtagTime, + }, + }, + 1, } - err = th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err = th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.Nil(t, err) + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 4, team.Id) posts, err = th.App.Srv().Store.Post().GetPostsCreatedAt(channel.Id, hashtagTime) @@ -2119,7 +2191,7 @@ func TestImportimportMultiplePosts(t *testing.T) { require.Len(t, posts, 1, "Unexpected number of posts found.") post = posts[0] - postBool = post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id + postBool = post.Message != *data.Post.Message || post.CreateAt != *data.Post.CreateAt || post.UserId != user.Id require.False(t, postBool, "Post properties not as expected") require.Equal(t, "#hashtagmashupcity", post.Hashtags, "Hashtags not as expected: %s", post.Hashtags) @@ -2134,20 +2206,26 @@ func TestImportimportMultiplePosts(t *testing.T) { require.Nil(t, err, "Failed to get user from database.") flagsTime := hashtagTime + 1 - data = &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - Message: ptrStr("Message with Favorites"), - CreateAt: &flagsTime, - FlaggedBy: &[]string{ - username, - username2, + data = LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message with Favorites"), + CreateAt: &flagsTime, + FlaggedBy: &[]string{ + username, + username2, + }, + }, }, + 1, } - err = th.App.importMultiplePosts([]*PostImportData{data}, false) - require.Nil(t, err, "Expected success.") + errLine, err = th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) + assert.Nil(t, err, "Expected success.") + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 5, team.Id) @@ -2158,7 +2236,7 @@ func TestImportimportMultiplePosts(t *testing.T) { require.Len(t, posts, 1, "Unexpected number of posts found.") post = posts[0] - postBool = post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id + postBool = post.Message != *data.Post.Message || post.CreateAt != *data.Post.CreateAt || post.UserId != user.Id require.False(t, postBool, "Post properties not as expected") checkPreference(t, th.App, user.Id, model.PREFERENCE_CATEGORY_FLAGGED_POST, post.Id, "true") @@ -2167,20 +2245,26 @@ func TestImportimportMultiplePosts(t *testing.T) { // Post with reaction. reactionPostTime := hashtagTime + 2 reactionTime := hashtagTime + 3 - data = &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - Message: ptrStr("Message with reaction"), - CreateAt: &reactionPostTime, - Reactions: &[]ReactionImportData{{ - User: &user2.Username, - EmojiName: ptrStr("+1"), - CreateAt: &reactionTime, - }}, + data = LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message with reaction"), + CreateAt: &reactionPostTime, + Reactions: &[]ReactionImportData{{ + User: &user2.Username, + EmojiName: ptrStr("+1"), + CreateAt: &reactionTime, + }}, + }, + }, + 1, } - err = th.App.importMultiplePosts([]*PostImportData{data}, false) - require.Nil(t, err, "Expected success.") + errLine, err = th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) + assert.Nil(t, err, "Expected success.") + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 6, team.Id) @@ -2191,7 +2275,7 @@ func TestImportimportMultiplePosts(t *testing.T) { require.Len(t, posts, 1, "Unexpected number of posts found.") post = posts[0] - postBool = post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id || !post.HasReactions + postBool = post.Message != *data.Post.Message || post.CreateAt != *data.Post.CreateAt || post.UserId != user.Id || !post.HasReactions require.False(t, postBool, "Post properties not as expected") reactions, err := th.App.Srv().Store.Reaction().GetForPost(post.Id, false) @@ -2202,20 +2286,26 @@ func TestImportimportMultiplePosts(t *testing.T) { // Post with reply. replyPostTime := hashtagTime + 4 replyTime := hashtagTime + 5 - data = &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - Message: ptrStr("Message with reply"), - CreateAt: &replyPostTime, - Replies: &[]ReplyImportData{{ - User: &user2.Username, - Message: ptrStr("Message reply"), - CreateAt: &replyTime, - }}, + data = LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message with reply"), + CreateAt: &replyPostTime, + Replies: &[]ReplyImportData{{ + User: &user2.Username, + Message: ptrStr("Message reply"), + CreateAt: &replyTime, + }}, + }, + }, + 1, } - err = th.App.importMultiplePosts([]*PostImportData{data}, false) - require.Nil(t, err, "Expected success.") + errLine, err = th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) + assert.Nil(t, err, "Expected success.") + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 8, team.Id) @@ -2226,7 +2316,7 @@ func TestImportimportMultiplePosts(t *testing.T) { require.Len(t, posts, 1, "Unexpected number of posts found.") post = posts[0] - postBool = post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id + postBool = post.Message != *data.Post.Message || post.CreateAt != *data.Post.CreateAt || post.UserId != user.Id require.False(t, postBool, "Post properties not as expected") // Check the reply values. @@ -2236,62 +2326,80 @@ func TestImportimportMultiplePosts(t *testing.T) { require.Len(t, replies, 1, "Unexpected number of posts found.") reply := replies[0] - replyBool := reply.Message != *(*data.Replies)[0].Message || reply.CreateAt != *(*data.Replies)[0].CreateAt || reply.UserId != user2.Id + replyBool := reply.Message != *(*data.Post.Replies)[0].Message || reply.CreateAt != *(*data.Post.Replies)[0].CreateAt || reply.UserId != user2.Id require.False(t, replyBool, "Post properties not as expected") require.Equal(t, post.Id, reply.RootId, "Unexpected reply RootId") // Update post with replies. - data = &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &user2.Username, - Message: ptrStr("Message with reply"), - CreateAt: &replyPostTime, - Replies: &[]ReplyImportData{{ - User: &username, - Message: ptrStr("Message reply"), - CreateAt: &replyTime, - }}, + data = LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &user2.Username, + Message: ptrStr("Message with reply"), + CreateAt: &replyPostTime, + Replies: &[]ReplyImportData{{ + User: &username, + Message: ptrStr("Message reply"), + CreateAt: &replyTime, + }}, + }, + }, + 1, } - err = th.App.importMultiplePosts([]*PostImportData{data}, false) - require.Nil(t, err, "Expected success.") + errLine, err = th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) + assert.Nil(t, err, "Expected success.") + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 8, team.Id) // Create new post with replies based on the previous one. - data = &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &user2.Username, - Message: ptrStr("Message with reply 2"), - CreateAt: &replyPostTime, - Replies: &[]ReplyImportData{{ - User: &username, - Message: ptrStr("Message reply"), - CreateAt: &replyTime, - }}, + data = LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &user2.Username, + Message: ptrStr("Message with reply 2"), + CreateAt: &replyPostTime, + Replies: &[]ReplyImportData{{ + User: &username, + Message: ptrStr("Message reply"), + CreateAt: &replyTime, + }}, + }, + }, + 1, } - err = th.App.importMultiplePosts([]*PostImportData{data}, false) - require.Nil(t, err, "Expected success.") + errLine, err = th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) + assert.Nil(t, err, "Expected success.") + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 10, team.Id) // Create new reply for existing post with replies. - data = &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &user2.Username, - Message: ptrStr("Message with reply"), - CreateAt: &replyPostTime, - Replies: &[]ReplyImportData{{ - User: &username, - Message: ptrStr("Message reply 2"), - CreateAt: &replyTime, - }}, + data = LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &user2.Username, + Message: ptrStr("Message with reply"), + CreateAt: &replyPostTime, + Replies: &[]ReplyImportData{{ + User: &username, + Message: ptrStr("Message reply 2"), + CreateAt: &replyTime, + }}, + }, + }, + 1, } - err = th.App.importMultiplePosts([]*PostImportData{data}, false) - require.Nil(t, err, "Expected success.") + errLine, err = th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) + assert.Nil(t, err, "Expected success.") + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 11, team.Id) } @@ -2348,90 +2456,132 @@ func TestImportImportPost(t *testing.T) { replyTime := hashtagTime + 5 t.Run("Try adding an invalid post in dry run mode", func(t *testing.T) { - data := &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + }, + }, + 12, } - err := th.App.importMultiplePosts([]*PostImportData{data}, true) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, true) assert.NotNil(t, err) + assert.Equal(t, data.LineNumber, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) }) t.Run("Try adding a valid post in dry run mode", func(t *testing.T) { - data := &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - Message: ptrStr("Hello"), - CreateAt: ptrInt64(model.GetMillis()), + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Hello"), + CreateAt: ptrInt64(model.GetMillis()), + }, + }, + 1, } - err := th.App.importMultiplePosts([]*PostImportData{data}, true) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, true) assert.Nil(t, err) + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) }) t.Run("Try adding an invalid post in apply mode", func(t *testing.T) { - data := &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - CreateAt: ptrInt64(model.GetMillis()), + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + CreateAt: ptrInt64(model.GetMillis()), + }, + }, + 2, } - err := th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.NotNil(t, err) + assert.Equal(t, data.LineNumber, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) }) t.Run("Try adding a valid post with invalid team in apply mode", func(t *testing.T) { - data := &PostImportData{ - Team: ptrStr(model.NewId()), - Channel: &channelName, - User: &username, - Message: ptrStr("Message"), - CreateAt: ptrInt64(model.GetMillis()), + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: ptrStr(model.NewId()), + Channel: &channelName, + User: &username, + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + }, + }, + 7, } - err := th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.NotNil(t, err) + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) }) t.Run("Try adding a valid post with invalid channel in apply mode", func(t *testing.T) { - data := &PostImportData{ - Team: &teamName, - Channel: ptrStr(model.NewId()), - User: &username, - Message: ptrStr("Message"), - CreateAt: ptrInt64(model.GetMillis()), + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: ptrStr(model.NewId()), + User: &username, + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + }, + }, + 8, } - err := th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.NotNil(t, err) + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) }) t.Run("Try adding a valid post with invalid user in apply mode", func(t *testing.T) { - data := &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: ptrStr(model.NewId()), - Message: ptrStr("Message"), - CreateAt: ptrInt64(model.GetMillis()), + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: ptrStr(model.NewId()), + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + }, + }, + 9, } - err := th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.NotNil(t, err) + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) }) t.Run("Try adding a valid post in apply mode", func(t *testing.T) { - data := &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - Message: ptrStr("Message"), - CreateAt: &time, + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message"), + CreateAt: &time, + }, + }, + 1, } - err := th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.Nil(t, err) + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 1, team.Id) // Check the post values. @@ -2441,20 +2591,26 @@ func TestImportImportPost(t *testing.T) { require.Len(t, posts, 1, "Unexpected number of posts found.") post := posts[0] - postBool := post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id + postBool := post.Message != *data.Post.Message || post.CreateAt != *data.Post.CreateAt || post.UserId != user.Id require.False(t, postBool, "Post properties not as expected") }) t.Run("Update the post", func(t *testing.T) { - data := &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username2, - Message: ptrStr("Message"), - CreateAt: &time, + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username2, + Message: ptrStr("Message"), + CreateAt: &time, + }, + }, + 1, } - err := th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.Nil(t, err) + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 1, team.Id) // Check the post values. @@ -2464,47 +2620,65 @@ func TestImportImportPost(t *testing.T) { require.Len(t, posts, 1, "Unexpected number of posts found.") post := posts[0] - postBool := post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user2.Id + postBool := post.Message != *data.Post.Message || post.CreateAt != *data.Post.CreateAt || post.UserId != user2.Id require.False(t, postBool, "Post properties not as expected") }) t.Run("Save the post with a different time", func(t *testing.T) { newTime := time + 1 - data := &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - Message: ptrStr("Message"), - CreateAt: &newTime, + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message"), + CreateAt: &newTime, + }, + }, + 1, } - err := th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.Nil(t, err) + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 2, team.Id) }) t.Run("Save the post with a different message", func(t *testing.T) { - data := &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - Message: ptrStr("Message 2"), - CreateAt: &time, + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message 2"), + CreateAt: &time, + }, + }, + 1, } - err := th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.Nil(t, err) + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 3, team.Id) }) t.Run("Test with hashtag", func(t *testing.T) { - data := &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - Message: ptrStr("Message 2 #hashtagmashupcity"), - CreateAt: &hashtagTime, + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message 2 #hashtagmashupcity"), + CreateAt: &hashtagTime, + }, + }, + 1, } - err := th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) assert.Nil(t, err) + assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 4, team.Id) posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(channel.Id, hashtagTime) @@ -2513,7 +2687,7 @@ func TestImportImportPost(t *testing.T) { require.Len(t, posts, 1, "Unexpected number of posts found.") post := posts[0] - postBool := post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id + postBool := post.Message != *data.Post.Message || post.CreateAt != *data.Post.CreateAt || post.UserId != user.Id require.False(t, postBool, "Post properties not as expected") require.Equal(t, "#hashtagmashupcity", post.Hashtags, "Hashtags not as expected: %s", post.Hashtags) @@ -2521,20 +2695,26 @@ func TestImportImportPost(t *testing.T) { t.Run("Post with flags", func(t *testing.T) { flagsTime := hashtagTime + 1 - data := &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - Message: ptrStr("Message with Favorites"), - CreateAt: &flagsTime, - FlaggedBy: &[]string{ - username, - username2, + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message with Favorites"), + CreateAt: &flagsTime, + FlaggedBy: &[]string{ + username, + username2, + }, + }, }, + 1, } - err := th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) require.Nil(t, err, "Expected success.") + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 5, team.Id) @@ -2545,7 +2725,7 @@ func TestImportImportPost(t *testing.T) { require.Len(t, posts, 1, "Unexpected number of posts found.") post := posts[0] - postBool := post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id + postBool := post.Message != *data.Post.Message || post.CreateAt != *data.Post.CreateAt || post.UserId != user.Id require.False(t, postBool, "Post properties not as expected") checkPreference(t, th.App, user.Id, model.PREFERENCE_CATEGORY_FLAGGED_POST, post.Id, "true") @@ -2555,20 +2735,26 @@ func TestImportImportPost(t *testing.T) { t.Run("Post with reaction", func(t *testing.T) { reactionPostTime := hashtagTime + 2 reactionTime := hashtagTime + 3 - data := &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - Message: ptrStr("Message with reaction"), - CreateAt: &reactionPostTime, - Reactions: &[]ReactionImportData{{ - User: &user2.Username, - EmojiName: ptrStr("+1"), - CreateAt: &reactionTime, - }}, + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message with reaction"), + CreateAt: &reactionPostTime, + Reactions: &[]ReactionImportData{{ + User: &user2.Username, + EmojiName: ptrStr("+1"), + CreateAt: &reactionTime, + }}, + }, + }, + 1, } - err := th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) require.Nil(t, err, "Expected success.") + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 6, team.Id) @@ -2579,7 +2765,7 @@ func TestImportImportPost(t *testing.T) { require.Len(t, posts, 1, "Unexpected number of posts found.") post := posts[0] - postBool := post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id || !post.HasReactions + postBool := post.Message != *data.Post.Message || post.CreateAt != *data.Post.CreateAt || post.UserId != user.Id || !post.HasReactions require.False(t, postBool, "Post properties not as expected") reactions, err := th.App.Srv().Store.Reaction().GetForPost(post.Id, false) @@ -2589,20 +2775,26 @@ func TestImportImportPost(t *testing.T) { }) t.Run("Post with reply", func(t *testing.T) { - data := &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username, - Message: ptrStr("Message with reply"), - CreateAt: &replyPostTime, - Replies: &[]ReplyImportData{{ - User: &user2.Username, - Message: ptrStr("Message reply"), - CreateAt: &replyTime, - }}, + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message with reply"), + CreateAt: &replyPostTime, + Replies: &[]ReplyImportData{{ + User: &user2.Username, + Message: ptrStr("Message reply"), + CreateAt: &replyTime, + }}, + }, + }, + 1, } - err := th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) require.Nil(t, err, "Expected success.") + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 8, team.Id) @@ -2613,7 +2805,7 @@ func TestImportImportPost(t *testing.T) { require.Len(t, posts, 1, "Unexpected number of posts found.") post := posts[0] - postBool := post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id + postBool := post.Message != *data.Post.Message || post.CreateAt != *data.Post.CreateAt || post.UserId != user.Id require.False(t, postBool, "Post properties not as expected") // Check the reply values. @@ -2623,65 +2815,83 @@ func TestImportImportPost(t *testing.T) { require.Len(t, replies, 1, "Unexpected number of posts found.") reply := replies[0] - replyBool := reply.Message != *(*data.Replies)[0].Message || reply.CreateAt != *(*data.Replies)[0].CreateAt || reply.UserId != user2.Id + replyBool := reply.Message != *(*data.Post.Replies)[0].Message || reply.CreateAt != *(*data.Post.Replies)[0].CreateAt || reply.UserId != user2.Id require.False(t, replyBool, "Post properties not as expected") require.Equal(t, post.Id, reply.RootId, "Unexpected reply RootId") }) t.Run("Update post with replies", func(t *testing.T) { - data := &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &user2.Username, - Message: ptrStr("Message with reply"), - CreateAt: &replyPostTime, - Replies: &[]ReplyImportData{{ - User: &username, - Message: ptrStr("Message reply"), - CreateAt: &replyTime, - }}, + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &user2.Username, + Message: ptrStr("Message with reply"), + CreateAt: &replyPostTime, + Replies: &[]ReplyImportData{{ + User: &username, + Message: ptrStr("Message reply"), + CreateAt: &replyTime, + }}, + }, + }, + 1, } - err := th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) require.Nil(t, err, "Expected success.") + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 8, team.Id) }) t.Run("Create new post with replies based on the previous one", func(t *testing.T) { - data := &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &user2.Username, - Message: ptrStr("Message with reply 2"), - CreateAt: &replyPostTime, - Replies: &[]ReplyImportData{{ - User: &username, - Message: ptrStr("Message reply"), - CreateAt: &replyTime, - }}, + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &user2.Username, + Message: ptrStr("Message with reply 2"), + CreateAt: &replyPostTime, + Replies: &[]ReplyImportData{{ + User: &username, + Message: ptrStr("Message reply"), + CreateAt: &replyTime, + }}, + }, + }, + 1, } - err := th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) require.Nil(t, err, "Expected success.") + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 10, team.Id) }) t.Run("Create new reply for existing post with replies", func(t *testing.T) { - data := &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &user2.Username, - Message: ptrStr("Message with reply"), - CreateAt: &replyPostTime, - Replies: &[]ReplyImportData{{ - User: &username, - Message: ptrStr("Message reply 2"), - CreateAt: &replyTime, - }}, + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &user2.Username, + Message: ptrStr("Message with reply"), + CreateAt: &replyPostTime, + Replies: &[]ReplyImportData{{ + User: &username, + Message: ptrStr("Message reply 2"), + CreateAt: &replyTime, + }}, + }, + }, + 1, } - err := th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) require.Nil(t, err, "Expected success.") + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 11, team.Id) }) @@ -2883,174 +3093,228 @@ func TestImportImportDirectPost(t *testing.T) { initialDate := model.GetMillis() t.Run("Try adding an invalid post in dry run mode", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + }, + User: ptrStr(th.BasicUser.Username), + CreateAt: ptrInt64(model.GetMillis()), + }, }, - User: ptrStr(th.BasicUser.Username), - CreateAt: ptrInt64(model.GetMillis()), + 7, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, true) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, true) require.NotNil(t, err) + require.Equal(t, data.LineNumber, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, "") }) t.Run("Try adding a valid post in dry run mode", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + }, }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message"), - CreateAt: ptrInt64(model.GetMillis()), + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, true) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, true) require.Nil(t, err) + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, "") }) t.Run("Try adding an invalid post in apply mode", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - model.NewId(), + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + model.NewId(), + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + }, }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message"), - CreateAt: ptrInt64(model.GetMillis()), + 9, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.NotNil(t, err) + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, "") }) t.Run("Try adding a valid post in apply mode", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message"), + CreateAt: ptrInt64(initialDate), + }, }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message"), - CreateAt: ptrInt64(initialDate), + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.Nil(t, err) + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 1, "") // Check the post values. - posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(directChannel.Id, *data.CreateAt) + posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(directChannel.Id, *data.DirectPost.CreateAt) require.Nil(t, err) require.Len(t, posts, 1) post := posts[0] - require.Equal(t, post.Message, *data.Message) - require.Equal(t, post.CreateAt, *data.CreateAt) + require.Equal(t, post.Message, *data.DirectPost.Message) + require.Equal(t, post.CreateAt, *data.DirectPost.CreateAt) require.Equal(t, post.UserId, th.BasicUser.Id) }) t.Run("Import the post again", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message"), + CreateAt: ptrInt64(initialDate), + }, }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message"), - CreateAt: ptrInt64(initialDate), + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.Nil(t, err) + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 1, "") // Check the post values. - posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(directChannel.Id, *data.CreateAt) + posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(directChannel.Id, *data.DirectPost.CreateAt) require.Nil(t, err) require.Len(t, posts, 1) post := posts[0] - require.Equal(t, post.Message, *data.Message) - require.Equal(t, post.CreateAt, *data.CreateAt) + require.Equal(t, post.Message, *data.DirectPost.Message) + require.Equal(t, post.CreateAt, *data.DirectPost.CreateAt) require.Equal(t, post.UserId, th.BasicUser.Id) }) t.Run("Save the post with a different time", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message"), + CreateAt: ptrInt64(initialDate + 1), + }, }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message"), - CreateAt: ptrInt64(initialDate + 1), + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.Nil(t, err) + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 2, "") }) t.Run("Save the post with a different message", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message 2"), + CreateAt: ptrInt64(initialDate + 1), + }, }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message 2"), - CreateAt: ptrInt64(initialDate + 1), + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.Nil(t, err) + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 3, "") }) t.Run("Test with hashtag", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message 2 #hashtagmashupcity"), + CreateAt: ptrInt64(initialDate + 2), + }, }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message 2 #hashtagmashupcity"), - CreateAt: ptrInt64(initialDate + 2), + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.Nil(t, err) + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 4, "") - posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(directChannel.Id, *data.CreateAt) + posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(directChannel.Id, *data.DirectPost.CreateAt) require.Nil(t, err) require.Len(t, posts, 1) post := posts[0] - require.Equal(t, post.Message, *data.Message) - require.Equal(t, post.CreateAt, *data.CreateAt) + require.Equal(t, post.Message, *data.DirectPost.Message) + require.Equal(t, post.CreateAt, *data.DirectPost.CreateAt) require.Equal(t, post.UserId, th.BasicUser.Id) require.Equal(t, post.Hashtags, "#hashtagmashupcity") }) t.Run("Test with some flags", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + }, + FlaggedBy: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + }, }, - FlaggedBy: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, - }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message"), - CreateAt: ptrInt64(model.GetMillis()), + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.Nil(t, err) + require.Equal(t, 0, errLine) // Check the post values. - posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(directChannel.Id, *data.CreateAt) + posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(directChannel.Id, *data.DirectPost.CreateAt) require.Nil(t, err) require.Len(t, posts, 1) @@ -3090,186 +3354,240 @@ func TestImportImportDirectPost(t *testing.T) { initialPostCount = result t.Run("Try adding an invalid post in dry run mode", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, - user3.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + user3.Username, + }, + User: ptrStr(th.BasicUser.Username), + CreateAt: ptrInt64(model.GetMillis()), + }, }, - User: ptrStr(th.BasicUser.Username), - CreateAt: ptrInt64(model.GetMillis()), + 4, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, true) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, true) require.NotNil(t, err) + require.Equal(t, data.LineNumber, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, "") }) t.Run("Try adding a valid post in dry run mode", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, - user3.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + user3.Username, + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + }, }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message"), - CreateAt: ptrInt64(model.GetMillis()), + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, true) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, true) require.Nil(t, err) + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, "") }) t.Run("Try adding an invalid post in apply mode", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, - user3.Username, - model.NewId(), + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + user3.Username, + model.NewId(), + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + }, }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message"), - CreateAt: ptrInt64(model.GetMillis()), + 8, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.NotNil(t, err) + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, "") }) t.Run("Try adding a valid post in apply mode", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, - user3.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + user3.Username, + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message"), + CreateAt: ptrInt64(initialDate + 10), + }, }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message"), - CreateAt: ptrInt64(initialDate + 10), + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.Nil(t, err) + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 1, "") // Check the post values. - posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(groupChannel.Id, *data.CreateAt) + posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(groupChannel.Id, *data.DirectPost.CreateAt) require.Nil(t, err) require.Len(t, posts, 1) post := posts[0] - require.Equal(t, post.Message, *data.Message) - require.Equal(t, post.CreateAt, *data.CreateAt) + require.Equal(t, post.Message, *data.DirectPost.Message) + require.Equal(t, post.CreateAt, *data.DirectPost.CreateAt) require.Equal(t, post.UserId, th.BasicUser.Id) }) t.Run("Import the post again", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, - user3.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + user3.Username, + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message"), + CreateAt: ptrInt64(initialDate + 10), + }, }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message"), - CreateAt: ptrInt64(initialDate + 10), + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.Nil(t, err) + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 1, "") // Check the post values. - posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(groupChannel.Id, *data.CreateAt) + posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(groupChannel.Id, *data.DirectPost.CreateAt) require.Nil(t, err) require.Len(t, posts, 1) post := posts[0] - require.Equal(t, post.Message, *data.Message) - require.Equal(t, post.CreateAt, *data.CreateAt) + require.Equal(t, post.Message, *data.DirectPost.Message) + require.Equal(t, post.CreateAt, *data.DirectPost.CreateAt) require.Equal(t, post.UserId, th.BasicUser.Id) }) t.Run("Save the post with a different time", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, - user3.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + user3.Username, + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message"), + CreateAt: ptrInt64(initialDate + 11), + }, }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message"), - CreateAt: ptrInt64(initialDate + 11), + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.Nil(t, err) + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 2, "") }) t.Run("Save the post with a different message", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, - user3.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + user3.Username, + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message 2"), + CreateAt: ptrInt64(initialDate + 11), + }, }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message 2"), - CreateAt: ptrInt64(initialDate + 11), + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.Nil(t, err) + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 3, "") }) t.Run("Test with hashtag", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, - user3.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + user3.Username, + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message 2 #hashtagmashupcity"), + CreateAt: ptrInt64(initialDate + 12), + }, }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message 2 #hashtagmashupcity"), - CreateAt: ptrInt64(initialDate + 12), + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.Nil(t, err) + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 4, "") - posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(groupChannel.Id, *data.CreateAt) + posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(groupChannel.Id, *data.DirectPost.CreateAt) require.Nil(t, err) require.Len(t, posts, 1) post := posts[0] - require.Equal(t, post.Message, *data.Message) - require.Equal(t, post.CreateAt, *data.CreateAt) + require.Equal(t, post.Message, *data.DirectPost.Message) + require.Equal(t, post.CreateAt, *data.DirectPost.CreateAt) require.Equal(t, post.UserId, th.BasicUser.Id) require.Equal(t, post.Hashtags, "#hashtagmashupcity") }) t.Run("Test with some flags", func(t *testing.T) { - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, - user3.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + user3.Username, + }, + FlaggedBy: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + }, }, - FlaggedBy: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, - }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message"), - CreateAt: ptrInt64(model.GetMillis()), + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.Nil(t, err) + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 5, "") // Check the post values. - posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(groupChannel.Id, *data.CreateAt) + posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(groupChannel.Id, *data.DirectPost.CreateAt) require.Nil(t, err) require.Len(t, posts, 1) @@ -3281,34 +3599,40 @@ func TestImportImportDirectPost(t *testing.T) { t.Run("Post with reaction", func(t *testing.T) { reactionPostTime := ptrInt64(initialDate + 22) reactionTime := ptrInt64(initialDate + 23) - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, - user3.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + user3.Username, + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message with reaction"), + CreateAt: reactionPostTime, + Reactions: &[]ReactionImportData{{ + User: ptrStr(th.BasicUser2.Username), + EmojiName: ptrStr("+1"), + CreateAt: reactionTime, + }}, + }, }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message with reaction"), - CreateAt: reactionPostTime, - Reactions: &[]ReactionImportData{{ - User: ptrStr(th.BasicUser2.Username), - EmojiName: ptrStr("+1"), - CreateAt: reactionTime, - }}, + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.Nil(t, err, "Expected success.") + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 6, "") // Check the post values. - posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(groupChannel.Id, *data.CreateAt) + posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(groupChannel.Id, *data.DirectPost.CreateAt) require.Nil(t, err) require.Len(t, posts, 1, "Unexpected number of posts found.") post := posts[0] - postBool := post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != th.BasicUser.Id || !post.HasReactions + postBool := post.Message != *data.DirectPost.Message || post.CreateAt != *data.DirectPost.CreateAt || post.UserId != th.BasicUser.Id || !post.HasReactions require.False(t, postBool, "Post properties not as expected") reactions, err := th.App.Srv().Store.Reaction().GetForPost(post.Id, false) @@ -3320,34 +3644,40 @@ func TestImportImportDirectPost(t *testing.T) { t.Run("Post with reply", func(t *testing.T) { replyPostTime := ptrInt64(initialDate + 25) replyTime := ptrInt64(initialDate + 26) - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, - user3.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + user3.Username, + }, + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message with reply"), + CreateAt: replyPostTime, + Replies: &[]ReplyImportData{{ + User: ptrStr(th.BasicUser2.Username), + Message: ptrStr("Message reply"), + CreateAt: replyTime, + }}, + }, }, - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message with reply"), - CreateAt: replyPostTime, - Replies: &[]ReplyImportData{{ - User: ptrStr(th.BasicUser2.Username), - Message: ptrStr("Message reply"), - CreateAt: replyTime, - }}, + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.Nil(t, err, "Expected success.") + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 8, "") // Check the post values. - posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(groupChannel.Id, *data.CreateAt) + posts, err := th.App.Srv().Store.Post().GetPostsCreatedAt(groupChannel.Id, *data.DirectPost.CreateAt) require.Nil(t, err) require.Len(t, posts, 1, "Unexpected number of posts found.") post := posts[0] - postBool := post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != th.BasicUser.Id + postBool := post.Message != *data.DirectPost.Message || post.CreateAt != *data.DirectPost.CreateAt || post.UserId != th.BasicUser.Id require.False(t, postBool, "Post properties not as expected") // Check the reply values. @@ -3357,7 +3687,7 @@ func TestImportImportDirectPost(t *testing.T) { require.Len(t, replies, 1, "Unexpected number of posts found.") reply := replies[0] - replyBool := reply.Message != *(*data.Replies)[0].Message || reply.CreateAt != *(*data.Replies)[0].CreateAt || reply.UserId != th.BasicUser2.Id + replyBool := reply.Message != *(*data.DirectPost.Replies)[0].Message || reply.CreateAt != *(*data.DirectPost.Replies)[0].CreateAt || reply.UserId != th.BasicUser2.Id require.False(t, replyBool, "Post properties not as expected") require.Equal(t, post.Id, reply.RootId, "Unexpected reply RootId") @@ -3366,23 +3696,29 @@ func TestImportImportDirectPost(t *testing.T) { t.Run("Update post with replies", func(t *testing.T) { replyPostTime := ptrInt64(initialDate + 25) replyTime := ptrInt64(initialDate + 26) - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, - user3.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + user3.Username, + }, + User: ptrStr(th.BasicUser2.Username), + Message: ptrStr("Message with reply"), + CreateAt: replyPostTime, + Replies: &[]ReplyImportData{{ + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message reply"), + CreateAt: replyTime, + }}, + }, }, - User: ptrStr(th.BasicUser2.Username), - Message: ptrStr("Message with reply"), - CreateAt: replyPostTime, - Replies: &[]ReplyImportData{{ - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message reply"), - CreateAt: replyTime, - }}, + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.Nil(t, err, "Expected success.") + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 8, "") }) @@ -3390,23 +3726,29 @@ func TestImportImportDirectPost(t *testing.T) { t.Run("Create new post with replies based on the previous one", func(t *testing.T) { replyPostTime := ptrInt64(initialDate + 27) replyTime := ptrInt64(initialDate + 28) - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, - user3.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + user3.Username, + }, + User: ptrStr(th.BasicUser2.Username), + Message: ptrStr("Message with reply 2"), + CreateAt: replyPostTime, + Replies: &[]ReplyImportData{{ + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message reply"), + CreateAt: replyTime, + }}, + }, }, - User: ptrStr(th.BasicUser2.Username), - Message: ptrStr("Message with reply 2"), - CreateAt: replyPostTime, - Replies: &[]ReplyImportData{{ - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message reply"), - CreateAt: replyTime, - }}, + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.Nil(t, err, "Expected success.") + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 10, "") }) @@ -3414,23 +3756,29 @@ func TestImportImportDirectPost(t *testing.T) { t.Run("Create new reply for existing post with replies", func(t *testing.T) { replyPostTime := ptrInt64(initialDate + 25) replyTime := ptrInt64(initialDate + 29) - data := &DirectPostImportData{ - ChannelMembers: &[]string{ - th.BasicUser.Username, - th.BasicUser2.Username, - user3.Username, + data := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + th.BasicUser.Username, + th.BasicUser2.Username, + user3.Username, + }, + User: ptrStr(th.BasicUser2.Username), + Message: ptrStr("Message with reply"), + CreateAt: replyPostTime, + Replies: &[]ReplyImportData{{ + User: ptrStr(th.BasicUser.Username), + Message: ptrStr("Message reply 2"), + CreateAt: replyTime, + }}, + }, }, - User: ptrStr(th.BasicUser2.Username), - Message: ptrStr("Message with reply"), - CreateAt: replyPostTime, - Replies: &[]ReplyImportData{{ - User: ptrStr(th.BasicUser.Username), - Message: ptrStr("Message reply 2"), - CreateAt: replyTime, - }}, + 1, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{data}, false) require.Nil(t, err, "Expected success.") + require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 11, "") }) @@ -3566,24 +3914,30 @@ func TestImportPostAndRepliesWithAttachments(t *testing.T) { testsDir, _ := fileutils.FindDir("tests") testImage := filepath.Join(testsDir, "test.png") testMarkDown := filepath.Join(testsDir, "test-attachments.md") - data := &PostImportData{ - Team: &teamName, - Channel: &channelName, - User: &username3, - Message: ptrStr("Message with reply"), - CreateAt: &attachmentsPostTime, - Attachments: &[]AttachmentImportData{{Path: &testImage}, {Path: &testMarkDown}}, - Replies: &[]ReplyImportData{{ - User: &user4.Username, - Message: ptrStr("Message reply"), - CreateAt: &attachmentsReplyTime, - Attachments: &[]AttachmentImportData{{Path: &testImage}}, - }}, + data := LineImportWorkerData{ + LineImportData{ + Post: &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username3, + Message: ptrStr("Message with reply"), + CreateAt: &attachmentsPostTime, + Attachments: &[]AttachmentImportData{{Path: &testImage}, {Path: &testMarkDown}}, + Replies: &[]ReplyImportData{{ + User: &user4.Username, + Message: ptrStr("Message reply"), + CreateAt: &attachmentsReplyTime, + Attachments: &[]AttachmentImportData{{Path: &testImage}}, + }}, + }, + }, + 19, } t.Run("import with attachment", func(t *testing.T) { - err := th.App.importMultiplePosts([]*PostImportData{data}, false) + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) require.Nil(t, err) + require.Equal(t, 0, errLine) attachments := GetAttachments(user3.Id, th, t) require.Len(t, attachments, 2) @@ -3598,9 +3952,10 @@ func TestImportPostAndRepliesWithAttachments(t *testing.T) { }) t.Run("import existing post with new attachment", func(t *testing.T) { - data.Attachments = &[]AttachmentImportData{{Path: &testImage}} - err := th.App.importMultiplePosts([]*PostImportData{data}, false) + data.Post.Attachments = &[]AttachmentImportData{{Path: &testImage}} + errLine, err := th.App.importMultiplePostLines([]LineImportWorkerData{data}, false) require.Nil(t, err) + require.Equal(t, 0, errLine) attachments := GetAttachments(user3.Id, th, t) require.Len(t, attachments, 1) @@ -3614,24 +3969,30 @@ func TestImportPostAndRepliesWithAttachments(t *testing.T) { }) t.Run("Reply with Attachments in Direct Pos", func(t *testing.T) { - directImportData := &DirectPostImportData{ - ChannelMembers: &[]string{ - user3.Username, - user2.Username, + directImportData := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + user3.Username, + user2.Username, + }, + User: &user3.Username, + Message: ptrStr("Message with Replies"), + CreateAt: ptrInt64(model.GetMillis()), + Replies: &[]ReplyImportData{{ + User: &user2.Username, + Message: ptrStr("Message reply with attachment"), + CreateAt: ptrInt64(model.GetMillis()), + Attachments: &[]AttachmentImportData{{Path: &testImage}}, + }}, + }, }, - User: &user3.Username, - Message: ptrStr("Message with Replies"), - CreateAt: ptrInt64(model.GetMillis()), - Replies: &[]ReplyImportData{{ - User: &user2.Username, - Message: ptrStr("Message reply with attachment"), - CreateAt: ptrInt64(model.GetMillis()), - Attachments: &[]AttachmentImportData{{Path: &testImage}}, - }}, + 7, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{directImportData}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{directImportData}, false) require.Nil(t, err, "Expected success.") + require.Equal(t, 0, errLine) attachments := GetAttachments(user2.Id, th, t) require.Len(t, attachments, 1) @@ -3641,7 +4002,6 @@ func TestImportPostAndRepliesWithAttachments(t *testing.T) { } func TestImportDirectPostWithAttachments(t *testing.T) { - th := Setup(t) defer th.TearDown() @@ -3674,20 +4034,26 @@ func TestImportDirectPostWithAttachments(t *testing.T) { user2, appErr := th.App.GetUserByUsername(username2) require.Nil(t, appErr, "Failed to get user2 from database.") - directImportData := &DirectPostImportData{ - ChannelMembers: &[]string{ - user1.Username, - user2.Username, + directImportData := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + user1.Username, + user2.Username, + }, + User: &user1.Username, + Message: ptrStr("Direct message"), + CreateAt: ptrInt64(model.GetMillis()), + Attachments: &[]AttachmentImportData{{Path: &testImage}}, + }, }, - User: &user1.Username, - Message: ptrStr("Direct message"), - CreateAt: ptrInt64(model.GetMillis()), - Attachments: &[]AttachmentImportData{{Path: &testImage}}, + 3, } t.Run("Regular import of attachment", func(t *testing.T) { - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{directImportData}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{directImportData}, false) require.Nil(t, err, "Expected success.") + require.Equal(t, 0, errLine) attachments := GetAttachments(user1.Id, th, t) require.Len(t, attachments, 1) @@ -3696,46 +4062,59 @@ func TestImportDirectPostWithAttachments(t *testing.T) { }) t.Run("Attempt to import again with same file entirely, should NOT add an attachment", func(t *testing.T) { - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{directImportData}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{directImportData}, false) require.Nil(t, err, "Expected success.") + require.Equal(t, 0, errLine) attachments := GetAttachments(user1.Id, th, t) require.Len(t, attachments, 1) }) t.Run("Attempt to import again with same name and size but different content, SHOULD add an attachment", func(t *testing.T) { - directImportDataFake := &DirectPostImportData{ - ChannelMembers: &[]string{ - user1.Username, - user2.Username, + directImportDataFake := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + user1.Username, + user2.Username, + }, + User: &user1.Username, + Message: ptrStr("Direct message"), + CreateAt: ptrInt64(model.GetMillis()), + Attachments: &[]AttachmentImportData{{Path: &testImageFake}}, + }, }, - User: &user1.Username, - Message: ptrStr("Direct message"), - CreateAt: ptrInt64(model.GetMillis()), - Attachments: &[]AttachmentImportData{{Path: &testImageFake}}, + 2, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{directImportDataFake}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{directImportDataFake}, false) require.Nil(t, err, "Expected success.") + require.Equal(t, 0, errLine) attachments := GetAttachments(user1.Id, th, t) require.Len(t, attachments, 2) }) t.Run("Attempt to import again with same data, SHOULD add an attachment, since it's different name", func(t *testing.T) { - directImportData2 := &DirectPostImportData{ - ChannelMembers: &[]string{ - user1.Username, - user2.Username, + directImportData2 := LineImportWorkerData{ + LineImportData{ + DirectPost: &DirectPostImportData{ + ChannelMembers: &[]string{ + user1.Username, + user2.Username, + }, + User: &user1.Username, + Message: ptrStr("Direct message"), + CreateAt: ptrInt64(model.GetMillis()), + Attachments: &[]AttachmentImportData{{Path: &testImage2}}, + }, }, - User: &user1.Username, - Message: ptrStr("Direct message"), - CreateAt: ptrInt64(model.GetMillis()), - Attachments: &[]AttachmentImportData{{Path: &testImage2}}, + 2, } - err := th.App.importMultipleDirectPosts([]*DirectPostImportData{directImportData2}, false) + errLine, err := th.App.importMultipleDirectPostLines([]LineImportWorkerData{directImportData2}, false) require.Nil(t, err, "Expected success.") + require.Equal(t, 0, errLine) attachments := GetAttachments(user1.Id, th, t) require.Len(t, attachments, 3) diff --git a/store/opentracing_layer.go b/store/opentracing_layer.go index 5434d92d4b..8d8427218f 100644 --- a/store/opentracing_layer.go +++ b/store/opentracing_layer.go @@ -4932,7 +4932,7 @@ func (s *OpenTracingLayerPostStore) Overwrite(post *model.Post) (*model.Post, *m return resultVar0, resultVar1 } -func (s *OpenTracingLayerPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) { +func (s *OpenTracingLayerPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) { origCtx := s.Root.Store.Context() span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "PostStore.OverwriteMultiple") s.Root.Store.SetContext(newCtx) @@ -4941,13 +4941,13 @@ func (s *OpenTracingLayerPostStore) OverwriteMultiple(posts []*model.Post) ([]*m }() defer span.Finish() - resultVar0, resultVar1 := s.PostStore.OverwriteMultiple(posts) - if resultVar1 != nil { - span.LogFields(spanlog.Error(resultVar1)) + resultVar0, resultVar1, resultVar2 := s.PostStore.OverwriteMultiple(posts) + if resultVar2 != nil { + span.LogFields(spanlog.Error(resultVar2)) ext.Error.Set(span, true) } - return resultVar0, resultVar1 + return resultVar0, resultVar1, resultVar2 } func (s *OpenTracingLayerPostStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, *model.AppError) { @@ -5022,7 +5022,7 @@ func (s *OpenTracingLayerPostStore) Save(post *model.Post) (*model.Post, *model. return resultVar0, resultVar1 } -func (s *OpenTracingLayerPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) { +func (s *OpenTracingLayerPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) { origCtx := s.Root.Store.Context() span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "PostStore.SaveMultiple") s.Root.Store.SetContext(newCtx) @@ -5031,13 +5031,13 @@ func (s *OpenTracingLayerPostStore) SaveMultiple(posts []*model.Post) ([]*model. }() defer span.Finish() - resultVar0, resultVar1 := s.PostStore.SaveMultiple(posts) - if resultVar1 != nil { - span.LogFields(spanlog.Error(resultVar1)) + resultVar0, resultVar1, resultVar2 := s.PostStore.SaveMultiple(posts) + if resultVar2 != nil { + span.LogFields(spanlog.Error(resultVar2)) ext.Error.Set(span, true) } - return resultVar0, resultVar1 + return resultVar0, resultVar1, resultVar2 } func (s *OpenTracingLayerPostStore) Search(teamId string, userId string, params *model.SearchParams) (*model.PostList, *model.AppError) { diff --git a/store/sqlstore/post_store.go b/store/sqlstore/post_store.go index 2cb9a863ca..3ef45c76e9 100644 --- a/store/sqlstore/post_store.go +++ b/store/sqlstore/post_store.go @@ -99,19 +99,19 @@ func (s *SqlPostStore) createIndexesIfNotExists() { s.CreateFullTextIndexIfNotExists("idx_posts_hashtags_txt", "Posts", "Hashtags") } -func (s *SqlPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) { +func (s *SqlPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) { channelNewPosts := make(map[string]int) maxDateNewPosts := make(map[string]int64) rootIds := make(map[string]int) maxDateRootIds := make(map[string]int64) - for _, post := range posts { + for idx, post := range posts { if len(post.Id) > 0 { - return nil, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.existing.app_error", nil, "id="+post.Id, http.StatusBadRequest) + return nil, idx, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.existing.app_error", nil, "id="+post.Id, http.StatusBadRequest) } post.PreSave() maxPostSize := s.GetMaxPostSize() if err := post.IsValid(maxPostSize); err != nil { - return nil, err + return nil, idx, err } currentChannelCount, ok := channelNewPosts[post.ChannelId] @@ -153,11 +153,11 @@ func (s *SqlPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model. } sql, args, err := query.ToSql() if err != nil { - return nil, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.app_error", nil, err.Error(), http.StatusInternalServerError) + return nil, -1, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.app_error", nil, err.Error(), http.StatusInternalServerError) } if _, err := s.GetMaster().Exec(sql, args...); err != nil { - return nil, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.app_error", nil, err.Error(), http.StatusInternalServerError) + return nil, -1, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.app_error", nil, err.Error(), http.StatusInternalServerError) } for channelId, count := range channelNewPosts { @@ -190,11 +190,11 @@ func (s *SqlPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model. } } - return posts, nil + return posts, -1, nil } func (s *SqlPostStore) Save(post *model.Post) (*model.Post, *model.AppError) { - posts, err := s.SaveMultiple([]*model.Post{post}) + posts, _, err := s.SaveMultiple([]*model.Post{post}) if err != nil { return nil, err } @@ -270,40 +270,40 @@ func (s *SqlPostStore) Update(newPost *model.Post, oldPost *model.Post) (*model. return newPost, nil } -func (s *SqlPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) { +func (s *SqlPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) { updateAt := model.GetMillis() maxPostSize := s.GetMaxPostSize() - for _, post := range posts { + for idx, post := range posts { post.UpdateAt = updateAt if appErr := post.IsValid(maxPostSize); appErr != nil { - return nil, appErr + return nil, idx, appErr } } tx, err := s.GetMaster().Begin() if err != nil { - return nil, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, err.Error(), http.StatusInternalServerError) + return nil, -1, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, err.Error(), http.StatusInternalServerError) } - for _, post := range posts { + for idx, post := range posts { if _, err = tx.Update(post); err != nil { txErr := tx.Rollback() if txErr != nil { - return nil, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, txErr.Error(), http.StatusInternalServerError) + return nil, idx, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, txErr.Error(), http.StatusInternalServerError) } - return nil, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, "id="+post.Id+", "+err.Error(), http.StatusInternalServerError) + return nil, idx, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, "id="+post.Id+", "+err.Error(), http.StatusInternalServerError) } } err = tx.Commit() if err != nil { - return nil, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, err.Error(), http.StatusInternalServerError) + return nil, -1, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, err.Error(), http.StatusInternalServerError) } - return posts, nil + return posts, -1, nil } func (s *SqlPostStore) Overwrite(post *model.Post) (*model.Post, *model.AppError) { - posts, err := s.OverwriteMultiple([]*model.Post{post}) + posts, _, err := s.OverwriteMultiple([]*model.Post{post}) if err != nil { return nil, err } diff --git a/store/store.go b/store/store.go index f080a6dfe8..c29c924ee2 100644 --- a/store/store.go +++ b/store/store.go @@ -232,7 +232,7 @@ type ChannelMemberHistoryStore interface { } type PostStore interface { - SaveMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) + SaveMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) Save(post *model.Post) (*model.Post, *model.AppError) Update(newPost *model.Post, oldPost *model.Post) (*model.Post, *model.AppError) Get(id string, skipFetchThreads bool) (*model.PostList, *model.AppError) @@ -260,7 +260,7 @@ type PostStore interface { InvalidateLastPostTimeCache(channelId string) GetPostsCreatedAt(channelId string, time int64) ([]*model.Post, *model.AppError) Overwrite(post *model.Post) (*model.Post, *model.AppError) - OverwriteMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) + OverwriteMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) GetPostsByIds(postIds []string) ([]*model.Post, *model.AppError) GetPostsBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.PostForIndexing, *model.AppError) PermanentDeleteBatch(endTime int64, limit int64) (int64, *model.AppError) diff --git a/store/storetest/mocks/PostStore.go b/store/storetest/mocks/PostStore.go index 2a6a06e8a7..39ad2e446d 100644 --- a/store/storetest/mocks/PostStore.go +++ b/store/storetest/mocks/PostStore.go @@ -638,7 +638,7 @@ func (_m *PostStore) Overwrite(post *model.Post) (*model.Post, *model.AppError) } // OverwriteMultiple provides a mock function with given fields: posts -func (_m *PostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) { +func (_m *PostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) { ret := _m.Called(posts) var r0 []*model.Post @@ -650,16 +650,23 @@ func (_m *PostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, *mod } } - var r1 *model.AppError - if rf, ok := ret.Get(1).(func([]*model.Post) *model.AppError); ok { + var r1 int + if rf, ok := ret.Get(1).(func([]*model.Post) int); ok { r1 = rf(posts) } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(*model.AppError) + r1 = ret.Get(1).(int) + } + + var r2 *model.AppError + if rf, ok := ret.Get(2).(func([]*model.Post) *model.AppError); ok { + r2 = rf(posts) + } else { + if ret.Get(2) != nil { + r2 = ret.Get(2).(*model.AppError) } } - return r0, r1 + return r0, r1, r2 } // PermanentDeleteBatch provides a mock function with given fields: endTime, limit @@ -743,7 +750,7 @@ func (_m *PostStore) Save(post *model.Post) (*model.Post, *model.AppError) { } // SaveMultiple provides a mock function with given fields: posts -func (_m *PostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) { +func (_m *PostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) { ret := _m.Called(posts) var r0 []*model.Post @@ -755,16 +762,23 @@ func (_m *PostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.Ap } } - var r1 *model.AppError - if rf, ok := ret.Get(1).(func([]*model.Post) *model.AppError); ok { + var r1 int + if rf, ok := ret.Get(1).(func([]*model.Post) int); ok { r1 = rf(posts) } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(*model.AppError) + r1 = ret.Get(1).(int) + } + + var r2 *model.AppError + if rf, ok := ret.Get(2).(func([]*model.Post) *model.AppError); ok { + r2 = rf(posts) + } else { + if ret.Get(2) != nil { + r2 = ret.Get(2).(*model.AppError) } } - return r0, r1 + return r0, r1, r2 } // Search provides a mock function with given fields: teamId, userId, params diff --git a/store/storetest/post_store.go b/store/storetest/post_store.go index 379c4de516..4f56590f2b 100644 --- a/store/storetest/post_store.go +++ b/store/storetest/post_store.go @@ -218,8 +218,9 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) { p4.Message = "zz" + model.NewId() + "b" t.Run("Save correctly a new set of posts", func(t *testing.T) { - newPosts, err := ss.Post().SaveMultiple([]*model.Post{&p1, &p2, &p3}) + newPosts, errIdx, err := ss.Post().SaveMultiple([]*model.Post{&p1, &p2, &p3}) require.Nil(t, err) + require.Equal(t, -1, errIdx) for _, post := range newPosts { storedPost, err := ss.Post().GetSingle(post.Id) assert.Nil(t, err) @@ -253,8 +254,9 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) { o4.UserId = model.NewId() o4.Message = "zz" + model.NewId() + "b" - newPosts, err := ss.Post().SaveMultiple([]*model.Post{&o1, &o2, &o3, &o4}) + newPosts, errIdx, err := ss.Post().SaveMultiple([]*model.Post{&o1, &o2, &o3, &o4}) require.Nil(t, err, "couldn't save item") + require.Equal(t, -1, errIdx) assert.Len(t, newPosts, 4) assert.Equal(t, int64(2), newPosts[0].ReplyCount) assert.Equal(t, int64(2), newPosts[1].ReplyCount) @@ -263,8 +265,9 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) { }) t.Run("Try to save mixed, already saved and not saved posts", func(t *testing.T) { - newPosts, err := ss.Post().SaveMultiple([]*model.Post{&p4, &p3}) + newPosts, errIdx, err := ss.Post().SaveMultiple([]*model.Post{&p4, &p3}) require.NotNil(t, err) + require.Equal(t, 1, errIdx) require.Nil(t, newPosts) storedPost, err := ss.Post().GetSingle(p3.Id) assert.Nil(t, err) @@ -289,7 +292,7 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) { replyPost.Message = "zz" + model.NewId() + "b" replyPost.RootId = rootPost.Id - _, err := ss.Post().SaveMultiple([]*model.Post{&rootPost, &replyPost}) + _, _, err := ss.Post().SaveMultiple([]*model.Post{&rootPost, &replyPost}) require.Nil(t, err) rrootPost, err := ss.Post().GetSingle(rootPost.Id) @@ -308,7 +311,7 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) { replyPost3.Message = "zz" + model.NewId() + "b" replyPost3.RootId = rootPost.Id - _, err = ss.Post().SaveMultiple([]*model.Post{&replyPost2, &replyPost3}) + _, _, err = ss.Post().SaveMultiple([]*model.Post{&replyPost2, &replyPost3}) require.Nil(t, err) rrootPost2, err := ss.Post().GetSingle(rootPost.Id) @@ -341,7 +344,7 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) { post3.UserId = model.NewId() post3.Message = "zz" + model.NewId() + "b" - _, err = ss.Post().SaveMultiple([]*model.Post{&post1, &post2, &post3}) + _, _, err = ss.Post().SaveMultiple([]*model.Post{&post1, &post2, &post3}) require.Nil(t, err) rchannel, err := ss.Channel().Get(channel.Id, false) @@ -2143,8 +2146,9 @@ func testPostStoreOverwriteMultiple(t *testing.T, ss store.Store) { o3a := ro3.Clone() o3a.Message = ro3.Message + "WWWWWWW" - _, err = ss.Post().OverwriteMultiple([]*model.Post{o1a, o2a, o3a}) + _, errIdx, err := ss.Post().OverwriteMultiple([]*model.Post{o1a, o2a, o3a}) require.Nil(t, err) + require.Equal(t, -1, errIdx) r1, err = ss.Post().Get(o1.Id, false) require.Nil(t, err) @@ -2172,8 +2176,9 @@ func testPostStoreOverwriteMultiple(t *testing.T, ss store.Store) { o5a.Filenames = []string{} o5a.FileIds = []string{} - _, err = ss.Post().OverwriteMultiple([]*model.Post{o4a, o5a}) + _, errIdx, err := ss.Post().OverwriteMultiple([]*model.Post{o4a, o5a}) require.Nil(t, err) + require.Equal(t, -1, errIdx) r4, err = ss.Post().Get(o4.Id, false) require.Nil(t, err) diff --git a/store/timer_layer.go b/store/timer_layer.go index 84314e1b66..8864cd30ce 100644 --- a/store/timer_layer.go +++ b/store/timer_layer.go @@ -4476,20 +4476,20 @@ func (s *TimerLayerPostStore) Overwrite(post *model.Post) (*model.Post, *model.A return resultVar0, resultVar1 } -func (s *TimerLayerPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) { +func (s *TimerLayerPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) { start := timemodule.Now() - resultVar0, resultVar1 := s.PostStore.OverwriteMultiple(posts) + resultVar0, resultVar1, resultVar2 := s.PostStore.OverwriteMultiple(posts) elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second) if s.Root.Metrics != nil { success := "false" - if resultVar1 == nil { + if resultVar2 == nil { success = "true" } s.Root.Metrics.ObserveStoreMethodDuration("PostStore.OverwriteMultiple", success, elapsed) } - return resultVar0, resultVar1 + return resultVar0, resultVar1, resultVar2 } func (s *TimerLayerPostStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, *model.AppError) { @@ -4556,20 +4556,20 @@ func (s *TimerLayerPostStore) Save(post *model.Post) (*model.Post, *model.AppErr return resultVar0, resultVar1 } -func (s *TimerLayerPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) { +func (s *TimerLayerPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) { start := timemodule.Now() - resultVar0, resultVar1 := s.PostStore.SaveMultiple(posts) + resultVar0, resultVar1, resultVar2 := s.PostStore.SaveMultiple(posts) elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second) if s.Root.Metrics != nil { success := "false" - if resultVar1 == nil { + if resultVar2 == nil { success = "true" } s.Root.Metrics.ObserveStoreMethodDuration("PostStore.SaveMultiple", success, elapsed) } - return resultVar0, resultVar1 + return resultVar0, resultVar1, resultVar2 } func (s *TimerLayerPostStore) Search(teamId string, userId string, params *model.SearchParams) (*model.PostList, *model.AppError) {