mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

[MM-25714] Keeps track of the import lines of the posts while importing to report the right line on error (#14752)

Просмотр исходного кода 
* [MM-25714] Keeps track of the import lines of the posts while importing to report the right line on error

* Adding review comments

* Reverse the order of the error and error line params
Этот коммит содержится в:
Miguel de la Cruz
2020-06-08 12:12:07 +02:00
коммит произвёл GitHub
родитель ea06c291df
Коммит 6cd898fab7
9 изменённых файлов: 1220 добавлений и 776 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

36
app/import.go
Просмотреть файл

@@ -30,31 +30,31 @@ func stopOnError(err LineImportWorkerError) bool {
}
func (a *App) bulkImportWorker(dryRun bool, wg *sync.WaitGroup, lines <-chan LineImportWorkerData, errors chan<- LineImportWorkerError) {
posts := []*PostImportData{}
directPosts := []*DirectPostImportData{}
postLines := []LineImportWorkerData{}
directPostLines := []LineImportWorkerData{}
for line := range lines {
switch {
case line.LineImportData.Type == "post":
posts = append(posts, line.Post)
postLines = append(postLines, line)
if line.Post == nil {
errors <- LineImportWorkerError{model.NewAppError("BulkImport", "app.import.import_line.null_post.error", nil, "", http.StatusBadRequest), line.LineNumber}
}
if len(posts) >= importMultiplePostsThreshold {
if err := a.importMultiplePosts(posts, dryRun); err != nil {
errors <- LineImportWorkerError{err, line.LineNumber}
if len(postLines) >= importMultiplePostsThreshold {
if errLine, err := a.importMultiplePostLines(postLines, dryRun); err != nil {
errors <- LineImportWorkerError{err, errLine}
}
posts = []*PostImportData{}
postLines = []LineImportWorkerData{}
}
case line.LineImportData.Type == "direct_post":
directPosts = append(directPosts, line.DirectPost)
directPostLines = append(directPostLines, line)
if line.DirectPost == nil {
errors <- LineImportWorkerError{model.NewAppError("BulkImport", "app.import.import_line.null_direct_post.error", nil, "", http.StatusBadRequest), line.LineNumber}
}
if len(directPosts) >= importMultiplePostsThreshold {
if err := a.importMultipleDirectPosts(directPosts, dryRun); err != nil {
errors <- LineImportWorkerError{err, line.LineNumber}
if len(directPostLines) >= importMultiplePostsThreshold {
if errLine, err := a.importMultipleDirectPostLines(directPostLines, dryRun); err != nil {
errors <- LineImportWorkerError{err, errLine}
}
directPosts = []*DirectPostImportData{}
directPostLines = []LineImportWorkerData{}
}
default:
if err := a.importLine(line.LineImportData, dryRun); err != nil {
@@ -63,14 +63,14 @@ func (a *App) bulkImportWorker(dryRun bool, wg *sync.WaitGroup, lines <-chan Lin
}
}
if len(posts) > 0 {
if err := a.importMultiplePosts(posts, dryRun); err != nil {
errors <- LineImportWorkerError{err, 0}
if len(postLines) > 0 {
if errLine, err := a.importMultiplePostLines(postLines, dryRun); err != nil {
errors <- LineImportWorkerError{err, errLine}
}
}
if len(directPosts) > 0 {
if err := a.importMultipleDirectPosts(directPosts, dryRun); err != nil {
errors <- LineImportWorkerError{err, 0}
if len(directPostLines) > 0 {
if errLine, err := a.importMultipleDirectPostLines(directPostLines, dryRun); err != nil {
errors <- LineImportWorkerError{err, errLine}
}
}
wg.Done()

200
app/import_functions.go
Просмотреть файл

@@ -7,6 +7,7 @@ import (
"bytes"
"crypto/sha1"
"errors"
"fmt"
"io"
"net/http"
"os"
@@ -1030,12 +1031,12 @@ func (a *App) importReplies(data []ReplyImportData, post *model.Post, teamId str
}
if len(postsForCreateList) > 0 {
if _, err := a.Srv().Store.Post().SaveMultiple(postsForCreateList); err != nil {
if _, _, err := a.Srv().Store.Post().SaveMultiple(postsForCreateList); err != nil {
return err
}
}
if _, err := a.Srv().Store.Post().OverwriteMultiple(postsForOverwriteList); err != nil {
if _, _, err := a.Srv().Store.Post().OverwriteMultiple(postsForOverwriteList); err != nil {
return err
}
@@ -1097,6 +1098,7 @@ type postAndData struct {
directPostData *DirectPostImportData
replyData *ReplyImportData
team *model.Team
lineNumber int
}
func (a *App) getUsersByUsernames(usernames []string) (map[string]*model.User, *model.AppError) {
@@ -1159,64 +1161,76 @@ func (a *App) getChannelsForPosts(teams map[string]*model.Team, data []*PostImpo
return channels, nil
}
func (a *App) importMultiplePosts(data []*PostImportData, dryRun bool) *model.AppError {
if len(data) == 0 {
return nil
// getPostStrID returns a string ID composed of several post fields to
// uniquely identify a post before it's imported, so it has no ID yet
func getPostStrID(post *model.Post) string {
return fmt.Sprintf("%d%s%s", post.CreateAt, post.ChannelId, post.Message)
}
// importMultiplePostLines will return an error and the line that
// caused it whenever possible
func (a *App) importMultiplePostLines(lines []LineImportWorkerData, dryRun bool) (int, *model.AppError) {
if len(lines) == 0 {
return 0, nil
}
for _, postData := range data {
if err := validatePostImportData(postData, a.MaxPostSize()); err != nil {
return err
for _, line := range lines {
if err := validatePostImportData(line.Post, a.MaxPostSize()); err != nil {
return line.LineNumber, err
}
}
// If this is a Dry Run, do not continue any further.
if dryRun {
return nil
return 0, nil
}
usernames := []string{}
teamNames := []string{}
for _, postData := range data {
usernames = append(usernames, *postData.User)
if postData.FlaggedBy != nil {
usernames = append(usernames, *postData.FlaggedBy...)
teamNames := make([]string, len(lines))
postsData := make([]*PostImportData, len(lines))
for i, line := range lines {
usernames = append(usernames, *line.Post.User)
if line.Post.FlaggedBy != nil {
usernames = append(usernames, *line.Post.FlaggedBy...)
}
teamNames = append(teamNames, *postData.Team)
teamNames[i] = *line.Post.Team
postsData[i] = line.Post
}
users, err := a.getUsersByUsernames(usernames)
if err != nil {
return err
return 0, err
}
teams, err := a.getTeamsByNames(teamNames)
if err != nil {
return err
return 0, err
}
channels, err := a.getChannelsForPosts(teams, data)
channels, err := a.getChannelsForPosts(teams, postsData)
if err != nil {
return err
return 0, err
}
postsWithData := []postAndData{}
postsForCreateList := []*model.Post{}
postsForCreateMap := map[string]int{}
postsForOverwriteList := []*model.Post{}
postsForOverwriteMap := map[string]int{}
for _, postData := range data {
team := teams[*postData.Team]
channel := channels[*postData.Channel]
user := users[*postData.User]
for _, line := range lines {
team := teams[*line.Post.Team]
channel := channels[*line.Post.Channel]
user := users[*line.Post.User]
// Check if this post already exists.
posts, err := a.Srv().Store.Post().GetPostsCreatedAt(channel.Id, *postData.CreateAt)
posts, err := a.Srv().Store.Post().GetPostsCreatedAt(channel.Id, *line.Post.CreateAt)
if err != nil {
return err
return line.LineNumber, err
}
var post *model.Post
for _, p := range posts {
if p.Message == *postData.Message {
if p.Message == *line.Post.Message {
post = p
break
}
@@ -1227,18 +1241,18 @@ func (a *App) importMultiplePosts(data []*PostImportData, dryRun bool) *model.Ap
}
post.ChannelId = channel.Id
post.Message = *postData.Message
post.Message = *line.Post.Message
post.UserId = user.Id
post.CreateAt = *postData.CreateAt
post.CreateAt = *line.Post.CreateAt
post.Hashtags, _ = model.ParseHashtags(post.Message)
if postData.Props != nil {
post.Props = *postData.Props
if line.Post.Props != nil {
post.Props = *line.Post.Props
}
fileIds, err := a.uploadAttachments(postData.Attachments, post, team.Id, dryRun)
fileIds, err := a.uploadAttachments(line.Post.Attachments, post, team.Id, dryRun)
if err != nil {
return err
return line.LineNumber, err
}
for _, fileID := range post.FileIds {
if _, ok := fileIds[fileID]; !ok {
@@ -1252,20 +1266,34 @@ func (a *App) importMultiplePosts(data []*PostImportData, dryRun bool) *model.Ap
if len(post.Id) == 0 {
postsForCreateList = append(postsForCreateList, post)
postsForCreateMap[getPostStrID(post)] = line.LineNumber
} else {
postsForOverwriteList = append(postsForOverwriteList, post)
postsForOverwriteMap[getPostStrID(post)] = line.LineNumber
}
postsWithData = append(postsWithData, postAndData{post: post, postData: postData, team: team})
postsWithData = append(postsWithData, postAndData{post: post, postData: line.Post, team: team, lineNumber: line.LineNumber})
}
if len(postsForCreateList) > 0 {
if _, err := a.Srv().Store.Post().SaveMultiple(postsForCreateList); err != nil {
return err
if _, idx, err := a.Srv().Store.Post().SaveMultiple(postsForCreateList); err != nil {
if idx != -1 && idx < len(postsForCreateList) {
post := postsForCreateList[idx]
if lineNumber, ok := postsForCreateMap[getPostStrID(post)]; ok {
return lineNumber, err
}
}
return 0, err
}
}
if _, err := a.Srv().Store.Post().OverwriteMultiple(postsForOverwriteList); err != nil {
return err
if _, idx, err := a.Srv().Store.Post().OverwriteMultiple(postsForOverwriteList); err != nil {
if idx != -1 && idx < len(postsForOverwriteList) {
post := postsForOverwriteList[idx]
if lineNumber, ok := postsForOverwriteMap[getPostStrID(post)]; ok {
return lineNumber, err
}
}
return 0, err
}
for _, postWithData := range postsWithData {
@@ -1286,7 +1314,7 @@ func (a *App) importMultiplePosts(data []*PostImportData, dryRun bool) *model.Ap
if len(preferences) > 0 {
if err := a.Srv().Store.Preference().Save(&preferences); err != nil {
return model.NewAppError("BulkImport", "app.import.import_post.save_preferences.error", nil, err.Error(), http.StatusInternalServerError)
return postWithData.lineNumber, model.NewAppError("BulkImport", "app.import.import_post.save_preferences.error", nil, err.Error(), http.StatusInternalServerError)
}
}
}
@@ -1295,7 +1323,7 @@ func (a *App) importMultiplePosts(data []*PostImportData, dryRun bool) *model.Ap
for _, reaction := range *postWithData.postData.Reactions {
reaction := reaction
if err := a.importReaction(&reaction, postWithData.post, dryRun); err != nil {
return err
return postWithData.lineNumber, err
}
}
}
@@ -1303,12 +1331,12 @@ func (a *App) importMultiplePosts(data []*PostImportData, dryRun bool) *model.Ap
if postWithData.postData.Replies != nil && len(*postWithData.postData.Replies) > 0 {
err := a.importReplies(*postWithData.postData.Replies, postWithData.post, postWithData.team.Id, dryRun)
if err != nil {
return err
return postWithData.lineNumber, err
}
}
a.updateFileInfoWithPostId(postWithData.post)
}
return nil
return 0, nil
}
// uploadAttachments imports new attachments and returns current attachments of the post as a map
@@ -1408,44 +1436,48 @@ func (a *App) importDirectChannel(data *DirectChannelImportData, dryRun bool) *m
return nil
}
func (a *App) importMultipleDirectPosts(data []*DirectPostImportData, dryRun bool) *model.AppError {
if len(data) == 0 {
return nil
// importMultipleDirectPostLines will return an error and the line
// that caused it whenever possible
func (a *App) importMultipleDirectPostLines(lines []LineImportWorkerData, dryRun bool) (int, *model.AppError) {
if len(lines) == 0 {
return 0, nil
}
for _, postData := range data {
if err := validateDirectPostImportData(postData, a.MaxPostSize()); err != nil {
return err
for _, line := range lines {
if err := validateDirectPostImportData(line.DirectPost, a.MaxPostSize()); err != nil {
return line.LineNumber, err
}
}
// If this is a Dry Run, do not continue any further.
if dryRun {
return nil
return 0, nil
}
usernames := []string{}
for _, postData := range data {
usernames = append(usernames, *postData.User)
if postData.FlaggedBy != nil {
usernames = append(usernames, *postData.FlaggedBy...)
for _, line := range lines {
usernames = append(usernames, *line.DirectPost.User)
if line.DirectPost.FlaggedBy != nil {
usernames = append(usernames, *line.DirectPost.FlaggedBy...)
}
usernames = append(usernames, *postData.ChannelMembers...)
usernames = append(usernames, *line.DirectPost.ChannelMembers...)
}
users, err := a.getUsersByUsernames(usernames)
if err != nil {
return err
return 0, err
}
postsWithData := []postAndData{}
postsForCreateList := []*model.Post{}
postsForCreateMap := map[string]int{}
postsForOverwriteList := []*model.Post{}
postsForOverwriteMap := map[string]int{}
for _, postData := range data {
for _, line := range lines {
var userIds []string
var err *model.AppError
for _, username := range *postData.ChannelMembers {
for _, username := range *line.DirectPost.ChannelMembers {
user := users[username]
userIds = append(userIds, user.Id)
}
@@ -1455,28 +1487,28 @@ func (a *App) importMultipleDirectPosts(data []*DirectPostImportData, dryRun boo
if len(userIds) == 2 {
ch, err = a.GetOrCreateDirectChannel(userIds[0], userIds[1])
if err != nil && err.Id != store.CHANNEL_EXISTS_ERROR {
return model.NewAppError("BulkImport", "app.import.import_direct_post.create_direct_channel.error", nil, err.Error(), http.StatusBadRequest)
return line.LineNumber, model.NewAppError("BulkImport", "app.import.import_direct_post.create_direct_channel.error", nil, err.Error(), http.StatusBadRequest)
}
channel = ch
} else {
ch, err = a.createGroupChannel(userIds, userIds[0])
if err != nil && err.Id != store.CHANNEL_EXISTS_ERROR {
return model.NewAppError("BulkImport", "app.import.import_direct_post.create_group_channel.error", nil, err.Error(), http.StatusBadRequest)
return line.LineNumber, model.NewAppError("BulkImport", "app.import.import_direct_post.create_group_channel.error", nil, err.Error(), http.StatusBadRequest)
}
channel = ch
}
user := users[*postData.User]
user := users[*line.DirectPost.User]
// Check if this post already exists.
posts, err := a.Srv().Store.Post().GetPostsCreatedAt(channel.Id, *postData.CreateAt)
posts, err := a.Srv().Store.Post().GetPostsCreatedAt(channel.Id, *line.DirectPost.CreateAt)
if err != nil {
return err
return line.LineNumber, err
}
var post *model.Post
for _, p := range posts {
if p.Message == *postData.Message {
if p.Message == *line.DirectPost.Message {
post = p
break
}
@@ -1487,18 +1519,18 @@ func (a *App) importMultipleDirectPosts(data []*DirectPostImportData, dryRun boo
}
post.ChannelId = channel.Id
post.Message = *postData.Message
post.Message = *line.DirectPost.Message
post.UserId = user.Id
post.CreateAt = *postData.CreateAt
post.CreateAt = *line.DirectPost.CreateAt
post.Hashtags, _ = model.ParseHashtags(post.Message)
if postData.Props != nil {
post.Props = *postData.Props
if line.DirectPost.Props != nil {
post.Props = *line.DirectPost.Props
}
fileIds, err := a.uploadAttachments(postData.Attachments, post, "noteam", dryRun)
fileIds, err := a.uploadAttachments(line.DirectPost.Attachments, post, "noteam", dryRun)
if err != nil {
return err
return line.LineNumber, err
}
for _, fileID := range post.FileIds {
if _, ok := fileIds[fileID]; !ok {
@@ -1512,19 +1544,33 @@ func (a *App) importMultipleDirectPosts(data []*DirectPostImportData, dryRun boo
if len(post.Id) == 0 {
postsForCreateList = append(postsForCreateList, post)
postsForCreateMap[getPostStrID(post)] = line.LineNumber
} else {
postsForOverwriteList = append(postsForOverwriteList, post)
postsForOverwriteMap[getPostStrID(post)] = line.LineNumber
}
postsWithData = append(postsWithData, postAndData{post: post, directPostData: postData})
postsWithData = append(postsWithData, postAndData{post: post, directPostData: line.DirectPost, lineNumber: line.LineNumber})
}
if len(postsForCreateList) > 0 {
if _, err := a.Srv().Store.Post().SaveMultiple(postsForCreateList); err != nil {
return err
if _, idx, err := a.Srv().Store.Post().SaveMultiple(postsForCreateList); err != nil {
if idx != -1 && idx < len(postsForCreateList) {
post := postsForCreateList[idx]
if lineNumber, ok := postsForCreateMap[getPostStrID(post)]; ok {
return lineNumber, err
}
}
return 0, err
}
}
if _, err := a.Srv().Store.Post().OverwriteMultiple(postsForOverwriteList); err != nil {
return err
if _, idx, err := a.Srv().Store.Post().OverwriteMultiple(postsForOverwriteList); err != nil {
if idx != -1 && idx < len(postsForOverwriteList) {
post := postsForOverwriteList[idx]
if lineNumber, ok := postsForOverwriteMap[getPostStrID(post)]; ok {
return lineNumber, err
}
}
return 0, err
}
for _, postWithData := range postsWithData {
@@ -1544,7 +1590,7 @@ func (a *App) importMultipleDirectPosts(data []*DirectPostImportData, dryRun boo
if len(preferences) > 0 {
if err := a.Srv().Store.Preference().Save(&preferences); err != nil {
return model.NewAppError("BulkImport", "app.import.import_post.save_preferences.error", nil, err.Error(), http.StatusInternalServerError)
return postWithData.lineNumber, model.NewAppError("BulkImport", "app.import.import_post.save_preferences.error", nil, err.Error(), http.StatusInternalServerError)
}
}
}
@@ -1553,20 +1599,20 @@ func (a *App) importMultipleDirectPosts(data []*DirectPostImportData, dryRun boo
for _, reaction := range *postWithData.directPostData.Reactions {
reaction := reaction
if err := a.importReaction(&reaction, postWithData.post, dryRun); err != nil {
return err
return postWithData.lineNumber, err
}
}
}
if postWithData.directPostData.Replies != nil {
if err := a.importReplies(*postWithData.directPostData.Replies, postWithData.post, "noteam", dryRun); err != nil {
return err
return postWithData.lineNumber, err
}
}
a.updateFileInfoWithPostId(postWithData.post)
}
return nil
return 0, nil
}
func (a *App) importEmoji(data *EmojiImportData, dryRun bool) *model.AppError {

1625
app/import_functions_test.go
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

20
store/opentracing_layer.go
Просмотреть файл

@@ -4932,7 +4932,7 @@ func (s *OpenTracingLayerPostStore) Overwrite(post *model.Post) (*model.Post, *m
return resultVar0, resultVar1
}
func (s *OpenTracingLayerPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) {
func (s *OpenTracingLayerPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "PostStore.OverwriteMultiple")
s.Root.Store.SetContext(newCtx)
@@ -4941,13 +4941,13 @@ func (s *OpenTracingLayerPostStore) OverwriteMultiple(posts []*model.Post) ([]*m
}()
defer span.Finish()
resultVar0, resultVar1 := s.PostStore.OverwriteMultiple(posts)
if resultVar1 != nil {
span.LogFields(spanlog.Error(resultVar1))
resultVar0, resultVar1, resultVar2 := s.PostStore.OverwriteMultiple(posts)
if resultVar2 != nil {
span.LogFields(spanlog.Error(resultVar2))
ext.Error.Set(span, true)
}
return resultVar0, resultVar1
return resultVar0, resultVar1, resultVar2
}
func (s *OpenTracingLayerPostStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, *model.AppError) {
@@ -5022,7 +5022,7 @@ func (s *OpenTracingLayerPostStore) Save(post *model.Post) (*model.Post, *model.
return resultVar0, resultVar1
}
func (s *OpenTracingLayerPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) {
func (s *OpenTracingLayerPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "PostStore.SaveMultiple")
s.Root.Store.SetContext(newCtx)
@@ -5031,13 +5031,13 @@ func (s *OpenTracingLayerPostStore) SaveMultiple(posts []*model.Post) ([]*model.
}()
defer span.Finish()
resultVar0, resultVar1 := s.PostStore.SaveMultiple(posts)
if resultVar1 != nil {
span.LogFields(spanlog.Error(resultVar1))
resultVar0, resultVar1, resultVar2 := s.PostStore.SaveMultiple(posts)
if resultVar2 != nil {
span.LogFields(spanlog.Error(resultVar2))
ext.Error.Set(span, true)
}
return resultVar0, resultVar1
return resultVar0, resultVar1, resultVar2
}
func (s *OpenTracingLayerPostStore) Search(teamId string, userId string, params *model.SearchParams) (*model.PostList, *model.AppError) {

36
store/sqlstore/post_store.go
Просмотреть файл

@@ -99,19 +99,19 @@ func (s *SqlPostStore) createIndexesIfNotExists() {
s.CreateFullTextIndexIfNotExists("idx_posts_hashtags_txt", "Posts", "Hashtags")
}
func (s *SqlPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) {
func (s *SqlPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) {
channelNewPosts := make(map[string]int)
maxDateNewPosts := make(map[string]int64)
rootIds := make(map[string]int)
maxDateRootIds := make(map[string]int64)
for _, post := range posts {
for idx, post := range posts {
if len(post.Id) > 0 {
return nil, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.existing.app_error", nil, "id="+post.Id, http.StatusBadRequest)
return nil, idx, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.existing.app_error", nil, "id="+post.Id, http.StatusBadRequest)
}
post.PreSave()
maxPostSize := s.GetMaxPostSize()
if err := post.IsValid(maxPostSize); err != nil {
return nil, err
return nil, idx, err
}
currentChannelCount, ok := channelNewPosts[post.ChannelId]
@@ -153,11 +153,11 @@ func (s *SqlPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.
}
sql, args, err := query.ToSql()
if err != nil {
return nil, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.app_error", nil, err.Error(), http.StatusInternalServerError)
return nil, -1, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if _, err := s.GetMaster().Exec(sql, args...); err != nil {
return nil, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.app_error", nil, err.Error(), http.StatusInternalServerError)
return nil, -1, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.app_error", nil, err.Error(), http.StatusInternalServerError)
}
for channelId, count := range channelNewPosts {
@@ -190,11 +190,11 @@ func (s *SqlPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.
}
}
return posts, nil
return posts, -1, nil
}
func (s *SqlPostStore) Save(post *model.Post) (*model.Post, *model.AppError) {
posts, err := s.SaveMultiple([]*model.Post{post})
posts, _, err := s.SaveMultiple([]*model.Post{post})
if err != nil {
return nil, err
}
@@ -270,40 +270,40 @@ func (s *SqlPostStore) Update(newPost *model.Post, oldPost *model.Post) (*model.
return newPost, nil
}
func (s *SqlPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) {
func (s *SqlPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) {
updateAt := model.GetMillis()
maxPostSize := s.GetMaxPostSize()
for _, post := range posts {
for idx, post := range posts {
post.UpdateAt = updateAt
if appErr := post.IsValid(maxPostSize); appErr != nil {
return nil, appErr
return nil, idx, appErr
}
}
tx, err := s.GetMaster().Begin()
if err != nil {
return nil, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, err.Error(), http.StatusInternalServerError)
return nil, -1, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, err.Error(), http.StatusInternalServerError)
}
for _, post := range posts {
for idx, post := range posts {
if _, err = tx.Update(post); err != nil {
txErr := tx.Rollback()
if txErr != nil {
return nil, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, txErr.Error(), http.StatusInternalServerError)
return nil, idx, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, txErr.Error(), http.StatusInternalServerError)
}
return nil, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, "id="+post.Id+", "+err.Error(), http.StatusInternalServerError)
return nil, idx, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, "id="+post.Id+", "+err.Error(), http.StatusInternalServerError)
}
}
err = tx.Commit()
if err != nil {
return nil, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, err.Error(), http.StatusInternalServerError)
return nil, -1, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, err.Error(), http.StatusInternalServerError)
}
return posts, nil
return posts, -1, nil
}
func (s *SqlPostStore) Overwrite(post *model.Post) (*model.Post, *model.AppError) {
posts, err := s.OverwriteMultiple([]*model.Post{post})
posts, _, err := s.OverwriteMultiple([]*model.Post{post})
if err != nil {
return nil, err
}

4
store/store.go
Просмотреть файл

@@ -232,7 +232,7 @@ type ChannelMemberHistoryStore interface {
}
type PostStore interface {
SaveMultiple(posts []*model.Post) ([]*model.Post, *model.AppError)
SaveMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError)
Save(post *model.Post) (*model.Post, *model.AppError)
Update(newPost *model.Post, oldPost *model.Post) (*model.Post, *model.AppError)
Get(id string, skipFetchThreads bool) (*model.PostList, *model.AppError)
@@ -260,7 +260,7 @@ type PostStore interface {
InvalidateLastPostTimeCache(channelId string)
GetPostsCreatedAt(channelId string, time int64) ([]*model.Post, *model.AppError)
Overwrite(post *model.Post) (*model.Post, *model.AppError)
OverwriteMultiple(posts []*model.Post) ([]*model.Post, *model.AppError)
OverwriteMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError)
GetPostsByIds(postIds []string) ([]*model.Post, *model.AppError)
GetPostsBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.PostForIndexing, *model.AppError)
PermanentDeleteBatch(endTime int64, limit int64) (int64, *model.AppError)

38
store/storetest/mocks/PostStore.go
Просмотреть файл

@@ -638,7 +638,7 @@ func (_m *PostStore) Overwrite(post *model.Post) (*model.Post, *model.AppError)
}
// OverwriteMultiple provides a mock function with given fields: posts
func (_m *PostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) {
func (_m *PostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) {
ret := _m.Called(posts)
var r0 []*model.Post
@@ -650,16 +650,23 @@ func (_m *PostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, *mod
}
}
var r1 *model.AppError
if rf, ok := ret.Get(1).(func([]*model.Post) *model.AppError); ok {
var r1 int
if rf, ok := ret.Get(1).(func([]*model.Post) int); ok {
r1 = rf(posts)
} else {
if ret.Get(1) != nil {
r1 = ret.Get(1).(*model.AppError)
r1 = ret.Get(1).(int)
}
var r2 *model.AppError
if rf, ok := ret.Get(2).(func([]*model.Post) *model.AppError); ok {
r2 = rf(posts)
} else {
if ret.Get(2) != nil {
r2 = ret.Get(2).(*model.AppError)
}
}
return r0, r1
return r0, r1, r2
}
// PermanentDeleteBatch provides a mock function with given fields: endTime, limit
@@ -743,7 +750,7 @@ func (_m *PostStore) Save(post *model.Post) (*model.Post, *model.AppError) {
}
// SaveMultiple provides a mock function with given fields: posts
func (_m *PostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) {
func (_m *PostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) {
ret := _m.Called(posts)
var r0 []*model.Post
@@ -755,16 +762,23 @@ func (_m *PostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.Ap
}
}
var r1 *model.AppError
if rf, ok := ret.Get(1).(func([]*model.Post) *model.AppError); ok {
var r1 int
if rf, ok := ret.Get(1).(func([]*model.Post) int); ok {
r1 = rf(posts)
} else {
if ret.Get(1) != nil {
r1 = ret.Get(1).(*model.AppError)
r1 = ret.Get(1).(int)
}
var r2 *model.AppError
if rf, ok := ret.Get(2).(func([]*model.Post) *model.AppError); ok {
r2 = rf(posts)
} else {
if ret.Get(2) != nil {
r2 = ret.Get(2).(*model.AppError)
}
}
return r0, r1
return r0, r1, r2
}
// Search provides a mock function with given fields: teamId, userId, params

21
store/storetest/post_store.go
Просмотреть файл

@@ -218,8 +218,9 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) {
p4.Message = "zz" + model.NewId() + "b"
t.Run("Save correctly a new set of posts", func(t *testing.T) {
newPosts, err := ss.Post().SaveMultiple([]*model.Post{&p1, &p2, &p3})
newPosts, errIdx, err := ss.Post().SaveMultiple([]*model.Post{&p1, &p2, &p3})
require.Nil(t, err)
require.Equal(t, -1, errIdx)
for _, post := range newPosts {
storedPost, err := ss.Post().GetSingle(post.Id)
assert.Nil(t, err)
@@ -253,8 +254,9 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) {
o4.UserId = model.NewId()
o4.Message = "zz" + model.NewId() + "b"
newPosts, err := ss.Post().SaveMultiple([]*model.Post{&o1, &o2, &o3, &o4})
newPosts, errIdx, err := ss.Post().SaveMultiple([]*model.Post{&o1, &o2, &o3, &o4})
require.Nil(t, err, "couldn't save item")
require.Equal(t, -1, errIdx)
assert.Len(t, newPosts, 4)
assert.Equal(t, int64(2), newPosts[0].ReplyCount)
assert.Equal(t, int64(2), newPosts[1].ReplyCount)
@@ -263,8 +265,9 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) {
})
t.Run("Try to save mixed, already saved and not saved posts", func(t *testing.T) {
newPosts, err := ss.Post().SaveMultiple([]*model.Post{&p4, &p3})
newPosts, errIdx, err := ss.Post().SaveMultiple([]*model.Post{&p4, &p3})
require.NotNil(t, err)
require.Equal(t, 1, errIdx)
require.Nil(t, newPosts)
storedPost, err := ss.Post().GetSingle(p3.Id)
assert.Nil(t, err)
@@ -289,7 +292,7 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) {
replyPost.Message = "zz" + model.NewId() + "b"
replyPost.RootId = rootPost.Id
_, err := ss.Post().SaveMultiple([]*model.Post{&rootPost, &replyPost})
_, _, err := ss.Post().SaveMultiple([]*model.Post{&rootPost, &replyPost})
require.Nil(t, err)
rrootPost, err := ss.Post().GetSingle(rootPost.Id)
@@ -308,7 +311,7 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) {
replyPost3.Message = "zz" + model.NewId() + "b"
replyPost3.RootId = rootPost.Id
_, err = ss.Post().SaveMultiple([]*model.Post{&replyPost2, &replyPost3})
_, _, err = ss.Post().SaveMultiple([]*model.Post{&replyPost2, &replyPost3})
require.Nil(t, err)
rrootPost2, err := ss.Post().GetSingle(rootPost.Id)
@@ -341,7 +344,7 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) {
post3.UserId = model.NewId()
post3.Message = "zz" + model.NewId() + "b"
_, err = ss.Post().SaveMultiple([]*model.Post{&post1, &post2, &post3})
_, _, err = ss.Post().SaveMultiple([]*model.Post{&post1, &post2, &post3})
require.Nil(t, err)
rchannel, err := ss.Channel().Get(channel.Id, false)
@@ -2143,8 +2146,9 @@ func testPostStoreOverwriteMultiple(t *testing.T, ss store.Store) {
o3a := ro3.Clone()
o3a.Message = ro3.Message + "WWWWWWW"
_, err = ss.Post().OverwriteMultiple([]*model.Post{o1a, o2a, o3a})
_, errIdx, err := ss.Post().OverwriteMultiple([]*model.Post{o1a, o2a, o3a})
require.Nil(t, err)
require.Equal(t, -1, errIdx)
r1, err = ss.Post().Get(o1.Id, false)
require.Nil(t, err)
@@ -2172,8 +2176,9 @@ func testPostStoreOverwriteMultiple(t *testing.T, ss store.Store) {
o5a.Filenames = []string{}
o5a.FileIds = []string{}
_, err = ss.Post().OverwriteMultiple([]*model.Post{o4a, o5a})
_, errIdx, err := ss.Post().OverwriteMultiple([]*model.Post{o4a, o5a})
require.Nil(t, err)
require.Equal(t, -1, errIdx)
r4, err = ss.Post().Get(o4.Id, false)
require.Nil(t, err)

16
store/timer_layer.go
Просмотреть файл

@@ -4476,20 +4476,20 @@ func (s *TimerLayerPostStore) Overwrite(post *model.Post) (*model.Post, *model.A
return resultVar0, resultVar1
}
func (s *TimerLayerPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) {
func (s *TimerLayerPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) {
start := timemodule.Now()
resultVar0, resultVar1 := s.PostStore.OverwriteMultiple(posts)
resultVar0, resultVar1, resultVar2 := s.PostStore.OverwriteMultiple(posts)
elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
if s.Root.Metrics != nil {
success := "false"
if resultVar1 == nil {
if resultVar2 == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("PostStore.OverwriteMultiple", success, elapsed)
}
return resultVar0, resultVar1
return resultVar0, resultVar1, resultVar2
}
func (s *TimerLayerPostStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, *model.AppError) {
@@ -4556,20 +4556,20 @@ func (s *TimerLayerPostStore) Save(post *model.Post) (*model.Post, *model.AppErr
return resultVar0, resultVar1
}
func (s *TimerLayerPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) {
func (s *TimerLayerPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) {
start := timemodule.Now()
resultVar0, resultVar1 := s.PostStore.SaveMultiple(posts)
resultVar0, resultVar1, resultVar2 := s.PostStore.SaveMultiple(posts)
elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
if s.Root.Metrics != nil {
success := "false"
if resultVar1 == nil {
if resultVar2 == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("PostStore.SaveMultiple", success, elapsed)
}
return resultVar0, resultVar1
return resultVar0, resultVar1, resultVar2
}
func (s *TimerLayerPostStore) Search(teamId string, userId string, params *model.SearchParams) (*model.PostList, *model.AppError) {