mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

[MM-25714] Keeps track of the import lines of the posts while importing to report the right line on error (#14752)

Просмотр исходного кода 
* [MM-25714] Keeps track of the import lines of the posts while importing to report the right line on error

* Adding review comments

* Reverse the order of the error and error line params
Этот коммит содержится в:
Miguel de la Cruz
2020-06-08 12:12:07 +02:00
коммит произвёл GitHub
родитель ea06c291df
Коммит 6cd898fab7
9 изменённых файлов: 1220 добавлений и 776 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

20
store/opentracing_layer.go
Просмотреть файл

@@ -4932,7 +4932,7 @@ func (s *OpenTracingLayerPostStore) Overwrite(post *model.Post) (*model.Post, *m
return resultVar0, resultVar1
}
func (s *OpenTracingLayerPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) {
func (s *OpenTracingLayerPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "PostStore.OverwriteMultiple")
s.Root.Store.SetContext(newCtx)
@@ -4941,13 +4941,13 @@ func (s *OpenTracingLayerPostStore) OverwriteMultiple(posts []*model.Post) ([]*m
}()
defer span.Finish()
resultVar0, resultVar1 := s.PostStore.OverwriteMultiple(posts)
if resultVar1 != nil {
span.LogFields(spanlog.Error(resultVar1))
resultVar0, resultVar1, resultVar2 := s.PostStore.OverwriteMultiple(posts)
if resultVar2 != nil {
span.LogFields(spanlog.Error(resultVar2))
ext.Error.Set(span, true)
}
return resultVar0, resultVar1
return resultVar0, resultVar1, resultVar2
}
func (s *OpenTracingLayerPostStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, *model.AppError) {
@@ -5022,7 +5022,7 @@ func (s *OpenTracingLayerPostStore) Save(post *model.Post) (*model.Post, *model.
return resultVar0, resultVar1
}
func (s *OpenTracingLayerPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) {
func (s *OpenTracingLayerPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "PostStore.SaveMultiple")
s.Root.Store.SetContext(newCtx)
@@ -5031,13 +5031,13 @@ func (s *OpenTracingLayerPostStore) SaveMultiple(posts []*model.Post) ([]*model.
}()
defer span.Finish()
resultVar0, resultVar1 := s.PostStore.SaveMultiple(posts)
if resultVar1 != nil {
span.LogFields(spanlog.Error(resultVar1))
resultVar0, resultVar1, resultVar2 := s.PostStore.SaveMultiple(posts)
if resultVar2 != nil {
span.LogFields(spanlog.Error(resultVar2))
ext.Error.Set(span, true)
}
return resultVar0, resultVar1
return resultVar0, resultVar1, resultVar2
}
func (s *OpenTracingLayerPostStore) Search(teamId string, userId string, params *model.SearchParams) (*model.PostList, *model.AppError) {

36
store/sqlstore/post_store.go
Просмотреть файл

@@ -99,19 +99,19 @@ func (s *SqlPostStore) createIndexesIfNotExists() {
s.CreateFullTextIndexIfNotExists("idx_posts_hashtags_txt", "Posts", "Hashtags")
}
func (s *SqlPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) {
func (s *SqlPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) {
channelNewPosts := make(map[string]int)
maxDateNewPosts := make(map[string]int64)
rootIds := make(map[string]int)
maxDateRootIds := make(map[string]int64)
for _, post := range posts {
for idx, post := range posts {
if len(post.Id) > 0 {
return nil, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.existing.app_error", nil, "id="+post.Id, http.StatusBadRequest)
return nil, idx, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.existing.app_error", nil, "id="+post.Id, http.StatusBadRequest)
}
post.PreSave()
maxPostSize := s.GetMaxPostSize()
if err := post.IsValid(maxPostSize); err != nil {
return nil, err
return nil, idx, err
}
currentChannelCount, ok := channelNewPosts[post.ChannelId]
@@ -153,11 +153,11 @@ func (s *SqlPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.
}
sql, args, err := query.ToSql()
if err != nil {
return nil, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.app_error", nil, err.Error(), http.StatusInternalServerError)
return nil, -1, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if _, err := s.GetMaster().Exec(sql, args...); err != nil {
return nil, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.app_error", nil, err.Error(), http.StatusInternalServerError)
return nil, -1, model.NewAppError("SqlPostStore.Save", "store.sql_post.save.app_error", nil, err.Error(), http.StatusInternalServerError)
}
for channelId, count := range channelNewPosts {
@@ -190,11 +190,11 @@ func (s *SqlPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.
}
}
return posts, nil
return posts, -1, nil
}
func (s *SqlPostStore) Save(post *model.Post) (*model.Post, *model.AppError) {
posts, err := s.SaveMultiple([]*model.Post{post})
posts, _, err := s.SaveMultiple([]*model.Post{post})
if err != nil {
return nil, err
}
@@ -270,40 +270,40 @@ func (s *SqlPostStore) Update(newPost *model.Post, oldPost *model.Post) (*model.
return newPost, nil
}
func (s *SqlPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) {
func (s *SqlPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) {
updateAt := model.GetMillis()
maxPostSize := s.GetMaxPostSize()
for _, post := range posts {
for idx, post := range posts {
post.UpdateAt = updateAt
if appErr := post.IsValid(maxPostSize); appErr != nil {
return nil, appErr
return nil, idx, appErr
}
}
tx, err := s.GetMaster().Begin()
if err != nil {
return nil, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, err.Error(), http.StatusInternalServerError)
return nil, -1, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, err.Error(), http.StatusInternalServerError)
}
for _, post := range posts {
for idx, post := range posts {
if _, err = tx.Update(post); err != nil {
txErr := tx.Rollback()
if txErr != nil {
return nil, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, txErr.Error(), http.StatusInternalServerError)
return nil, idx, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, txErr.Error(), http.StatusInternalServerError)
}
return nil, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, "id="+post.Id+", "+err.Error(), http.StatusInternalServerError)
return nil, idx, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, "id="+post.Id+", "+err.Error(), http.StatusInternalServerError)
}
}
err = tx.Commit()
if err != nil {
return nil, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, err.Error(), http.StatusInternalServerError)
return nil, -1, model.NewAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, err.Error(), http.StatusInternalServerError)
}
return posts, nil
return posts, -1, nil
}
func (s *SqlPostStore) Overwrite(post *model.Post) (*model.Post, *model.AppError) {
posts, err := s.OverwriteMultiple([]*model.Post{post})
posts, _, err := s.OverwriteMultiple([]*model.Post{post})
if err != nil {
return nil, err
}

4
store/store.go
Просмотреть файл

@@ -232,7 +232,7 @@ type ChannelMemberHistoryStore interface {
}
type PostStore interface {
SaveMultiple(posts []*model.Post) ([]*model.Post, *model.AppError)
SaveMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError)
Save(post *model.Post) (*model.Post, *model.AppError)
Update(newPost *model.Post, oldPost *model.Post) (*model.Post, *model.AppError)
Get(id string, skipFetchThreads bool) (*model.PostList, *model.AppError)
@@ -260,7 +260,7 @@ type PostStore interface {
InvalidateLastPostTimeCache(channelId string)
GetPostsCreatedAt(channelId string, time int64) ([]*model.Post, *model.AppError)
Overwrite(post *model.Post) (*model.Post, *model.AppError)
OverwriteMultiple(posts []*model.Post) ([]*model.Post, *model.AppError)
OverwriteMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError)
GetPostsByIds(postIds []string) ([]*model.Post, *model.AppError)
GetPostsBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.PostForIndexing, *model.AppError)
PermanentDeleteBatch(endTime int64, limit int64) (int64, *model.AppError)

38
store/storetest/mocks/PostStore.go
Просмотреть файл

@@ -638,7 +638,7 @@ func (_m *PostStore) Overwrite(post *model.Post) (*model.Post, *model.AppError)
}
// OverwriteMultiple provides a mock function with given fields: posts
func (_m *PostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) {
func (_m *PostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) {
ret := _m.Called(posts)
var r0 []*model.Post
@@ -650,16 +650,23 @@ func (_m *PostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, *mod
}
}
var r1 *model.AppError
if rf, ok := ret.Get(1).(func([]*model.Post) *model.AppError); ok {
var r1 int
if rf, ok := ret.Get(1).(func([]*model.Post) int); ok {
r1 = rf(posts)
} else {
if ret.Get(1) != nil {
r1 = ret.Get(1).(*model.AppError)
r1 = ret.Get(1).(int)
}
var r2 *model.AppError
if rf, ok := ret.Get(2).(func([]*model.Post) *model.AppError); ok {
r2 = rf(posts)
} else {
if ret.Get(2) != nil {
r2 = ret.Get(2).(*model.AppError)
}
}
return r0, r1
return r0, r1, r2
}
// PermanentDeleteBatch provides a mock function with given fields: endTime, limit
@@ -743,7 +750,7 @@ func (_m *PostStore) Save(post *model.Post) (*model.Post, *model.AppError) {
}
// SaveMultiple provides a mock function with given fields: posts
func (_m *PostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) {
func (_m *PostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) {
ret := _m.Called(posts)
var r0 []*model.Post
@@ -755,16 +762,23 @@ func (_m *PostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.Ap
}
}
var r1 *model.AppError
if rf, ok := ret.Get(1).(func([]*model.Post) *model.AppError); ok {
var r1 int
if rf, ok := ret.Get(1).(func([]*model.Post) int); ok {
r1 = rf(posts)
} else {
if ret.Get(1) != nil {
r1 = ret.Get(1).(*model.AppError)
r1 = ret.Get(1).(int)
}
var r2 *model.AppError
if rf, ok := ret.Get(2).(func([]*model.Post) *model.AppError); ok {
r2 = rf(posts)
} else {
if ret.Get(2) != nil {
r2 = ret.Get(2).(*model.AppError)
}
}
return r0, r1
return r0, r1, r2
}
// Search provides a mock function with given fields: teamId, userId, params

21
store/storetest/post_store.go
Просмотреть файл

@@ -218,8 +218,9 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) {
p4.Message = "zz" + model.NewId() + "b"
t.Run("Save correctly a new set of posts", func(t *testing.T) {
newPosts, err := ss.Post().SaveMultiple([]*model.Post{&p1, &p2, &p3})
newPosts, errIdx, err := ss.Post().SaveMultiple([]*model.Post{&p1, &p2, &p3})
require.Nil(t, err)
require.Equal(t, -1, errIdx)
for _, post := range newPosts {
storedPost, err := ss.Post().GetSingle(post.Id)
assert.Nil(t, err)
@@ -253,8 +254,9 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) {
o4.UserId = model.NewId()
o4.Message = "zz" + model.NewId() + "b"
newPosts, err := ss.Post().SaveMultiple([]*model.Post{&o1, &o2, &o3, &o4})
newPosts, errIdx, err := ss.Post().SaveMultiple([]*model.Post{&o1, &o2, &o3, &o4})
require.Nil(t, err, "couldn't save item")
require.Equal(t, -1, errIdx)
assert.Len(t, newPosts, 4)
assert.Equal(t, int64(2), newPosts[0].ReplyCount)
assert.Equal(t, int64(2), newPosts[1].ReplyCount)
@@ -263,8 +265,9 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) {
})
t.Run("Try to save mixed, already saved and not saved posts", func(t *testing.T) {
newPosts, err := ss.Post().SaveMultiple([]*model.Post{&p4, &p3})
newPosts, errIdx, err := ss.Post().SaveMultiple([]*model.Post{&p4, &p3})
require.NotNil(t, err)
require.Equal(t, 1, errIdx)
require.Nil(t, newPosts)
storedPost, err := ss.Post().GetSingle(p3.Id)
assert.Nil(t, err)
@@ -289,7 +292,7 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) {
replyPost.Message = "zz" + model.NewId() + "b"
replyPost.RootId = rootPost.Id
_, err := ss.Post().SaveMultiple([]*model.Post{&rootPost, &replyPost})
_, _, err := ss.Post().SaveMultiple([]*model.Post{&rootPost, &replyPost})
require.Nil(t, err)
rrootPost, err := ss.Post().GetSingle(rootPost.Id)
@@ -308,7 +311,7 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) {
replyPost3.Message = "zz" + model.NewId() + "b"
replyPost3.RootId = rootPost.Id
_, err = ss.Post().SaveMultiple([]*model.Post{&replyPost2, &replyPost3})
_, _, err = ss.Post().SaveMultiple([]*model.Post{&replyPost2, &replyPost3})
require.Nil(t, err)
rrootPost2, err := ss.Post().GetSingle(rootPost.Id)
@@ -341,7 +344,7 @@ func testPostStoreSaveMultiple(t *testing.T, ss store.Store) {
post3.UserId = model.NewId()
post3.Message = "zz" + model.NewId() + "b"
_, err = ss.Post().SaveMultiple([]*model.Post{&post1, &post2, &post3})
_, _, err = ss.Post().SaveMultiple([]*model.Post{&post1, &post2, &post3})
require.Nil(t, err)
rchannel, err := ss.Channel().Get(channel.Id, false)
@@ -2143,8 +2146,9 @@ func testPostStoreOverwriteMultiple(t *testing.T, ss store.Store) {
o3a := ro3.Clone()
o3a.Message = ro3.Message + "WWWWWWW"
_, err = ss.Post().OverwriteMultiple([]*model.Post{o1a, o2a, o3a})
_, errIdx, err := ss.Post().OverwriteMultiple([]*model.Post{o1a, o2a, o3a})
require.Nil(t, err)
require.Equal(t, -1, errIdx)
r1, err = ss.Post().Get(o1.Id, false)
require.Nil(t, err)
@@ -2172,8 +2176,9 @@ func testPostStoreOverwriteMultiple(t *testing.T, ss store.Store) {
o5a.Filenames = []string{}
o5a.FileIds = []string{}
_, err = ss.Post().OverwriteMultiple([]*model.Post{o4a, o5a})
_, errIdx, err := ss.Post().OverwriteMultiple([]*model.Post{o4a, o5a})
require.Nil(t, err)
require.Equal(t, -1, errIdx)
r4, err = ss.Post().Get(o4.Id, false)
require.Nil(t, err)

16
store/timer_layer.go
Просмотреть файл

@@ -4476,20 +4476,20 @@ func (s *TimerLayerPostStore) Overwrite(post *model.Post) (*model.Post, *model.A
return resultVar0, resultVar1
}
func (s *TimerLayerPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) {
func (s *TimerLayerPostStore) OverwriteMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) {
start := timemodule.Now()
resultVar0, resultVar1 := s.PostStore.OverwriteMultiple(posts)
resultVar0, resultVar1, resultVar2 := s.PostStore.OverwriteMultiple(posts)
elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
if s.Root.Metrics != nil {
success := "false"
if resultVar1 == nil {
if resultVar2 == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("PostStore.OverwriteMultiple", success, elapsed)
}
return resultVar0, resultVar1
return resultVar0, resultVar1, resultVar2
}
func (s *TimerLayerPostStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, *model.AppError) {
@@ -4556,20 +4556,20 @@ func (s *TimerLayerPostStore) Save(post *model.Post) (*model.Post, *model.AppErr
return resultVar0, resultVar1
}
func (s *TimerLayerPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.AppError) {
func (s *TimerLayerPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, int, *model.AppError) {
start := timemodule.Now()
resultVar0, resultVar1 := s.PostStore.SaveMultiple(posts)
resultVar0, resultVar1, resultVar2 := s.PostStore.SaveMultiple(posts)
elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
if s.Root.Metrics != nil {
success := "false"
if resultVar1 == nil {
if resultVar2 == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("PostStore.SaveMultiple", success, elapsed)
}
return resultVar0, resultVar1
return resultVar0, resultVar1, resultVar2
}
func (s *TimerLayerPostStore) Search(teamId string, userId string, params *model.SearchParams) (*model.PostList, *model.AppError) {