mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

Adding correctness in the ReplyCount generation (#14047)

Просмотр исходного кода 
* Adding correctness in the ReplyCount generation

* Applying suggestion from reflog

* More reliable reply count generation

* Some tests fixed

* Adding i18n translation

* Fixing reply count on save behavior

Co-authored-by: mattermod <mattermod@users.noreply.github.com>
Этот коммит содержится в:
Jesús Espino
2020-03-30 19:30:30 +02:00
коммит произвёл GitHub
родитель 4fe25b1cdd
Коммит 383e45b13d
3 изменённых файлов: 175 добавлений и 12 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
i18n/en.json
Просмотреть файл

@@ -6618,6 +6618,10 @@
"id": "store.sql_post.permanent_delete_by_user.too_many.app_error",
"translation": "Unable to select the posts to delete for the user (too many), please re-run."
},
{
"id": "store.sql_post.populate_reply_count.app_error",
"translation": "Unable to get the post replies count"
},
{
"id": "store.sql_post.save.app_error",
"translation": "Unable to save the Post."

70
store/sqlstore/post_store.go
Просмотреть файл

@@ -172,12 +172,21 @@ func (s *SqlPostStore) SaveMultiple(posts []*model.Post) ([]*model.Post, *model.
}
}
unknownRepliesPosts := []*model.Post{}
for _, post := range posts {
if len(post.RootId) == 0 {
count, ok := rootIds[post.Id]
if ok {
post.ReplyCount += int64(count)
}
} else {
unknownRepliesPosts = append(unknownRepliesPosts, post)
}
}
if len(unknownRepliesPosts) > 0 {
if err := s.populateReplyCount(unknownRepliesPosts); err != nil {
mlog.Error("Unable to populate the reply count in some posts.", mlog.Err(err))
}
}
@@ -192,6 +201,42 @@ func (s *SqlPostStore) Save(post *model.Post) (*model.Post, *model.AppError) {
return posts[0], nil
}
func (s *SqlPostStore) populateReplyCount(posts []*model.Post) *model.AppError {
rootIds := []string{}
for _, post := range posts {
rootIds = append(rootIds, post.Id)
}
countList := []struct {
RootId string
Count int64
}{}
query := s.getQueryBuilder().Select("RootId, COUNT(Id)").From("Posts").Where(sq.Eq{"RootId": rootIds}).Where(sq.Eq{"DeleteAt": 0}).GroupBy("RootId")
queryString, args, err := query.ToSql()
if err != nil {
return model.NewAppError("SqlPostStore.populateReplyCount", "store.sql_post.populate_reply_count.app_error", nil, err.Error(), http.StatusInternalServerError)
}
_, err = s.GetMaster().Select(&countList, queryString, args...)
if err != nil {
return model.NewAppError("SqlPostStore.populateReplyCount", "store.sql_post.populate_reply_count.app_error", nil, err.Error(), http.StatusInternalServerError)
}
counts := map[string]int64{}
for _, count := range countList {
counts[count.RootId] = count.Count
}
for _, post := range posts {
count, ok := counts[post.RootId]
if !ok {
post.ReplyCount = 0
}
post.ReplyCount = count
}
return nil
}
func (s *SqlPostStore) Update(newPost *model.Post, oldPost *model.Post) (*model.Post, *model.AppError) {
newPost.UpdateAt = model.GetMillis()
newPost.PreCommit()
@@ -270,7 +315,7 @@ func (s *SqlPostStore) GetFlaggedPosts(userId string, offset int, limit int) (*m
pl := model.NewPostList()
var posts []*model.Post
if _, err := s.GetReplica().Select(&posts, "SELECT *, (SELECT count(Posts.Id) FROM Posts WHERE Posts.RootId = p.Id AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE Id IN (SELECT Name FROM Preferences WHERE UserId = :UserId AND Category = :Category) AND DeleteAt = 0 ORDER BY CreateAt DESC LIMIT :Limit OFFSET :Offset", map[string]interface{}{"UserId": userId, "Category": model.PREFERENCE_CATEGORY_FLAGGED_POST, "Offset": offset, "Limit": limit}); err != nil {
if _, err := s.GetReplica().Select(&posts, "SELECT *, (SELECT count(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE Id IN (SELECT Name FROM Preferences WHERE UserId = :UserId AND Category = :Category) AND DeleteAt = 0 ORDER BY CreateAt DESC LIMIT :Limit OFFSET :Offset", map[string]interface{}{"UserId": userId, "Category": model.PREFERENCE_CATEGORY_FLAGGED_POST, "Offset": offset, "Limit": limit}); err != nil {
return nil, model.NewAppError("SqlPostStore.GetFlaggedPosts", "store.sql_post.get_flagged_posts.app_error", nil, err.Error(), http.StatusInternalServerError)
}
@@ -289,7 +334,7 @@ func (s *SqlPostStore) GetFlaggedPostsForTeam(userId, teamId string, offset int,
query := `
SELECT
A.*, (SELECT count(Posts.Id) FROM Posts WHERE Posts.RootId = A.Id AND Posts.DeleteAt = 0) as ReplyCount
A.*, (SELECT count(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN A.RootId = '' THEN A.Id ELSE A.RootId END) AND Posts.DeleteAt = 0) as ReplyCount
FROM
(SELECT
*
@@ -331,7 +376,7 @@ func (s *SqlPostStore) GetFlaggedPostsForChannel(userId, channelId string, offse
var posts []*model.Post
query := `
SELECT
*, (SELECT count(Posts.Id) FROM Posts WHERE Posts.RootId = p.Id AND Posts.DeleteAt = 0) as ReplyCount
*, (SELECT count(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount
FROM Posts p
WHERE
Id IN (SELECT Name FROM Preferences WHERE UserId = :UserId AND Category = :Category)
@@ -359,7 +404,7 @@ func (s *SqlPostStore) Get(id string, skipFetchThreads bool) (*model.PostList, *
}
var post model.Post
postFetchQuery := "SELECT p.*, (SELECT count(Posts.Id) FROM Posts WHERE Posts.RootId = p.Id AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE p.Id = :Id AND p.DeleteAt = 0"
postFetchQuery := "SELECT p.*, (SELECT count(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE p.Id = :Id AND p.DeleteAt = 0"
err := s.GetReplica().SelectOne(&post, postFetchQuery, map[string]interface{}{"Id": id})
if err != nil {
return nil, model.NewAppError("SqlPostStore.GetPost", "store.sql_post.get.app_error", nil, "id="+id+err.Error(), http.StatusNotFound)
@@ -378,7 +423,7 @@ func (s *SqlPostStore) Get(id string, skipFetchThreads bool) (*model.PostList, *
}
var posts []*model.Post
_, err = s.GetReplica().Select(&posts, "SELECT *, (SELECT count(Id) FROM Posts WHERE RootId = p.Id AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE (Id = :Id OR RootId = :RootId) AND DeleteAt = 0", map[string]interface{}{"Id": rootId, "RootId": rootId})
_, err = s.GetReplica().Select(&posts, "SELECT *, (SELECT count(Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE (Id = :Id OR RootId = :RootId) AND DeleteAt = 0", map[string]interface{}{"Id": rootId, "RootId": rootId})
if err != nil {
return nil, model.NewAppError("SqlPostStore.GetPost", "store.sql_post.get.app_error", nil, "root_id="+rootId+err.Error(), http.StatusInternalServerError)
}
@@ -557,8 +602,8 @@ func (s *SqlPostStore) GetPostsSince(options model.GetPostsSinceOptions, allowFr
replyCountQuery1 := ""
replyCountQuery2 := ""
if options.SkipFetchThreads {
replyCountQuery1 = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE p1.RootId = '' AND Posts.RootId = p1.Id AND Posts.DeleteAt = 0) as ReplyCount`
replyCountQuery2 = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE p2.RootId = '' AND Posts.RootId = p2.Id AND Posts.DeleteAt = 0) as ReplyCount`
replyCountQuery1 = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p1.RootId = '' THEN p1.Id ELSE p1.RootId END) AND Posts.DeleteAt = 0) as ReplyCount`
replyCountQuery2 = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p2.RootId = '' THEN p2.Id ELSE p2.RootId END) AND Posts.DeleteAt = 0) as ReplyCount`
}
var query string
@@ -825,10 +870,11 @@ func (s *SqlPostStore) getRootPosts(channelId string, offset int, limit int, ski
var posts []*model.Post
var fetchQuery string
if skipFetchThreads {
fetchQuery = "SELECT p.*, (SELECT COUNT(Posts.Id) FROM Posts WHERE p.RootId = '' AND Posts.RootId = p.Id AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE ChannelId = :ChannelId AND DeleteAt = 0 ORDER BY CreateAt DESC LIMIT :Limit OFFSET :Offset"
fetchQuery = "SELECT p.*, (SELECT COUNT(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE ChannelId = :ChannelId AND DeleteAt = 0 ORDER BY CreateAt DESC LIMIT :Limit OFFSET :Offset"
} else {
fetchQuery = "SELECT * FROM Posts WHERE ChannelId = :ChannelId AND DeleteAt = 0 ORDER BY CreateAt DESC LIMIT :Limit OFFSET :Offset"
}
mlog.Debug(fetchQuery, mlog.Any("params", map[string]interface{}{"ChannelId": channelId, "Offset": offset, "Limit": limit}))
_, err := s.GetReplica().Select(&posts, fetchQuery, map[string]interface{}{"ChannelId": channelId, "Offset": offset, "Limit": limit})
if err != nil {
return nil, model.NewAppError("SqlPostStore.GetLinearPosts", "store.sql_post.get_root_posts.app_error", nil, "channelId="+channelId+err.Error(), http.StatusInternalServerError)
@@ -879,7 +925,7 @@ func (s *SqlPostStore) getParentsPosts(channelId string, offset int, limit int,
replyCountQuery := ""
whereStatement := "p.Id IN (" + placeholderString + ")"
if skipFetchThreads {
replyCountQuery = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE p.RootId = '' AND Posts.RootId = p.Id AND Posts.DeleteAt = 0) as ReplyCount`
replyCountQuery = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount`
} else {
whereStatement += " OR p.RootId IN (" + placeholderString + ")"
}
@@ -905,7 +951,7 @@ func (s *SqlPostStore) getParentsPostsPostgreSQL(channelId string, offset int, l
replyCountQuery := ""
onStatement := "q1.RootId = q2.Id"
if skipFetchThreads {
replyCountQuery = ` ,(SELECT COUNT(Posts.Id) FROM Posts WHERE q2.RootId = '' AND Posts.RootId = q2.Id AND Posts.DeleteAt = 0) as ReplyCount`
replyCountQuery = ` ,(SELECT COUNT(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN q2.RootId = '' THEN q2.Id ELSE q2.RootId END) AND Posts.DeleteAt = 0) as ReplyCount`
} else {
onStatement += " OR q1.RootId = q2.RootId"
}
@@ -1112,7 +1158,7 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search
searchQuery := `
SELECT
* ,(SELECT COUNT(Posts.Id) FROM Posts WHERE q2.RootId = '' AND Posts.RootId = q2.Id AND Posts.DeleteAt = 0) as ReplyCount
* ,(SELECT COUNT(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN q2.RootId = '' THEN q2.Id ELSE q2.RootId END) AND Posts.DeleteAt = 0) as ReplyCount
FROM
Posts q2
WHERE
@@ -1395,7 +1441,7 @@ func (s *SqlPostStore) GetPostsCreatedAt(channelId string, time int64) ([]*model
func (s *SqlPostStore) GetPostsByIds(postIds []string) ([]*model.Post, *model.AppError) {
keys, params := MapStringsToQueryParams(postIds, "Post")
query := `SELECT * FROM Posts WHERE Id IN ` + keys + ` ORDER BY CreateAt DESC`
query := `SELECT p.*, (SELECT count(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE p.Id IN ` + keys + ` ORDER BY CreateAt DESC`
var posts []*model.Post
_, err := s.GetReplica().Select(&posts, query, params)

113
store/storetest/post_store.go
Просмотреть файл

@@ -33,6 +33,7 @@ func TestPostStore(t *testing.T, ss store.Store, s SqlSupplier) {
t.Run("GetPostsWithDetails", func(t *testing.T) { testPostStoreGetPostsWithDetails(t, ss) })
t.Run("GetPostsBeforeAfter", func(t *testing.T) { testPostStoreGetPostsBeforeAfter(t, ss) })
t.Run("GetPostsSince", func(t *testing.T) { testPostStoreGetPostsSince(t, ss) })
t.Run("GetPosts", func(t *testing.T) { testPostStoreGetPosts(t, ss) })
t.Run("GetPostBeforeAfter", func(t *testing.T) { testPostStoreGetPostBeforeAfter(t, ss) })
t.Run("UserCountsWithPostsByDay", func(t *testing.T) { testUserCountsWithPostsByDay(t, ss) })
t.Run("PostCountsByDay", func(t *testing.T) { testPostCountsByDay(t, ss) })
@@ -1215,6 +1216,118 @@ func testPostStoreGetPostsSince(t *testing.T, ss store.Store) {
})
}
func testPostStoreGetPosts(t *testing.T, ss store.Store) {
channelId := model.NewId()
userId := model.NewId()
post1, err := ss.Post().Save(&model.Post{
ChannelId: channelId,
UserId: userId,
Message: "message",
})
require.Nil(t, err)
time.Sleep(time.Millisecond)
post2, err := ss.Post().Save(&model.Post{
ChannelId: channelId,
UserId: userId,
Message: "message",
})
require.Nil(t, err)
time.Sleep(time.Millisecond)
post3, err := ss.Post().Save(&model.Post{
ChannelId: channelId,
UserId: userId,
Message: "message",
})
require.Nil(t, err)
time.Sleep(time.Millisecond)
post4, err := ss.Post().Save(&model.Post{
ChannelId: channelId,
UserId: userId,
Message: "message",
})
require.Nil(t, err)
time.Sleep(time.Millisecond)
post5, err := ss.Post().Save(&model.Post{
ChannelId: channelId,
UserId: userId,
Message: "message",
RootId: post3.Id,
})
require.Nil(t, err)
time.Sleep(time.Millisecond)
post6, err := ss.Post().Save(&model.Post{
ChannelId: channelId,
UserId: userId,
Message: "message",
RootId: post1.Id,
})
require.Nil(t, err)
t.Run("should return the last posts created in a channel", func(t *testing.T) {
postList, err := ss.Post().GetPosts(model.GetPostsOptions{ChannelId: channelId, Page: 0, PerPage: 30, SkipFetchThreads: false}, false)
assert.Nil(t, err)
assert.Equal(t, []string{
post6.Id,
post5.Id,
post4.Id,
post3.Id,
post2.Id,
post1.Id,
}, postList.Order)
assert.Len(t, postList.Posts, 6)
assert.NotNil(t, postList.Posts[post1.Id])
assert.NotNil(t, postList.Posts[post2.Id])
assert.NotNil(t, postList.Posts[post3.Id])
assert.NotNil(t, postList.Posts[post4.Id])
assert.NotNil(t, postList.Posts[post5.Id])
assert.NotNil(t, postList.Posts[post6.Id])
})
t.Run("should return the last posts created in a channel and the threads and the reply count must be 0", func(t *testing.T) {
postList, err := ss.Post().GetPosts(model.GetPostsOptions{ChannelId: channelId, Page: 0, PerPage: 2, SkipFetchThreads: false}, false)
assert.Nil(t, err)
assert.Equal(t, []string{
post6.Id,
post5.Id,
}, postList.Order)
assert.Len(t, postList.Posts, 4)
require.NotNil(t, postList.Posts[post1.Id])
require.NotNil(t, postList.Posts[post3.Id])
require.NotNil(t, postList.Posts[post5.Id])
require.NotNil(t, postList.Posts[post6.Id])
assert.Equal(t, int64(0), postList.Posts[post1.Id].ReplyCount)
assert.Equal(t, int64(0), postList.Posts[post3.Id].ReplyCount)
assert.Equal(t, int64(0), postList.Posts[post5.Id].ReplyCount)
assert.Equal(t, int64(0), postList.Posts[post6.Id].ReplyCount)
})
t.Run("should return the last posts created in a channel without the threads and the reply count must be correct", func(t *testing.T) {
postList, err := ss.Post().GetPosts(model.GetPostsOptions{ChannelId: channelId, Page: 0, PerPage: 2, SkipFetchThreads: true}, false)
assert.Nil(t, err)
assert.Equal(t, []string{
post6.Id,
post5.Id,
}, postList.Order)
assert.Len(t, postList.Posts, 4)
assert.NotNil(t, postList.Posts[post5.Id])
assert.NotNil(t, postList.Posts[post6.Id])
assert.Equal(t, int64(1), postList.Posts[post5.Id].ReplyCount)
assert.Equal(t, int64(1), postList.Posts[post6.Id].ReplyCount)
})
}
func testPostStoreGetPostBeforeAfter(t *testing.T, ss store.Store) {
channelId := model.NewId()