Optimize queries and indexes on posts table (#13217)
* Optimize queries and indexes on posts table Added RootId column to two indexes. Query in getParentPosts split into two queries - for big channels can be ~1000x times faster in MySQL, but a bit slower in PostgreSQL. Rewritten query in GetPostsSince for MySQL - around 20% faster. Benchmark results (/1 - 140k posts in channel, /2 - 800 posts): benchmark old ns/op new ns/op delta BenchmarkPosts/postgres/GetFlaggedPostsForTeam/1-16 12068006 12130615 +0.52% BenchmarkPosts/postgres/GetFlaggedPostsForChannel/1-16 7334992 7388359 +0.73% BenchmarkPosts/postgres/GetPosts(skipThreads=true)/1-16 1845547 1979362 +7.25% BenchmarkPosts/postgres/GetPosts(skipThreads=false)/1-16 2260061 2595112 +14.82% BenchmarkPosts/postgres/GetPostsSince(skipThreads=true)/1-16 38510212 40625368 +5.49% BenchmarkPosts/postgres/GetPostsSince(skipThreads=false)/1-16 32389821 32581044 +0.59% BenchmarkPosts/postgres/GetFlaggedPostsForTeam/2-16 1604215 1584941 -1.20% BenchmarkPosts/postgres/GetFlaggedPostsForChannel/2-16 1278623 1277473 -0.09% BenchmarkPosts/postgres/GetPosts(skipThreads=true)/2-16 1921049 1984581 +3.31% BenchmarkPosts/postgres/GetPosts(skipThreads=false)/2-16 3478147 3000086 -13.74% BenchmarkPosts/postgres/GetPostsSince(skipThreads=true)/2-16 4813332 5198276 +8.00% BenchmarkPosts/postgres/GetPostsSince(skipThreads=false)/2-16 3475847 3816201 +9.79% BenchmarkPosts/mysql/GetFlaggedPostsForTeam/1-16 9674132 9708361 +0.35% BenchmarkPosts/mysql/GetFlaggedPostsForChannel/1-16 5780763 5818874 +0.66% BenchmarkPosts/mysql/GetPosts(skipThreads=true)/1-16 2261194 2268826 +0.34% BenchmarkPosts/mysql/GetPosts(skipThreads=false)/1-16 2371804023 3184120 -99.87% BenchmarkPosts/mysql/GetPostsSince(skipThreads=true)/1-16 35552813 27709811 -22.06% BenchmarkPosts/mysql/GetPostsSince(skipThreads=false)/1-16 28758400 22622865 -21.33% BenchmarkPosts/mysql/GetFlaggedPostsForTeam/2-16 1174064 1205933 +2.71% BenchmarkPosts/mysql/GetFlaggedPostsForChannel/2-16 1007026 1091551 +8.39% BenchmarkPosts/mysql/GetPosts(skipThreads=true)/2-16 2274397 2408730 +5.91% BenchmarkPosts/mysql/GetPosts(skipThreads=false)/2-16 7454395 2542741 -65.89% BenchmarkPosts/mysql/GetPostsSince(skipThreads=true)/2-16 8879200 6843435 -22.93% BenchmarkPosts/mysql/GetPostsSince(skipThreads=false)/2-16 6293932 5373276 -14.63% * old version of getParentsPosts for PostgreSQL because new version using two queries is around 15% slower. * review fixes * revert index changes
Этот коммит содержится в:
Patryk Pomykalski
коммит произвёл
Agniva De Sarker
Agniva De Sarker
родитель
d938d70dcd
Коммит
c41e9d970a
@@ -455,37 +455,69 @@ func (s *SqlPostStore) GetPostsSince(options model.GetPostsSinceOptions, allowFr
|
||||
replyCountQuery1 := ""
|
||||
replyCountQuery2 := ""
|
||||
if options.SkipFetchThreads {
|
||||
replyCountQuery1 = ` ,(SELECT COUNT(Posts.Id) FROM Posts WHERE p1.RootId = '' AND Posts.RootId = p1.Id AND Posts.DeleteAt = 0) as ReplyCount`
|
||||
replyCountQuery2 = ` ,(SELECT COUNT(Posts.Id) FROM Posts WHERE p2.RootId = '' AND Posts.RootId = p2.Id AND Posts.DeleteAt = 0) as ReplyCount`
|
||||
replyCountQuery1 = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE p1.RootId = '' AND Posts.RootId = p1.Id AND Posts.DeleteAt = 0) as ReplyCount`
|
||||
replyCountQuery2 = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE p2.RootId = '' AND Posts.RootId = p2.Id AND Posts.DeleteAt = 0) as ReplyCount`
|
||||
}
|
||||
var query string
|
||||
|
||||
_, err := s.GetReplica().Select(&posts,
|
||||
`(SELECT
|
||||
*`+replyCountQuery1+`
|
||||
FROM
|
||||
Posts p1
|
||||
WHERE
|
||||
(UpdateAt > :Time
|
||||
AND ChannelId = :ChannelId)
|
||||
LIMIT 1000)
|
||||
UNION
|
||||
// union of IDs and then join to get full posts is faster in mysql
|
||||
if s.DriverName() == model.DATABASE_DRIVER_MYSQL {
|
||||
query = `SELECT *` + replyCountQuery1 + ` FROM Posts p1 JOIN (
|
||||
(SELECT
|
||||
*`+replyCountQuery2+`
|
||||
FROM
|
||||
Posts p2
|
||||
WHERE
|
||||
Id
|
||||
IN
|
||||
(SELECT * FROM (SELECT
|
||||
RootId
|
||||
FROM
|
||||
Posts
|
||||
WHERE
|
||||
UpdateAt > :Time
|
||||
AND ChannelId = :ChannelId
|
||||
LIMIT 1000) temp_tab))
|
||||
ORDER BY CreateAt DESC`,
|
||||
map[string]interface{}{"ChannelId": options.ChannelId, "Time": options.Time})
|
||||
Id
|
||||
FROM
|
||||
Posts p2
|
||||
WHERE
|
||||
(UpdateAt > :Time
|
||||
AND ChannelId = :ChannelId)
|
||||
LIMIT 1000)
|
||||
UNION
|
||||
(SELECT
|
||||
Id
|
||||
FROM
|
||||
Posts p3
|
||||
WHERE
|
||||
Id
|
||||
IN
|
||||
(SELECT * FROM (SELECT
|
||||
RootId
|
||||
FROM
|
||||
Posts
|
||||
WHERE
|
||||
UpdateAt > :Time
|
||||
AND ChannelId = :ChannelId
|
||||
LIMIT 1000) temp_tab))
|
||||
) j ON p1.Id = j.Id
|
||||
ORDER BY CreateAt DESC`
|
||||
} else if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
|
||||
query = `
|
||||
(SELECT
|
||||
*` + replyCountQuery1 + `
|
||||
FROM
|
||||
Posts p1
|
||||
WHERE
|
||||
(UpdateAt > :Time
|
||||
AND ChannelId = :ChannelId)
|
||||
LIMIT 1000)
|
||||
UNION
|
||||
(SELECT
|
||||
*` + replyCountQuery2 + `
|
||||
FROM
|
||||
Posts p2
|
||||
WHERE
|
||||
Id
|
||||
IN
|
||||
(SELECT * FROM (SELECT
|
||||
RootId
|
||||
FROM
|
||||
Posts
|
||||
WHERE
|
||||
UpdateAt > :Time
|
||||
AND ChannelId = :ChannelId
|
||||
LIMIT 1000) temp_tab))
|
||||
ORDER BY CreateAt DESC`
|
||||
}
|
||||
_, err := s.GetReplica().Select(&posts, query, map[string]interface{}{"ChannelId": options.ChannelId, "Time": options.Time})
|
||||
|
||||
if err != nil {
|
||||
return nil, model.NewAppError("SqlPostStore.GetPostsSince", "store.sql_post.get_posts_since.app_error", nil, "channelId="+options.ChannelId+err.Error(), http.StatusInternalServerError)
|
||||
@@ -697,6 +729,70 @@ func (s *SqlPostStore) getRootPosts(channelId string, offset int, limit int, ski
|
||||
}
|
||||
|
||||
func (s *SqlPostStore) getParentsPosts(channelId string, offset int, limit int, skipFetchThreads bool) ([]*model.Post, *model.AppError) {
|
||||
if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
|
||||
return s.getParentsPostsPostgreSQL(channelId, offset, limit, skipFetchThreads)
|
||||
}
|
||||
|
||||
// query parent Ids first
|
||||
var roots []*struct {
|
||||
RootId string
|
||||
}
|
||||
rootQuery := `
|
||||
SELECT DISTINCT
|
||||
q.RootId
|
||||
FROM
|
||||
(SELECT
|
||||
RootId
|
||||
FROM
|
||||
Posts
|
||||
WHERE
|
||||
ChannelId = :ChannelId
|
||||
AND DeleteAt = 0
|
||||
ORDER BY CreateAt DESC
|
||||
LIMIT :Limit OFFSET :Offset) q
|
||||
WHERE q.RootId != ''`
|
||||
|
||||
_, err := s.GetReplica().Select(&roots, rootQuery, map[string]interface{}{"ChannelId": channelId, "Offset": offset, "Limit": limit})
|
||||
if err != nil {
|
||||
return nil, model.NewAppError("SqlPostStore.GetLinearPosts", "store.sql_post.get_parents_posts.app_error", nil, "channelId="+channelId+" err="+err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
if len(roots) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
params := make(map[string]interface{})
|
||||
placeholders := make([]string, len(roots))
|
||||
for idx, r := range roots {
|
||||
key := fmt.Sprintf(":Root%v", idx)
|
||||
params[key[1:]] = r.RootId
|
||||
placeholders[idx] = key
|
||||
}
|
||||
placeholderString := strings.Join(placeholders, ", ")
|
||||
params["ChannelId"] = channelId
|
||||
replyCountQuery := ""
|
||||
whereStatement := "p.Id IN (" + placeholderString + ")"
|
||||
if skipFetchThreads {
|
||||
replyCountQuery = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE p.RootId = '' AND Posts.RootId = p.Id AND Posts.DeleteAt = 0) as ReplyCount`
|
||||
} else {
|
||||
whereStatement += " OR p.RootId IN (" + placeholderString + ")"
|
||||
}
|
||||
var posts []*model.Post
|
||||
_, err = s.GetReplica().Select(&posts, `
|
||||
SELECT p.*`+replyCountQuery+`
|
||||
FROM
|
||||
Posts p
|
||||
WHERE
|
||||
(`+whereStatement+`)
|
||||
AND ChannelId = :ChannelId
|
||||
AND DeleteAt = 0
|
||||
ORDER BY CreateAt`,
|
||||
params)
|
||||
if err != nil {
|
||||
return nil, model.NewAppError("SqlPostStore.GetLinearPosts", "store.sql_post.get_parents_posts.app_error", nil, "channelId="+channelId+" err="+err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
return posts, nil
|
||||
}
|
||||
|
||||
func (s *SqlPostStore) getParentsPostsPostgreSQL(channelId string, offset int, limit int, skipFetchThreads bool) ([]*model.Post, *model.AppError) {
|
||||
var posts []*model.Post
|
||||
replyCountQuery := ""
|
||||
onStatement := "q1.RootId = q2.Id"
|
||||
@@ -707,27 +803,27 @@ func (s *SqlPostStore) getParentsPosts(channelId string, offset int, limit int,
|
||||
}
|
||||
_, err := s.GetReplica().Select(&posts,
|
||||
`SELECT q2.*`+replyCountQuery+`
|
||||
FROM
|
||||
Posts q2
|
||||
INNER JOIN
|
||||
(SELECT DISTINCT
|
||||
q3.RootId
|
||||
FROM
|
||||
(SELECT
|
||||
RootId
|
||||
FROM
|
||||
Posts
|
||||
WHERE
|
||||
ChannelId = :ChannelId1
|
||||
AND DeleteAt = 0
|
||||
ORDER BY CreateAt DESC
|
||||
LIMIT :Limit OFFSET :Offset) q3
|
||||
WHERE q3.RootId != '') q1
|
||||
ON `+onStatement+`
|
||||
WHERE
|
||||
ChannelId = :ChannelId2
|
||||
AND DeleteAt = 0
|
||||
ORDER BY CreateAt`,
|
||||
FROM
|
||||
Posts q2
|
||||
INNER JOIN
|
||||
(SELECT DISTINCT
|
||||
q3.RootId
|
||||
FROM
|
||||
(SELECT
|
||||
RootId
|
||||
FROM
|
||||
Posts
|
||||
WHERE
|
||||
ChannelId = :ChannelId1
|
||||
AND DeleteAt = 0
|
||||
ORDER BY CreateAt DESC
|
||||
LIMIT :Limit OFFSET :Offset) q3
|
||||
WHERE q3.RootId != '') q1
|
||||
ON `+onStatement+`
|
||||
WHERE
|
||||
ChannelId = :ChannelId2
|
||||
AND DeleteAt = 0
|
||||
ORDER BY CreateAt`,
|
||||
map[string]interface{}{"ChannelId1": channelId, "Offset": offset, "Limit": limit, "ChannelId2": channelId})
|
||||
if err != nil {
|
||||
return nil, model.NewAppError("SqlPostStore.GetLinearPosts", "store.sql_post.get_parents_posts.app_error", nil, "channelId="+channelId+" err="+err.Error(), http.StatusInternalServerError)
|
||||
|
||||
Ссылка в новой задаче
Block a user