From c41e9d970ab3d5f0d8c42322bcc5b27c997a8e43 Mon Sep 17 00:00:00 2001 From: Patryk Pomykalski Date: Fri, 10 Jan 2020 05:56:44 +0100 Subject: [PATCH] Optimize queries and indexes on posts table (#13217) * Optimize queries and indexes on posts table Added RootId column to two indexes. Query in getParentPosts split into two queries - for big channels can be ~1000x times faster in MySQL, but a bit slower in PostgreSQL. Rewritten query in GetPostsSince for MySQL - around 20% faster. Benchmark results (/1 - 140k posts in channel, /2 - 800 posts): benchmark old ns/op new ns/op delta BenchmarkPosts/postgres/GetFlaggedPostsForTeam/1-16 12068006 12130615 +0.52% BenchmarkPosts/postgres/GetFlaggedPostsForChannel/1-16 7334992 7388359 +0.73% BenchmarkPosts/postgres/GetPosts(skipThreads=true)/1-16 1845547 1979362 +7.25% BenchmarkPosts/postgres/GetPosts(skipThreads=false)/1-16 2260061 2595112 +14.82% BenchmarkPosts/postgres/GetPostsSince(skipThreads=true)/1-16 38510212 40625368 +5.49% BenchmarkPosts/postgres/GetPostsSince(skipThreads=false)/1-16 32389821 32581044 +0.59% BenchmarkPosts/postgres/GetFlaggedPostsForTeam/2-16 1604215 1584941 -1.20% BenchmarkPosts/postgres/GetFlaggedPostsForChannel/2-16 1278623 1277473 -0.09% BenchmarkPosts/postgres/GetPosts(skipThreads=true)/2-16 1921049 1984581 +3.31% BenchmarkPosts/postgres/GetPosts(skipThreads=false)/2-16 3478147 3000086 -13.74% BenchmarkPosts/postgres/GetPostsSince(skipThreads=true)/2-16 4813332 5198276 +8.00% BenchmarkPosts/postgres/GetPostsSince(skipThreads=false)/2-16 3475847 3816201 +9.79% BenchmarkPosts/mysql/GetFlaggedPostsForTeam/1-16 9674132 9708361 +0.35% BenchmarkPosts/mysql/GetFlaggedPostsForChannel/1-16 5780763 5818874 +0.66% BenchmarkPosts/mysql/GetPosts(skipThreads=true)/1-16 2261194 2268826 +0.34% BenchmarkPosts/mysql/GetPosts(skipThreads=false)/1-16 2371804023 3184120 -99.87% BenchmarkPosts/mysql/GetPostsSince(skipThreads=true)/1-16 35552813 27709811 -22.06% BenchmarkPosts/mysql/GetPostsSince(skipThreads=false)/1-16 28758400 22622865 -21.33% BenchmarkPosts/mysql/GetFlaggedPostsForTeam/2-16 1174064 1205933 +2.71% BenchmarkPosts/mysql/GetFlaggedPostsForChannel/2-16 1007026 1091551 +8.39% BenchmarkPosts/mysql/GetPosts(skipThreads=true)/2-16 2274397 2408730 +5.91% BenchmarkPosts/mysql/GetPosts(skipThreads=false)/2-16 7454395 2542741 -65.89% BenchmarkPosts/mysql/GetPostsSince(skipThreads=true)/2-16 8879200 6843435 -22.93% BenchmarkPosts/mysql/GetPostsSince(skipThreads=false)/2-16 6293932 5373276 -14.63% * old version of getParentsPosts for PostgreSQL because new version using two queries is around 15% slower. * review fixes * revert index changes --- store/sqlstore/post_store.go | 194 ++++++++++++++++++++++++++--------- 1 file changed, 145 insertions(+), 49 deletions(-) diff --git a/store/sqlstore/post_store.go b/store/sqlstore/post_store.go index 9e563e137a..e9d8bdf424 100644 --- a/store/sqlstore/post_store.go +++ b/store/sqlstore/post_store.go @@ -455,37 +455,69 @@ func (s *SqlPostStore) GetPostsSince(options model.GetPostsSinceOptions, allowFr replyCountQuery1 := "" replyCountQuery2 := "" if options.SkipFetchThreads { - replyCountQuery1 = ` ,(SELECT COUNT(Posts.Id) FROM Posts WHERE p1.RootId = '' AND Posts.RootId = p1.Id AND Posts.DeleteAt = 0) as ReplyCount` - replyCountQuery2 = ` ,(SELECT COUNT(Posts.Id) FROM Posts WHERE p2.RootId = '' AND Posts.RootId = p2.Id AND Posts.DeleteAt = 0) as ReplyCount` + replyCountQuery1 = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE p1.RootId = '' AND Posts.RootId = p1.Id AND Posts.DeleteAt = 0) as ReplyCount` + replyCountQuery2 = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE p2.RootId = '' AND Posts.RootId = p2.Id AND Posts.DeleteAt = 0) as ReplyCount` } + var query string - _, err := s.GetReplica().Select(&posts, - `(SELECT - *`+replyCountQuery1+` - FROM - Posts p1 - WHERE - (UpdateAt > :Time - AND ChannelId = :ChannelId) - LIMIT 1000) - UNION + // union of IDs and then join to get full posts is faster in mysql + if s.DriverName() == model.DATABASE_DRIVER_MYSQL { + query = `SELECT *` + replyCountQuery1 + ` FROM Posts p1 JOIN ( (SELECT - *`+replyCountQuery2+` - FROM - Posts p2 - WHERE - Id - IN - (SELECT * FROM (SELECT - RootId - FROM - Posts - WHERE - UpdateAt > :Time - AND ChannelId = :ChannelId - LIMIT 1000) temp_tab)) - ORDER BY CreateAt DESC`, - map[string]interface{}{"ChannelId": options.ChannelId, "Time": options.Time}) + Id + FROM + Posts p2 + WHERE + (UpdateAt > :Time + AND ChannelId = :ChannelId) + LIMIT 1000) + UNION + (SELECT + Id + FROM + Posts p3 + WHERE + Id + IN + (SELECT * FROM (SELECT + RootId + FROM + Posts + WHERE + UpdateAt > :Time + AND ChannelId = :ChannelId + LIMIT 1000) temp_tab)) + ) j ON p1.Id = j.Id + ORDER BY CreateAt DESC` + } else if s.DriverName() == model.DATABASE_DRIVER_POSTGRES { + query = ` + (SELECT + *` + replyCountQuery1 + ` + FROM + Posts p1 + WHERE + (UpdateAt > :Time + AND ChannelId = :ChannelId) + LIMIT 1000) + UNION + (SELECT + *` + replyCountQuery2 + ` + FROM + Posts p2 + WHERE + Id + IN + (SELECT * FROM (SELECT + RootId + FROM + Posts + WHERE + UpdateAt > :Time + AND ChannelId = :ChannelId + LIMIT 1000) temp_tab)) + ORDER BY CreateAt DESC` + } + _, err := s.GetReplica().Select(&posts, query, map[string]interface{}{"ChannelId": options.ChannelId, "Time": options.Time}) if err != nil { return nil, model.NewAppError("SqlPostStore.GetPostsSince", "store.sql_post.get_posts_since.app_error", nil, "channelId="+options.ChannelId+err.Error(), http.StatusInternalServerError) @@ -697,6 +729,70 @@ func (s *SqlPostStore) getRootPosts(channelId string, offset int, limit int, ski } func (s *SqlPostStore) getParentsPosts(channelId string, offset int, limit int, skipFetchThreads bool) ([]*model.Post, *model.AppError) { + if s.DriverName() == model.DATABASE_DRIVER_POSTGRES { + return s.getParentsPostsPostgreSQL(channelId, offset, limit, skipFetchThreads) + } + + // query parent Ids first + var roots []*struct { + RootId string + } + rootQuery := ` + SELECT DISTINCT + q.RootId + FROM + (SELECT + RootId + FROM + Posts + WHERE + ChannelId = :ChannelId + AND DeleteAt = 0 + ORDER BY CreateAt DESC + LIMIT :Limit OFFSET :Offset) q + WHERE q.RootId != ''` + + _, err := s.GetReplica().Select(&roots, rootQuery, map[string]interface{}{"ChannelId": channelId, "Offset": offset, "Limit": limit}) + if err != nil { + return nil, model.NewAppError("SqlPostStore.GetLinearPosts", "store.sql_post.get_parents_posts.app_error", nil, "channelId="+channelId+" err="+err.Error(), http.StatusInternalServerError) + } + if len(roots) == 0 { + return nil, nil + } + params := make(map[string]interface{}) + placeholders := make([]string, len(roots)) + for idx, r := range roots { + key := fmt.Sprintf(":Root%v", idx) + params[key[1:]] = r.RootId + placeholders[idx] = key + } + placeholderString := strings.Join(placeholders, ", ") + params["ChannelId"] = channelId + replyCountQuery := "" + whereStatement := "p.Id IN (" + placeholderString + ")" + if skipFetchThreads { + replyCountQuery = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE p.RootId = '' AND Posts.RootId = p.Id AND Posts.DeleteAt = 0) as ReplyCount` + } else { + whereStatement += " OR p.RootId IN (" + placeholderString + ")" + } + var posts []*model.Post + _, err = s.GetReplica().Select(&posts, ` + SELECT p.*`+replyCountQuery+` + FROM + Posts p + WHERE + (`+whereStatement+`) + AND ChannelId = :ChannelId + AND DeleteAt = 0 + ORDER BY CreateAt`, + params) + if err != nil { + return nil, model.NewAppError("SqlPostStore.GetLinearPosts", "store.sql_post.get_parents_posts.app_error", nil, "channelId="+channelId+" err="+err.Error(), http.StatusInternalServerError) + } + return posts, nil +} + +func (s *SqlPostStore) getParentsPostsPostgreSQL(channelId string, offset int, limit int, skipFetchThreads bool) ([]*model.Post, *model.AppError) { var posts []*model.Post replyCountQuery := "" onStatement := "q1.RootId = q2.Id" @@ -707,27 +803,27 @@ func (s *SqlPostStore) getParentsPosts(channelId string, offset int, limit int, } _, err := s.GetReplica().Select(&posts, `SELECT q2.*`+replyCountQuery+` - FROM - Posts q2 - INNER JOIN - (SELECT DISTINCT - q3.RootId - FROM - (SELECT - RootId - FROM - Posts - WHERE - ChannelId = :ChannelId1 - AND DeleteAt = 0 - ORDER BY CreateAt DESC - LIMIT :Limit OFFSET :Offset) q3 - WHERE q3.RootId != '') q1 - ON `+onStatement+` - WHERE - ChannelId = :ChannelId2 - AND DeleteAt = 0 - ORDER BY CreateAt`, + FROM + Posts q2 + INNER JOIN + (SELECT DISTINCT + q3.RootId + FROM + (SELECT + RootId + FROM + Posts + WHERE + ChannelId = :ChannelId1 + AND DeleteAt = 0 + ORDER BY CreateAt DESC + LIMIT :Limit OFFSET :Offset) q3 + WHERE q3.RootId != '') q1 + ON `+onStatement+` + WHERE + ChannelId = :ChannelId2 + AND DeleteAt = 0 + ORDER BY CreateAt`, map[string]interface{}{"ChannelId1": channelId, "Offset": offset, "Limit": limit, "ChannelId2": channelId}) if err != nil { return nil, model.NewAppError("SqlPostStore.GetLinearPosts", "store.sql_post.get_parents_posts.app_error", nil, "channelId="+channelId+" err="+err.Error(), http.StatusInternalServerError)