From 95fd1206705ff6e62f548619512698fefd0d36f0 Mon Sep 17 00:00:00 2001 From: Agniva De Sarker Date: Wed, 5 Jan 2022 10:27:40 +0530 Subject: [PATCH] MM-40799: Improve ReplyCount query (#19274) * MM-40799: Improve ReplyCount query This PR optimizes the reply count query to use only the indexed column to count the rows rather than using the ID column. This results in an index-only scan rather than an index scan. As a result, the query can be satisfied directly by scanning the index and there is no need to touch the heap. Count(*) also gives the same result, but is not recommended as it also verifies non-null columns and can take slightly longer. Before: ``` explain analyze SELECT p.*, (SELECT COUNT(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) AS ReplyCount FROM Posts p WHERE (CreateAt < (SELECT CreateAt FROM Posts WHERE Id = 'bguxt6zzcjytpffgza3dc7xody') AND p.ChannelId = '5xmynf36cinrzksuzhq7my1nbc' AND DeleteAt = 0) ORDER BY p.ChannelId, DeleteAt, CreateAt DESC LIMIT 30 OFFSET 0; QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Limit (cost=9.14..130616.04 rows=30 width=922) (actual time=19.208..19.454 rows=30 loops=1) InitPlan 2 (returns $2) -> Index Scan using posts_pkey on posts posts_1 (cost=0.56..8.58 rows=1 width=8) (actual time=0.028..0.030 rows=1 loops=1) Index Cond: ((id)::text = 'bguxt6zzcjytpffgza3dc7xody'::text) -> Index Scan Backward using idx_posts_channel_id_delete_at_create_at on posts p (cost=0.56..2720977.61 rows=625 width=922) (actual time=0.075..0.317 rows=30 loops=1) Index Cond: (((channelid)::text = '5xmynf36cinrzksuzhq7my1nbc'::text) AND (deleteat = 0) AND (createat < $2)) SubPlan 1 -> Aggregate (cost=4349.49..4349.50 rows=1 width=8) (actual time=0.007..0.007 rows=1 loops=30) -> Index Scan using idx_posts_root_id_delete_at on posts (cost=0.56..4346.36 rows=1253 width=27) (actual time=0.007..0.007 rows=0 loops=30) Index Cond: (((rootid)::text = (CASE WHEN ((p.rootid)::text = ''::text) THEN p.id ELSE p.rootid END)::text) AND (deleteat = 0)) ``` After: ``` explain analyze SELECT p.*, (SELECT COUNT(Posts.RootId) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) AS ReplyCount FROM Posts p WHERE (CreateAt < (SELECT CreateAt FROM Posts WHERE Id = 'bguxt6zzcjytpffgza3dc7xody') AND p.ChannelId = '5xmynf36cinrzksuzhq7my1nbc' AND DeleteAt = 0) ORDER BY p.ChannelId, DeleteAt, CreateAt DESC LIMIT 30 OFFSET 0; QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Limit (cost=9.14..130616.04 rows=30 width=922) (actual time=12.936..13.188 rows=30 loops=1) InitPlan 2 (returns $2) -> Index Scan using posts_pkey on posts posts_1 (cost=0.56..8.58 rows=1 width=8) (actual time=0.029..0.030 rows=1 loops=1) Index Cond: ((id)::text = 'bguxt6zzcjytpffgza3dc7xody'::text) -> Index Scan Backward using idx_posts_channel_id_delete_at_create_at on posts p (cost=0.56..2720977.61 rows=625 width=922) (actual time=0.076..0.324 rows=30 loops=1) Index Cond: (((channelid)::text = '5xmynf36cinrzksuzhq7my1nbc'::text) AND (deleteat = 0) AND (createat < $2)) SubPlan 1 -> Aggregate (cost=4349.49..4349.50 rows=1 width=8) (actual time=0.008..0.008 rows=1 loops=30) -> Index Only Scan using idx_posts_root_id_delete_at on posts (cost=0.56..4346.36 rows=1253 width=8) (actual time=0.007..0.007 rows=0 loops=30) Index Cond: ((rootid = (CASE WHEN ((p.rootid)::text = ''::text) THEN p.id ELSE p.rootid END)::text) AND (deleteat = 0)) Heap Fetches: 14 ``` https://mattermost.atlassian.net/browse/MM-40799 ```release-note NONE ``` * Use COUNT(*) for faster ```release-note NONE ``` --- store/sqlstore/post_store.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/store/sqlstore/post_store.go b/store/sqlstore/post_store.go index adae543dd8..3ea2dcc6fd 100644 --- a/store/sqlstore/post_store.go +++ b/store/sqlstore/post_store.go @@ -572,7 +572,7 @@ func (s *SqlPostStore) getFlaggedPosts(userId, channelId, teamId string, offset posts := []*postInternal{} query := ` SELECT - A.*, (SELECT count(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN A.RootId = '' THEN A.Id ELSE A.RootId END) AND Posts.DeleteAt = 0) as ReplyCount + A.*, (SELECT count(*) FROM Posts WHERE Posts.RootId = (CASE WHEN A.RootId = '' THEN A.Id ELSE A.RootId END) AND Posts.DeleteAt = 0) as ReplyCount FROM (SELECT * @@ -713,7 +713,7 @@ func (s *SqlPostStore) Get(ctx context.Context, id string, skipFetchThreads, col } var post postInternal - postFetchQuery := "SELECT p.*, (SELECT count(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE p.Id = ? AND p.DeleteAt = 0" + postFetchQuery := "SELECT p.*, (SELECT count(*) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE p.Id = ? AND p.DeleteAt = 0" err := s.DBXFromContext(ctx).Get(&post, postFetchQuery, id) if err != nil { if err == sql.ErrNoRows { @@ -736,7 +736,7 @@ func (s *SqlPostStore) Get(ctx context.Context, id string, skipFetchThreads, col } posts := []*postInternal{} - err = s.GetReplicaX().Select(&posts, "SELECT *, (SELECT count(Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE (Id = ? OR RootId = ?) AND DeleteAt = 0", rootId, rootId) + err = s.GetReplicaX().Select(&posts, "SELECT *, (SELECT count(*) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE (Id = ? OR RootId = ?) AND DeleteAt = 0", rootId, rootId) if err != nil { return nil, errors.Wrap(err, "failed to find Posts") } @@ -1199,8 +1199,8 @@ func (s *SqlPostStore) GetPostsSince(options model.GetPostsSinceOptions, allowFr replyCountQuery1 := "" replyCountQuery2 := "" if options.SkipFetchThreads { - replyCountQuery1 = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p1.RootId = '' THEN p1.Id ELSE p1.RootId END) AND Posts.DeleteAt = 0) as ReplyCount` - replyCountQuery2 = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN cte.RootId = '' THEN cte.Id ELSE cte.RootId END) AND Posts.DeleteAt = 0) as ReplyCount` + replyCountQuery1 = `, (SELECT COUNT(*) FROM Posts WHERE Posts.RootId = (CASE WHEN p1.RootId = '' THEN p1.Id ELSE p1.RootId END) AND Posts.DeleteAt = 0) as ReplyCount` + replyCountQuery2 = `, (SELECT COUNT(*) FROM Posts WHERE Posts.RootId = (CASE WHEN cte.RootId = '' THEN cte.Id ELSE cte.RootId END) AND Posts.DeleteAt = 0) as ReplyCount` } var query string var params []interface{} @@ -1378,7 +1378,7 @@ func (s *SqlPostStore) getPostsAround(before bool, options model.GetPostsOptions ) } query := s.getQueryBuilder().Select(columns...) - replyCountSubQuery := s.getQueryBuilder().Select("COUNT(Posts.Id)").From("Posts").Where(sq.Expr("Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0")) + replyCountSubQuery := s.getQueryBuilder().Select("COUNT(*)").From("Posts").Where(sq.Expr("Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0")) conditions := sq.And{ sq.Expr(`CreateAt `+direction+` (SELECT CreateAt FROM Posts WHERE Id = ?)`, options.PostId), @@ -1562,7 +1562,7 @@ func (s *SqlPostStore) getRootPosts(channelId string, offset int, limit int, ski posts := []*postInternal{} var fetchQuery string if skipFetchThreads { - fetchQuery = "SELECT p.*, (SELECT COUNT(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE ChannelId = ? AND DeleteAt = 0 ORDER BY CreateAt DESC LIMIT ? OFFSET ?" + fetchQuery = "SELECT p.*, (SELECT COUNT(*) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE ChannelId = ? AND DeleteAt = 0 ORDER BY CreateAt DESC LIMIT ? OFFSET ?" } else { fetchQuery = "SELECT * FROM Posts WHERE ChannelId = ? AND DeleteAt = 0 ORDER BY CreateAt DESC LIMIT ? OFFSET ?" } @@ -1615,7 +1615,7 @@ func (s *SqlPostStore) getParentsPosts(channelId string, offset int, limit int, replyCountQuery := "" whereStatement := "p.Id IN (" + placeholderString + ")" if skipFetchThreads { - replyCountQuery = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount` + replyCountQuery = `, (SELECT COUNT(*) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount` } else { whereStatement += " OR p.RootId IN (" + placeholderString + ")" } @@ -1641,7 +1641,7 @@ func (s *SqlPostStore) getParentsPostsPostgreSQL(channelId string, offset int, l replyCountQuery := "" onStatement := "q1.RootId = q2.Id" if skipFetchThreads { - replyCountQuery = ` ,(SELECT COUNT(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN q2.RootId = '' THEN q2.Id ELSE q2.RootId END) AND Posts.DeleteAt = 0) as ReplyCount` + replyCountQuery = ` ,(SELECT COUNT(*) FROM Posts WHERE Posts.RootId = (CASE WHEN q2.RootId = '' THEN q2.Id ELSE q2.RootId END) AND Posts.DeleteAt = 0) as ReplyCount` } else { onStatement += " OR q1.RootId = q2.RootId" } @@ -1844,7 +1844,7 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search baseQuery := s.getQueryBuilder().Select( "*", - "(SELECT COUNT(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN q2.RootId = '' THEN q2.Id ELSE q2.RootId END) AND Posts.DeleteAt = 0) as ReplyCount", + "(SELECT COUNT(*) FROM Posts WHERE Posts.RootId = (CASE WHEN q2.RootId = '' THEN q2.Id ELSE q2.RootId END) AND Posts.DeleteAt = 0) as ReplyCount", ).From("Posts q2"). Where("DeleteAt = 0"). Where(fmt.Sprintf("Type NOT LIKE '%s%%'", model.PostSystemMessagePrefix)). @@ -2172,7 +2172,7 @@ func (s *SqlPostStore) GetPostsCreatedAt(channelId string, time int64) ([]*model } func (s *SqlPostStore) GetPostsByIds(postIds []string) ([]*model.Post, error) { - baseQuery := s.getQueryBuilder().Select("p.*, (SELECT count(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount"). + baseQuery := s.getQueryBuilder().Select("p.*, (SELECT count(*) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount"). From("Posts p"). Where(sq.Eq{"p.Id": postIds}). OrderBy("CreateAt DESC")