From 971af6935c509191ab3a32d5d35435d60dab64f4 Mon Sep 17 00:00:00 2001 From: Agniva De Sarker Date: Tue, 26 Oct 2021 08:56:38 +0530 Subject: [PATCH] Optimize TotalUnreadThreads (#18788) We remove the left joins with inner joins and also remove the coalesce on posts.DeleteAt which improves the join plan from a nested loop to a hash join, and also allows the DeleteAt to be used as part of the index filter. Old plan ``` Aggregate (cost=666288.49..666288.50 rows=1 width=8) (actual time=5114.328..5119.991 rows=1 loops=1) -> Gather (cost=2880.69..666288.27 rows=87 width=8) (actual time=614.214..5119.575 rows=533 loops=1) Workers Planned: 2 Workers Launched: 2 -> Nested Loop (cost=1880.69..665279.57 rows=36 width=8) (actual time=607.821..5099.078 rows=178 loops=3) -> Hash Join (cost=1880.28..664932.93 rows=229 width=16) (actual time=224.374..1494.698 rows=1866781 loops=3) Hash Cond: ((posts.channelid)::text = (channels.id)::text) -> Parallel Seq Scan on posts (cost=0.00..662980.29 rows=27568 width=43) (actual time=201.053..1013.362 rows=4410819 loops=3) Filter: (COALESCE(deleteat, '0'::bigint) = 0) -> Hash (cost=1866.60..1866.60 rows=1094 width=27) (actual time=1.486..1.486 rows=1002 loops=3) Buckets: 2048 Batches: 1 Memory Usage: 74kB -> Index Scan using idx_channels_team_id_type on channels (cost=0.42..1866.60 rows=1094 width=27) (actual time=0.028..1.331 rows=1002 loops=3) Index Cond: ((teamid)::text = '8ywxyw9ocp8smxrmjzrkqhrdwe'::text) -> Index Scan using threadmemberships_pkey on threadmemberships (cost=0.41..1.50 rows=1 width=35) (actual time=0.002..0.002 rows=0 loops=5600344) Index Cond: (((postid)::text = (posts.rootid)::text) AND ((userid)::text = 'tc3p1yqw67d8idcp3g98awexqe'::text)) Filter: (following AND (posts.createat > lastviewed)) Rows Removed by Filter: 0 Planning Time: 2.005 ms JIT: Functions: 59 Options: Inlining true, Optimization true, Expressions true, Deforming true Timing: Generation 9.905 ms, Inlining 122.509 ms, Optimization 293.764 ms, Emission 186.064 ms, Total 612.241 ms Execution Time: 5125.739 ms ``` New plan ``` Aggregate (cost=683129.17..683129.18 rows=1 width=8) (actual time=1650.746..1656.455 rows=1 loops=1) -> Gather (cost=3766.65..683085.55 rows=17450 width=27) (actual time=641.073..1656.086 rows=533 loops=1) Workers Planned: 2 Workers Launched: 2 -> Hash Join (cost=2766.65..680340.55 rows=7271 width=27) (actual time=617.096..1635.203 rows=178 loops=3) Hash Cond: ((posts.rootid)::text = (threadmemberships.postid)::text) Join Filter: (posts.createat > threadmemberships.lastviewed) Rows Removed by Join Filter: 6 -> Hash Join (cost=1880.28..679334.08 rows=45742 width=16) (actual time=15.196..1242.832 rows=1866781 loops=3) Hash Cond: ((posts.channelid)::text = (channels.id)::text) -> Parallel Seq Scan on posts (cost=0.00..662980.29 rows=5513543 width=43) (actual time=0.028..775.118 rows=4410819 loops=3) Filter: (deleteat = 0) -> Hash (cost=1866.60..1866.60 rows=1094 width=27) (actual time=1.572..1.573 rows=1002 loops=3) Buckets: 2048 Batches: 1 Memory Usage: 74kB -> Index Scan using idx_channels_team_id_type on channels (cost=0.42..1866.60 rows=1094 width=27) (actual time=0.025..1.372 rows=1002 loops=3) Index Cond: ((teamid)::text = '8ywxyw9ocp8smxrmjzrkqhrdwe'::text) -> Hash (cost=823.41..823.41 rows=5037 width=35) (actual time=256.240..256.241 rows=5007 loops=3) Buckets: 8192 Batches: 1 Memory Usage: 417kB -> Bitmap Heap Scan on threadmemberships (cost=159.45..823.41 rows=5037 width=35) (actual time=253.011..255.255 rows=5007 loops=3) Recheck Cond: ((userid)::text = 'tc3p1yqw67d8idcp3g98awexqe'::text) Filter: following Heap Blocks: exact=601 -> Bitmap Index Scan on idx_thread_memberships_user_id (cost=0.00..158.19 rows=5037 width=0) (actual time=0.627..0.627 rows=5037 loops=3) Index Cond: ((userid)::text = 'tc3p1yqw67d8idcp3g98awexqe'::text) Planning Time: 1.952 ms JIT: Functions: 83 Options: Inlining true, Optimization true, Expressions true, Deforming true Timing: Generation 10.148 ms, Inlining 115.819 ms, Optimization 395.923 ms, Emission 244.130 ms, Total 766.021 ms Execution Time: 1661.791 ms ``` No special test has been done on MySQL since these are pretty basic optimizations and should apply to both equally. https://community-daily.mattermost.com/boards/workspace/zyoahc9uapdn3xdptac6jb69ic/285b80a3-257d-41f6-8cf4-ed80ca9d92e5/495cdb4d-c13a-4992-8eb9-80cfee2819a4?c=c9aa17f3-b918-4a78-86aa-01562e418634 ```release-note NONE ``` --- store/sqlstore/thread_store.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/store/sqlstore/thread_store.go b/store/sqlstore/thread_store.go index 8a1cce8f20..247d137ed7 100644 --- a/store/sqlstore/thread_store.go +++ b/store/sqlstore/thread_store.go @@ -149,7 +149,7 @@ func (s *SqlThreadStore) GetThreadsForUser(userId, teamId string, opts model.Get if !opts.Deleted { fetchConditions = sq.And{ fetchConditions, - sq.Eq{"COALESCE(Posts.DeleteAt, 0)": 0}, + sq.Eq{"Posts.DeleteAt": 0}, } } @@ -169,9 +169,9 @@ func (s *SqlThreadStore) GetThreadsForUser(userId, teamId string, opts model.Get go func() { repliesQuery, repliesQueryArgs, _ := s.getQueryBuilder(). Select("COUNT(DISTINCT(Posts.RootId))"). - From("Posts"). - LeftJoin("ThreadMemberships ON Posts.RootId = ThreadMemberships.PostId"). - LeftJoin("Channels ON Posts.ChannelId = Channels.Id"). + From("Posts, ThreadMemberships, Channels"). + Where("Posts.RootId = ThreadMemberships.PostId"). + Where("Posts.ChannelId = Channels.Id"). Where(fetchConditions). Where("Posts.CreateAt > ThreadMemberships.LastViewed").ToSql()