From ce9d5d77bb98991e66041cff0249d6ffdf0a115a Mon Sep 17 00:00:00 2001 From: Agniva De Sarker Date: Mon, 13 Feb 2023 20:33:45 +0530 Subject: [PATCH] MM-50435: Improve data retention queries on MySQL (#22304) For some queries MySQL performed poorly until we slightly tune the query. Here are the results: Old query ``` // 5.7 mysql> explain SELECT Posts.Id FROM Posts LEFT JOIN Channels ON Posts.ChannelId = Channels.Id WHERE Channels.Id IS NULL LIMIT 3000; +----+-------------+----------+------------+--------+---------------+--------------------------------+---------+----------------------------+---------+----------+--------------------------------------+ | id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra | +----+-------------+----------+------------+--------+---------------+--------------------------------+---------+----------------------------+---------+----------+--------------------------------------+ | 1 | SIMPLE | Posts | NULL | index | NULL | idx_posts_channel_id_update_at | 116 | NULL | 6439997 | 100.00 | Using index | | 1 | SIMPLE | Channels | NULL | eq_ref | PRIMARY | PRIMARY | 106 | mattermost.Posts.ChannelId | 1 | 100.00 | Using where; Not exists; Using index | +----+-------------+----------+------------+--------+---------------+--------------------------------+---------+----------------------------+---------+----------+--------------------------------------+ 2 rows in set, 1 warning (0.16 sec) // 8.0 | -> Limit: 3000 row(s) (cost=13321292.71 rows=3000) (actual time=14291.753..14291.753 rows=0 loops=1) -> Filter: (Channels.Id is null) (cost=13321292.71 rows=10895918) (actual time=14291.751..14291.751 rows=0 loops=1) -> Nested loop antijoin (cost=13321292.71 rows=10895918) (actual time=14291.749..14291.749 rows=0 loops=1) -> Covering index scan on Posts using idx_posts_channel_id_update_at (cost=1337847.05 rows=10895918) (actual time=6.258..10928.120 rows=11824842 loops=1) -> Single-row covering index lookup on Channels using PRIMARY (Id=Posts.ChannelId) (cost=1.00 rows=1) (actual time=0.000..0.000 rows=1 loops=11824842) ``` New query ``` // 5.7 mysql> EXPLAIN SELECT Posts.Id FROM Posts WHERE Posts.ChannelId NOT IN (SELECT Id FROM Channels) LIMIT 3000; +----+-------------+----------+------------+-------+---------------+--------------------------------+---------+------+---------+----------+--------------------------+ | id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra | +----+-------------+----------+------------+-------+---------------+--------------------------------+---------+------+---------+----------+--------------------------+ | 1 | PRIMARY | Posts | NULL | index | NULL | idx_posts_channel_id_update_at | 116 | NULL | 6440001 | 100.00 | Using where; Using index | | 2 | SUBQUERY | Channels | NULL | index | PRIMARY | idx_channels_update_at | 9 | NULL | 60964 | 100.00 | Using index | +----+-------------+----------+------------+-------+---------------+--------------------------------+---------+------+---------+----------+--------------------------+ 2 rows in set, 1 warning (0.17 sec) // 8.0 | -> Limit: 3000 row(s) (cost=1337847.05 rows=3000) (actual time=8309.610..8309.610 rows=0 loops=1) -> Filter: (Posts.ChannelId,Posts.ChannelId in (select #2) is false) (cost=1337847.05 rows=10895918) (actual time=8309.609..8309.609 rows=0 loops=1) -> Covering index scan on Posts using idx_posts_channel_id_update_at (cost=1337847.05 rows=10895918) (actual time=0.127..2938.252 rows=11824842 loops=1) -> Select #2 (subquery in condition; run only once) -> Filter: ((Posts.ChannelId = ``.Id)) (cost=57224.17..57224.17 rows=1) (actual time=0.014..0.014 rows=1 loops=136308) -> Limit: 1 row(s) (cost=57224.07..57224.07 rows=1) (actual time=0.013..0.013 rows=1 loops=136308) -> Index lookup on using (Id=Posts.ChannelId) (actual time=0.013..0.013 rows=1 loops=136308) -> Materialize with deduplication (cost=57224.07..57224.07 rows=266638) (actual time=1361.488..1361.488 rows=270959 loops=1) -> Covering index scan on Channels using idx_channels_update_at (cost=30560.27 rows=266638) (actual time=1.807..264.116 rows=270959 loops=1) ``` We can see that in the base case, MySQL does an anitjoin which is actually fine in most queries but in this query we are selecting posts.Id, but joining on posts.channelid. In such a case, if we use a nested query, suddenly MySQL can materialize the sub-query and do only a covering index scan on Posts. Also compare the costs. Old one has 13321292.71 whereas new one has 1337847.05. So an order of magnitude improvement. Old query ``` Limit (cost=6561.27..655145.99 rows=1 width=27) (actual time=3897.766..3902.683 rows=0 loops=1) -> Gather (cost=6561.27..655145.99 rows=1 width=27) (actual time=3702.143..3707.058 rows=0 loops=1) Workers Planned: 2 Workers Launched: 2 -> Parallel Hash Anti Join (cost=5561.27..654145.89 rows=1 width=27) (actual time=3631.030..3631.032 rows=0 loops=3) Hash Cond: ((posts.channelid)::text = (channels.id)::text) -> Parallel Seq Scan on posts (cost=0.00..627918.55 rows=5510955 width=54) (actual time=0.719..2701.470 rows=4410848 loops=3) -> Parallel Hash (cost=4874.45..4874.45 rows=54945 width=27) (actual time=193.211..193.212 rows=43956 loops=3) Buckets: 262144 Batches: 1 Memory Usage: 10368kB -> Parallel Seq Scan on channels (cost=0.00..4874.45 rows=54945 width=27) (actual time=105.575..146.071 rows=43956 loops=3) Planning Time: 41.365 ms JIT: Functions: 31 Options: Inlining true, Optimization true, Expressions true, Deforming true Timing: Generation 11.106 ms, Inlining 237.381 ms, Optimization 169.412 ms, Emission 103.889 ms, Total 521.787 ms Execution Time: 4297.552 ms ``` New query ``` Limit (cost=1000.00..9419975.49 rows=3000 width=27) -> Gather (cost=1000.00..20761674119.53 rows=6612717 width=27) Workers Planned: 2 -> Parallel Seq Scan on posts (cost=0.00..20761011847.83 rows=2755299 width=27) Filter: (NOT (SubPlan 1)) SubPlan 1 -> Materialize (cost=0.00..7205.04 rows=131869 width=27) -> Seq Scan on channels (cost=0.00..5643.69 rows=131869 width=27) JIT: Functions: 9 Options: Inlining true, Optimization true, Expressions true, Deforming true (11 rows) ``` Note: this was so bad, I couldn't even wait to finish EXPLAIN ANALYZE as it was taking more than 10 minutes to finish. Old query ``` EXPLAIN ANALYZE SELECT Threads.PostId FROM Threads LEFT JOIN Channels ON Threads.ChannelId = Channels.Id WHERE Channels.Id IS NULL LIMIT 3000; // 8.0 ------------------+ | -> Limit: 3000 row(s) (cost=727387.95 rows=3000) (actual time=725.905..725.905 rows=0 loops=1) -> Filter: (Channels.Id is null) (cost=727387.95 rows=807045) (actual time=725.903..725.903 rows=0 loops=1) -> Nested loop antijoin (cost=727387.95 rows=807045) (actual time=725.901..725.901 rows=0 loops=1) -> Covering index scan on Threads using idx_threads_channel_id_last_reply_at (cost=92668.15 rows=807045) (actual time=0.201..214.332 rows=846313 loops=1) -> Single-row covering index lookup on Channels using PRIMARY (Id=Threads.ChannelId) (cost=0.69 rows=1) (actual time=0.001..0.001 rows=1 loops=846313) | ``` New query ``` // 8.0 mysql> EXPLAIN ANALYZE SELECT Threads.PostId FROM Threads WHERE Threads.ChannelId NOT IN (SELECT Id FROM Channels) LIMIT 3000; -> Limit: 3000 row(s) (cost=92668.15 rows=3000) (actual time=1770.798..1770.798 rows=0 loops=1) -> Filter: (Threads.ChannelId,Threads.ChannelId in (select #2) is false) (cost=92668.15 rows=807045) (actual time=1770.796..1770.796 rows=0 loops=1) -> Covering index scan on Threads using idx_threads_channel_id_last_reply_at (cost=92668.15 rows=807045) (actual time=0.191..197.768 rows=846313 loops=1) -> Select #2 (subquery in condition; run only once) -> Filter: ((Threads.ChannelId = ``.Id)) (cost=56081.83..56081.83 rows=1) (actual time=0.011..0.011 rows=1 loops=114202) -> Limit: 1 row(s) (cost=56081.73..56081.73 rows=1) (actual time=0.011..0.011 rows=1 loops=114202) -> Index lookup on using (Id=Threads.ChannelId) (actual time=0.011..0.011 rows=1 loops=114202) -> Materialize with deduplication (cost=56081.73..56081.73 rows=266638) (actual time=901.592..901.592 rows=270959 loops=1) -> Covering index scan on Channels using idx_channels_update_at (cost=29417.93 rows=266638) (actual time=0.129..57.806 rows=270959 loops=1) | ``` I did not test this query in Community but I expect the same logic to happen. Again, notice the improvement in cost. 727387.95 vs 727387.95 Old query ``` EXPLAIN ANALYZE SELECT Threads.PostId FROM Threads LEFT JOIN Channels ON Threads.ChannelId = Channels.Id WHERE Channels.Id IS NULL LIMIT 3000; Limit (cost=6561.27..74898.36 rows=1 width=27) (actual time=628.141..630.437 rows=0 loops=1) -> Gather (cost=6561.27..74898.36 rows=1 width=27) (actual time=628.138..630.433 rows=0 loops=1) Workers Planned: 2 Workers Launched: 2 -> Parallel Hash Anti Join (cost=5561.27..73898.26 rows=1 width=27) (actual time=612.672..612.674 rows=0 loops=3) Hash Cond: ((threads.channelid)::text = (channels.id)::text) -> Parallel Seq Scan on threads (cost=0.00..66852.00 rows=396000 width=54) (actual time=0.308..505.021 rows=315151 loops=3) -> Parallel Hash (cost=4874.45..4874.45 rows=54945 width=27) (actual time=21.031..21.031 rows=43956 loops=3) Buckets: 262144 Batches: 1 Memory Usage: 10336kB -> Parallel Seq Scan on channels (cost=0.00..4874.45 rows=54945 width=27) (actual time=0.018..7.959 rows=43956 loops=3) ``` New query ``` [bigdb] # EXPLAIN SELECT Threads.PostId FROM Threads WHERE Threads.ChannelId NOT IN (SELECT Id FROM Channels) LIMIT 3000; QUERY PLAN ------------------------------------------------------------------------------------------- Limit (cost=1000.00..9420102.76 rows=3000 width=27) -> Gather (cost=1000.00..1491986877.26 rows=475200 width=27) Workers Planned: 2 -> Parallel Seq Scan on threads (cost=0.00..1491938357.26 rows=198000 width=27) Filter: (NOT (SubPlan 1)) SubPlan 1 -> Materialize (cost=0.00..7205.04 rows=131869 width=27) -> Seq Scan on channels (cost=0.00..5643.69 rows=131869 width=27) JIT: Functions: 9 Options: Inlining true, Optimization true, Expressions true, Deforming true ``` As expected, Postgres behaves correctly in the original query. https://mattermost.atlassian.net/browse/MM-50435 ```release-note NONE ``` --- store/sqlstore/post_store.go | 37 +++++++++++++++++++++++++--------- store/sqlstore/thread_store.go | 19 ++++++++++++++++- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/store/sqlstore/post_store.go b/store/sqlstore/post_store.go index 0312058075..192b8d2834 100644 --- a/store/sqlstore/post_store.go +++ b/store/sqlstore/post_store.go @@ -2423,16 +2423,35 @@ func (s *SqlPostStore) PermanentDeleteBatchForRetentionPolicies(now, globalPolic // DeleteOrphanedRows removes entries from Posts when a corresponding channel no longer exists. func (s *SqlPostStore) DeleteOrphanedRows(limit int) (deleted int64, err error) { + var query string // We need the extra level of nesting to deal with MySQL's locking - const query = ` - DELETE FROM Posts WHERE Id IN ( - SELECT * FROM ( - SELECT Posts.Id FROM Posts - LEFT JOIN Channels ON Posts.ChannelId = Channels.Id - WHERE Channels.Id IS NULL - LIMIT ? - ) AS A - )` + if s.DriverName() == model.DatabaseDriverMysql { + // MySQL fails to do a proper antijoin if the selecting column + // and the joining column are different. In that case, doing a subquery + // leads to a faster plan because MySQL materializes the sub-query + // and does a covering index scan on Posts table. More details on the PR with + // this commit. + query = ` + DELETE FROM Posts WHERE Id IN ( + SELECT * FROM ( + SELECT Posts.Id FROM Posts + WHERE Posts.ChannelId NOT IN (SELECT Id FROM Channels USE INDEX (PRIMARY)) + LIMIT ? + ) AS A + )` + } else { + query = ` + DELETE FROM Posts WHERE Id IN ( + SELECT * FROM ( + SELECT Posts.Id FROM Posts + LEFT JOIN Channels ON Posts.ChannelId = Channels.Id + WHERE Channels.Id IS NULL + LIMIT ? + ) AS A + )` + + } + result, err := s.GetMasterX().Exec(query, limit) if err != nil { return diff --git a/store/sqlstore/thread_store.go b/store/sqlstore/thread_store.go index 735e4b835f..de402c701f 100644 --- a/store/sqlstore/thread_store.go +++ b/store/sqlstore/thread_store.go @@ -900,16 +900,33 @@ func (s *SqlThreadStore) PermanentDeleteBatchThreadMembershipsForRetentionPolici // DeleteOrphanedRows removes orphaned rows from Threads and ThreadMemberships func (s *SqlThreadStore) DeleteOrphanedRows(limit int) (deleted int64, err error) { + var threadsQuery string // We need the extra level of nesting to deal with MySQL's locking - const threadsQuery = ` + if s.DriverName() == model.DatabaseDriverMysql { + // MySQL fails to do a proper antijoin if the selecting column + // and the joining column are different. In that case, doing a subquery + // leads to a faster plan because MySQL materializes the sub-query + // and does a covering index scan on Threads table. More details on the PR with + // this commit. + threadsQuery = ` DELETE FROM Threads WHERE PostId IN ( SELECT * FROM ( + SELECT Threads.PostId FROM Threads + WHERE Threads.ChannelId NOT IN (SELECT Id FROM Channels USE INDEX(PRIMARY)) + LIMIT ? + ) AS A + )` + } else { + threadsQuery = ` + DELETE FROM Threads WHERE PostId IN ( + SELECT * FROM ( SELECT Threads.PostId FROM Threads LEFT JOIN Channels ON Threads.ChannelId = Channels.Id WHERE Channels.Id IS NULL LIMIT ? ) AS A )` + } // We only delete a thread membership if the entire thread no longer exists, // not if the root post has been deleted const threadMembershipsQuery = `