MM-28444: Optimize GetPostsSince in postgres (#15411)
GetPostsSince is used when loading posts for a channel. An opportunity for optimization is that the primary SQL query is repeated twice and then a UNION is constructed for the results. ``` SELECT * FROM Posts WHERE UpdateAt > :Time AND ChannelId = :ChannelId LIMIT 1000 ``` But we can use a CTE for this which caches the results to be reused later. This leads to the main query being executed once rather than twice. And from Postgres 12 onwards, CTEs can be inlined which opens the door to further optimizations. From the docs (https://www.postgresql.org/docs/10/queries-with.html) > A useful property of WITH queries is that they are evaluated only once per execution of the parent query, even if they are referred to more than once by the parent query or sibling WITH queries. Thus, expensive calculations that are needed in multiple places can be placed within a WITH query to avoid redundant work. Another possible application is to prevent unwanted multiple evaluations of functions with side-effects. However, the other side of this coin is that the optimizer is less able to push restrictions from the parent query down into a WITH query than an ordinary subquery. In our case, the caveat does not apply because we are only filtering columns and not rows, so we can safely use it. Following are the query plan comparisons: Old: http://tatiyants.com/pev/#/plans/plan_1599394993105 New: http://tatiyants.com/pev/#/plans/plan_1599395970886 As we can see, in old bitmap index scan+heap scan happens twice, but in the new one, it happens only once. This has been load tested with a large dataset and confirmed to exhibit good improvements.
Этот коммит содержится в:
коммит произвёл
GitHub
родитель
7612888052
Коммит
1fde5112b6
@@ -6,13 +6,14 @@ package sqlstore
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"github.com/mattermost/mattermost-server/v5/store/searchlayer"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/mattermost/mattermost-server/v5/store/searchlayer"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
sq "github.com/Masterminds/squirrel"
|
||||
@@ -613,7 +614,7 @@ func (s *SqlPostStore) GetPostsSince(options model.GetPostsSinceOptions, allowFr
|
||||
replyCountQuery2 := ""
|
||||
if options.SkipFetchThreads {
|
||||
replyCountQuery1 = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p1.RootId = '' THEN p1.Id ELSE p1.RootId END) AND Posts.DeleteAt = 0) as ReplyCount`
|
||||
replyCountQuery2 = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p2.RootId = '' THEN p2.Id ELSE p2.RootId END) AND Posts.DeleteAt = 0) as ReplyCount`
|
||||
replyCountQuery2 = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN cte.RootId = '' THEN cte.Id ELSE cte.RootId END) AND Posts.DeleteAt = 0) as ReplyCount`
|
||||
}
|
||||
var query string
|
||||
|
||||
@@ -647,35 +648,19 @@ func (s *SqlPostStore) GetPostsSince(options model.GetPostsSinceOptions, allowFr
|
||||
) j ON p1.Id = j.Id
|
||||
ORDER BY CreateAt DESC`
|
||||
} else if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
|
||||
query = `
|
||||
(SELECT
|
||||
*` + replyCountQuery1 + `
|
||||
FROM
|
||||
Posts p1
|
||||
WHERE
|
||||
(UpdateAt > :Time
|
||||
AND ChannelId = :ChannelId)
|
||||
LIMIT 1000)
|
||||
UNION
|
||||
(SELECT
|
||||
*` + replyCountQuery2 + `
|
||||
FROM
|
||||
Posts p2
|
||||
WHERE
|
||||
Id
|
||||
IN
|
||||
(SELECT * FROM (SELECT
|
||||
RootId
|
||||
FROM
|
||||
Posts
|
||||
WHERE
|
||||
UpdateAt > :Time
|
||||
AND ChannelId = :ChannelId
|
||||
LIMIT 1000) temp_tab))
|
||||
ORDER BY CreateAt DESC`
|
||||
query = `WITH cte AS (SELECT
|
||||
*
|
||||
FROM
|
||||
Posts
|
||||
WHERE
|
||||
UpdateAt > :Time AND ChannelId = :ChannelId
|
||||
LIMIT 1000)
|
||||
(SELECT *` + replyCountQuery2 + ` FROM cte)
|
||||
UNION
|
||||
(SELECT *` + replyCountQuery1 + ` FROM Posts p1 WHERE id in (SELECT rootid FROM cte))
|
||||
ORDER BY CreateAt DESC`
|
||||
}
|
||||
_, err := s.GetReplica().Select(&posts, query, map[string]interface{}{"ChannelId": options.ChannelId, "Time": options.Time})
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to find Posts with channelId=%s", options.ChannelId)
|
||||
}
|
||||
|
||||
Ссылка в новой задаче
Block a user