From 18c8a2bb53e452ec853dcc8f3db83a5436f44b73 Mon Sep 17 00:00:00 2001 From: Agniva De Sarker Date: Fri, 17 Jun 2022 19:52:56 +0530 Subject: [PATCH] MM-25554: Remove hardcoded english as the search config (#20472) We use the default_text_search_config param from the DB and pass that as the search config instead of hardcoding it to english. The docs do say that if no param is passed, by default it will automatically fall back to default_text_search_config. But I have seen different query plans being created when the parameter is not passed versus when it is passed. I am not sure why this is happening, but to be safe, I have taken the search config during startup and stored it in the SqlStore struct. For reference, this is what happens without a search config passed: ``` [bigdb] # explain analyze SELECT *, (SELECT COUNT(*) FROM Posts WHERE Posts.RootId = (CASE WHEN q2.RootId = '' THEN q2.Id ELSE q2.RootId END) AND Posts.DeleteAt = 0) as ReplyCount FROM Posts q2 WHERE q2.DeleteAt = 0 AND q2.Type NOT LIKE 'system_%' AND to_tsvector(Message) @@ to_tsquery('sapiente') AND ChannelId IN (SELECT Id FROM Channels, ChannelMembers WHERE Id = ChannelId AND Channels.DeleteAt = 0 AND ChannelMembe rs.UserId = 'tc3p1yqw67d8idcp3g98awexqe' AND (TeamId = '8ywxyw9ocp8smxrmjzrkqhrdwe' OR TeamId = '') AND Id IN ('h1x9asxr7idjpqfmg8q67us49h')) ORDER BY q2.CreateAt DESC LIMIT 100; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Limit (cost=8200.93..8720.23 rows=9 width=382) (actual time=177.277..177.948 rows=100 loops=1) -> Result (cost=8200.93..8720.23 rows=9 width=382) (actual time=177.274..177.939 rows=100 loops=1) -> Sort (cost=8200.93..8200.95 rows=9 width=374) (actual time=177.245..177.271 rows=100 loops=1) Sort Key: q2.createat DESC Sort Method: top-N heapsort Memory: 161kB -> Nested Loop Semi Join (cost=71.53..8200.79 rows=9 width=374) (actual time=5.910..176.815 rows=532 loops=1) -> Bitmap Heap Scan on posts q2 (cost=70.56..8183.64 rows=9 width=374) (actual time=5.815..176.457 rows=532 loops=1) Recheck Cond: ((channelid)::text = 'h1x9asxr7idjpqfmg8q67us49h'::text) Filter: (((type)::text !~~ 'system_%'::text) AND (deleteat = 0) AND (to_tsvector((message)::text) @@ to_tsquery('sapiente'::text))) Rows Removed by Filter: 5244 Heap Blocks: exact=944 -> Bitmap Index Scan on idx_posts_channel_id_update_at (cost=0.00..70.56 rows=1866 width=0) (actual time=1.883..1.883 rows=5776 loops=1) Index Cond: ((channelid)::text = 'h1x9asxr7idjpqfmg8q67us49h'::text) -> Materialize (cost=0.98..17.04 rows=1 width=27) (actual time=0.000..0.000 rows=1 loops=532) -> Nested Loop (cost=0.98..17.03 rows=1 width=27) (actual time=0.080..0.080 rows=1 loops=1) -> Index Scan using channels_pkey on channels (cost=0.42..8.45 rows=1 width=27) (actual time=0.041..0.042 rows=1 loops=1) Index Cond: ((id)::text = 'h1x9asxr7idjpqfmg8q67us49h'::text) Filter: ((deleteat = 0) AND (((teamid)::text = '8ywxyw9ocp8smxrmjzrkqhrdwe'::text) OR ((teamid)::text = ''::text))) -> Index Only Scan using idx_channelmembers_user_id_channel_id_last_viewed_at on channelmembers (cost=0.56..8.58 rows=1 width=27) (actual time=0.034..0.034 rows=1 loops=1) Index Cond: ((userid = 'tc3p1yqw67d8idcp3g98awexqe'::text) AND (channelid = 'h1x9asxr7idjpqfmg8q67us49h'::text)) Heap Fetches: 1 SubPlan 1 -> Aggregate (cost=57.68..57.69 rows=1 width=8) (actual time=0.006..0.006 rows=1 loops=100) -> Index Only Scan using idx_posts_root_id_delete_at on posts (cost=0.56..54.44 rows=1294 width=0) (actual time=0.005..0.005 rows=3 loops=100) Index Cond: ((rootid = (CASE WHEN ((q2.rootid)::text = ''::text) THEN q2.id ELSE q2.rootid END)::text) AND (deleteat = 0)) Heap Fetches: 0 Planning Time: 2.178 ms Execution Time: 178.155 ms ``` We can see it using a top-N heapsort. And this is what happens when an explicit search config is passed: ``` [bigdb] # explain analyze SELECT *, (SELECT COUNT(*) FROM Posts WHERE Posts.RootId = (CASE WHEN q2.RootId = '' THEN q2.Id ELSE q2.RootId END) AND Posts.DeleteAt = 0) as ReplyCount FROM Posts q2 WHERE q2.DeleteAt = 0 AND q2.Type NOT LIKE 'system_%' AND to_tsvector('pg_catalog.english', Message) @@ to_tsquery('pg_catalog.english', 'sapiente') AND ChannelId IN (SELECT Id FROM Channels, ChannelMembers WHERE Id = ChannelId AND Channels.DeleteAt = 0 AND ChannelMembers.UserId = 'tc3p1yqw67d8idcp3g98awexqe' AND (TeamId = '8ywxyw9ocp8smxrmjzrkqhrdwe' OR TeamId = '') AND Id IN ('h1x9asxr7idjpqfmg8q67us49h')) ORDER BY q2.CreateAt DESC LIMIT 100; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Limit (cost=1.54..10679.04 rows=100 width=382) (actual time=0.237..31.405 rows=100 loops=1) -> Nested Loop Semi Join (cost=1.54..17512.64 rows=164 width=382) (actual time=0.236..31.387 rows=100 loops=1) -> Index Scan Backward using idx_posts_channel_id_delete_at_create_at on posts q2 (cost=0.56..8032.81 rows=164 width=374) (actual time=0.121..30.124 rows=100 loops=1) Index Cond: (((channelid)::text = 'h1x9asxr7idjpqfmg8q67us49h'::text) AND (deleteat = 0)) Filter: (((type)::text !~~ 'system_%'::text) AND (to_tsvector('english'::regconfig, (message)::text) @@ '''sapient'''::tsquery)) Rows Removed by Filter: 1088 -> Materialize (cost=0.98..17.04 rows=1 width=27) (actual time=0.001..0.001 rows=1 loops=100) -> Nested Loop (cost=0.98..17.03 rows=1 width=27) (actual time=0.074..0.075 rows=1 loops=1) -> Index Scan using channels_pkey on channels (cost=0.42..8.45 rows=1 width=27) (actual time=0.032..0.032 rows=1 loops=1) Index Cond: ((id)::text = 'h1x9asxr7idjpqfmg8q67us49h'::text) Filter: ((deleteat = 0) AND (((teamid)::text = '8ywxyw9ocp8smxrmjzrkqhrdwe'::text) OR ((teamid)::text = ''::text))) -> Index Only Scan using idx_channelmembers_user_id_channel_id_last_viewed_at on channelmembers (cost=0.56..8.58 rows=1 width=27) (actual time=0.041..0.041 rows=1 loops=1) Index Cond: ((userid = 'tc3p1yqw67d8idcp3g98awexqe'::text) AND (channelid = 'h1x9asxr7idjpqfmg8q67us49h'::text)) Heap Fetches: 1 SubPlan 1 -> Aggregate (cost=57.68..57.69 rows=1 width=8) (actual time=0.010..0.010 rows=1 loops=100) -> Index Only Scan using idx_posts_root_id_delete_at on posts (cost=0.56..54.44 rows=1294 width=0) (actual time=0.009..0.009 rows=3 loops=100) Index Cond: ((rootid = (CASE WHEN ((q2.rootid)::text = ''::text) THEN q2.id ELSE q2.rootid END)::text) AND (deleteat = 0)) Heap Fetches: 0 Planning Time: 0.401 ms Execution Time: 31.466 ms ``` https://mattermost.atlassian.net/browse/MM-25554 ```release-note NONE ``` --- build/docker/postgres.conf | 1 + store/sqlstore/channel_store.go | 4 ++-- store/sqlstore/file_info_store.go | 6 +++--- store/sqlstore/post_store.go | 2 +- store/sqlstore/store.go | 18 +++++++++++++++++- 5 files changed, 24 insertions(+), 7 deletions(-) diff --git a/build/docker/postgres.conf b/build/docker/postgres.conf index 1c2b235647..2e7d3ca0ba 100644 --- a/build/docker/postgres.conf +++ b/build/docker/postgres.conf @@ -2,3 +2,4 @@ max_connections = 300 listen_addresses = '*' fsync = off full_page_writes = off +default_text_search_config = 'pg_catalog.english' \ No newline at end of file diff --git a/store/sqlstore/channel_store.go b/store/sqlstore/channel_store.go index 7ba6eebcfb..4ecfc65915 100644 --- a/store/sqlstore/channel_store.go +++ b/store/sqlstore/channel_store.go @@ -3484,7 +3484,7 @@ func (s SqlChannelStore) buildFulltextClause(term string, searchColumns string) fulltextTerm = strings.Join(splitTerm, " & ") - fulltextClause = fmt.Sprintf("((to_tsvector('english', %s)) @@ to_tsquery('english', :FulltextTerm))", convertMySQLFullTextColumnsToPostgres(searchColumns)) + fulltextClause = fmt.Sprintf("((to_tsvector('%[1]s', %[2]s)) @@ to_tsquery('%[1]s', :FulltextTerm))", s.pgDefaultTextSearchConfig, convertMySQLFullTextColumnsToPostgres(searchColumns)) } else if s.DriverName() == model.DatabaseDriverMysql { splitTerm := strings.Fields(fulltextTerm) for i, t := range strings.Fields(fulltextTerm) { @@ -3525,7 +3525,7 @@ func (s SqlChannelStore) buildFulltextClauseX(term string, searchColumns ...stri // join the search term with & fulltextTerm = strings.Join(splitTerm, " & ") - expr := fmt.Sprintf("((to_tsvector('english', %s)) @@ to_tsquery('english', ?))", strings.Join(searchColumns, " || ' ' || ")) + expr := fmt.Sprintf("((to_tsvector('%[1]s', %[2]s)) @@ to_tsquery('%[1]s', ?))", s.pgDefaultTextSearchConfig, strings.Join(searchColumns, " || ' ' || ")) return sq.Expr(expr, fulltextTerm) } diff --git a/store/sqlstore/file_info_store.go b/store/sqlstore/file_info_store.go index b9eb515992..abbf82e58f 100644 --- a/store/sqlstore/file_info_store.go +++ b/store/sqlstore/file_info_store.go @@ -633,9 +633,9 @@ func (fs SqlFileInfoStore) Search(paramsList []*model.SearchParams, userId, team } query = query.Where(sq.Or{ - sq.Expr("to_tsvector('english', FileInfo.Name) @@ to_tsquery('english', ?)", queryTerms), - sq.Expr("to_tsvector('english', Translate(FileInfo.Name, '.,-', ' ')) @@ to_tsquery('english', ?)", queryTerms), - sq.Expr("to_tsvector('english', FileInfo.Content) @@ to_tsquery('english', ?)", queryTerms), + sq.Expr(fmt.Sprintf("to_tsvector('%[1]s', FileInfo.Name) @@ to_tsquery('%[1]s', ?)", fs.pgDefaultTextSearchConfig), queryTerms), + sq.Expr(fmt.Sprintf("to_tsvector('%[1]s', Translate(FileInfo.Name, '.,-', ' ')) @@ to_tsquery('%[1]s', ?)", fs.pgDefaultTextSearchConfig), queryTerms), + sq.Expr(fmt.Sprintf("to_tsvector('%[1]s', FileInfo.Content) @@ to_tsquery('%[1]s', ?)", fs.pgDefaultTextSearchConfig), queryTerms), }) } else if fs.DriverName() == model.DatabaseDriverMysql { var err error diff --git a/store/sqlstore/post_store.go b/store/sqlstore/post_store.go index cf1bee5176..c2dd8f966f 100644 --- a/store/sqlstore/post_store.go +++ b/store/sqlstore/post_store.go @@ -1919,7 +1919,7 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search termsClause = "(" + strings.Join(strings.Fields(terms), " & ") + ")" + excludeClause } - searchClause := fmt.Sprintf("to_tsvector('english', %s) @@ to_tsquery('english', ?)", searchType) + searchClause := fmt.Sprintf("to_tsvector('%[1]s', %[2]s) @@ to_tsquery('%[1]s', ?)", s.pgDefaultTextSearchConfig, searchType) baseQuery = baseQuery.Where(searchClause, termsClause) } else if s.DriverName() == model.DatabaseDriverMysql { if searchType == "Message" { diff --git a/store/sqlstore/store.go b/store/sqlstore/store.go index 903df4194c..a0804fb4fc 100644 --- a/store/sqlstore/store.go +++ b/store/sqlstore/store.go @@ -131,7 +131,8 @@ type SqlStore struct { licenseMutex sync.RWMutex metrics einterfaces.MetricsInterface - isBinaryParam bool + isBinaryParam bool + pgDefaultTextSearchConfig string } func New(settings model.SqlSettings, metrics einterfaces.MetricsInterface) *SqlStore { @@ -169,6 +170,11 @@ func New(settings model.SqlSettings, metrics einterfaces.MetricsInterface) *SqlS mlog.Fatal("Failed to compute binary param", mlog.Err(err)) } + store.pgDefaultTextSearchConfig, err = store.computeDefaultTextSearchConfig() + if err != nil { + mlog.Fatal("Failed to compute default text search config", mlog.Err(err)) + } + store.stores.team = newSqlTeamStore(store) store.stores.channel = newSqlChannelStore(store, metrics) store.stores.post = newSqlPostStore(store, metrics) @@ -337,6 +343,16 @@ func (ss *SqlStore) computeBinaryParam() (bool, error) { return DSNHasBinaryParam(*ss.settings.DataSource) } +func (ss *SqlStore) computeDefaultTextSearchConfig() (string, error) { + if ss.DriverName() != model.DatabaseDriverPostgres { + return "", nil + } + + var defaultTextSearchConfig string + err := ss.GetMasterX().Get(&defaultTextSearchConfig, `SHOW default_text_search_config`) + return defaultTextSearchConfig, err +} + func (ss *SqlStore) IsBinaryParamEnabled() bool { return ss.isBinaryParam }