From 8eadf849bbe5d2c8736d85d2b9690cf5b81ac981 Mon Sep 17 00:00:00 2001 From: Agniva De Sarker Date: Wed, 19 Mar 2025 12:10:29 +0530 Subject: [PATCH] MM-60790: Prevent stemming in DB search when search term is a fully quoted string (#30214) When a search term is a fully quoted string, we want to avoid stemming if it is supported by the search backend. For Postgres, it does provide a feature by which if we use the "simple" search config, then no stemming is performed and an exact match with the word is done without having to resort to LIKE queries. Unfortunately, for ES/OS this is not an option because the message field is a text field, which means ES/OS will analyze it, stem it and store it in its root form. Therefore, no exact match can be possible with ES/OS. The only solution here is to have yet another keyword field for message which will store it in its raw form. But this will effectively double the disk storage for post indices and not a good design choice. Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-term-query.html#avoid-term-query-text-fields https://mattermost.atlassian.net/browse/MM-60790 ```release-note NONE ``` * re-arrange the tests to run only on DB ```release-note NONE ``` --------- Co-authored-by: Mattermost Build --- .../channels/store/searchtest/post_layer.go | 64 +++++++++++++++++++ server/channels/store/sqlstore/post_store.go | 21 ++++-- 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/server/channels/store/searchtest/post_layer.go b/server/channels/store/searchtest/post_layer.go index e8ba847c37..37475097a1 100644 --- a/server/channels/store/searchtest/post_layer.go +++ b/server/channels/store/searchtest/post_layer.go @@ -35,6 +35,11 @@ var searchPostStoreTests = []searchTest{ Fn: testSearchANDORQuotesCombinations, Tags: []string{EnginePostgres, EngineMySQL, EngineElasticSearch}, }, + { + Name: "Should be able to search without stemming", + Fn: testStemming, + Tags: []string{EnginePostgres, EngineMySQL}, + }, { // Postgres supports search with and without quotes Name: "Should be able to search for email addresses with or without quotes", @@ -462,6 +467,65 @@ func testSearchANDORQuotesCombinations(t *testing.T, th *SearchTestHelper) { } } +func testStemming(t *testing.T, th *SearchTestHelper) { + p1, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "great minds think", "", model.PostTypeDefault, 0, false) + require.NoError(t, err) + p2, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "mindful of what you think", "", model.PostTypeDefault, 0, false) + require.NoError(t, err) + + defer th.deleteUserPosts(th.User.Id) + + testCases := []struct { + name string + terms string + orTerms bool + expectedLen int + expectedIDs []string + }{ + { + name: "simple search, no stemming", + terms: `"minds think"`, + orTerms: false, + expectedLen: 1, + expectedIDs: []string{p1.Id}, + }, + { + name: "simple search, single word, no stemming", + terms: `"minds"`, + orTerms: false, + expectedLen: 1, + expectedIDs: []string{p1.Id}, + }, + { + name: "non-simple search, stemming", + terms: `minds think`, + orTerms: true, + expectedLen: 2, + expectedIDs: []string{p1.Id, p2.Id}, + }, + { + name: "simple search, no stemming, no results", + terms: `"mind"`, + orTerms: false, + expectedLen: 0, + expectedIDs: []string{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + params := &model.SearchParams{Terms: tc.terms, OrTerms: tc.orTerms} + results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.NoError(t, err) + + require.Len(t, results.Posts, tc.expectedLen) + for _, id := range tc.expectedIDs { + th.checkPostInSearchResults(t, id, results.Posts) + } + }) + } +} + func testSearchEmailAddresses(t *testing.T, th *SearchTestHelper) { p1, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "email test@test.com", "", model.PostTypeDefault, 0, false) require.NoError(t, err) diff --git a/server/channels/store/sqlstore/post_store.go b/server/channels/store/sqlstore/post_store.go index 8b2b136b39..69f25044b5 100644 --- a/server/channels/store/sqlstore/post_store.go +++ b/server/channels/store/sqlstore/post_store.go @@ -28,6 +28,7 @@ import ( // Regex to get quoted strings var quotedStringsRegex = regexp.MustCompile(`("[^"]*")`) +var wildCardRegex = regexp.MustCompile(`\*($| )`) type SqlPostStore struct { *SqlStore @@ -2088,12 +2089,10 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search // we've already confirmed that we have a channel or user to search for } else if s.DriverName() == model.DatabaseDriverPostgres { // Parse text for wildcards - var wildcard *regexp.Regexp - if wildcard, err = regexp.Compile(`\*($| )`); err == nil { - terms = wildcard.ReplaceAllLiteralString(terms, ":* ") - excludedTerms = wildcard.ReplaceAllLiteralString(excludedTerms, ":* ") - } + terms = wildCardRegex.ReplaceAllLiteralString(terms, ":* ") + excludedTerms = wildCardRegex.ReplaceAllLiteralString(excludedTerms, ":* ") + simpleSearch := false // Replace spaces with to_tsquery symbols replaceSpaces := func(input string, excludedInput bool) string { if input == "" { @@ -2105,6 +2104,11 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search // Replace spaces within quoted strings with '<->' input = quotedStringsRegex.ReplaceAllStringFunc(input, func(match string) string { + // If the whole search term is a quoted string, + // we don't want to do stemming. + if input == match { + simpleSearch = true + } return strings.Replace(match, " ", "<->", -1) }) @@ -2124,7 +2128,12 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search tsQueryClause += " &!(" + excludedClause + ")" } - searchClause := fmt.Sprintf("to_tsvector('%[1]s', %[2]s) @@ to_tsquery('%[1]s', ?)", s.pgDefaultTextSearchConfig, searchType) + textSearchCfg := s.pgDefaultTextSearchConfig + if simpleSearch { + textSearchCfg = "simple" + } + + searchClause := fmt.Sprintf("to_tsvector('%[1]s', %[2]s) @@ to_tsquery('%[1]s', ?)", textSearchCfg, searchType) baseQuery = baseQuery.Where(searchClause, tsQueryClause) } else if s.DriverName() == model.DatabaseDriverMysql { if searchType == "Message" {