MM-60790: Prevent stemming in DB search when search term is a fully quoted string (#30214)

When a search term is a fully quoted string, we want to avoid
stemming if it is supported by the search backend.

For Postgres, it does provide a feature by which if we use the
"simple" search config, then no stemming is performed and an exact
match with the word is done without having to resort
to LIKE queries.

Unfortunately, for ES/OS this is not an option because
the message field is a text field, which means ES/OS will analyze it,
stem it and store it in its root form. Therefore, no exact match
can be possible with ES/OS.

The only solution here is to have yet another keyword field
for message which will store it in its raw form. But this
will effectively double the disk storage for post indices
and not a good design choice.

Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-term-query.html#avoid-term-query-text-fields

https://mattermost.atlassian.net/browse/MM-60790

```release-note
NONE
```

* re-arrange the tests to run only on DB

```release-note
NONE
```

---------

Co-authored-by: Mattermost Build <build@mattermost.com>
Этот коммит содержится в:
Agniva De Sarker
2025-03-19 12:10:29 +05:30
коммит произвёл GitHub
родитель c951a9cfac
Коммит 8eadf849bb
2 изменённых файлов: 79 добавлений и 6 удалений

Просмотреть файл

@@ -35,6 +35,11 @@ var searchPostStoreTests = []searchTest{
Fn: testSearchANDORQuotesCombinations,
Tags: []string{EnginePostgres, EngineMySQL, EngineElasticSearch},
},
{
Name: "Should be able to search without stemming",
Fn: testStemming,
Tags: []string{EnginePostgres, EngineMySQL},
},
{
// Postgres supports search with and without quotes
Name: "Should be able to search for email addresses with or without quotes",
@@ -462,6 +467,65 @@ func testSearchANDORQuotesCombinations(t *testing.T, th *SearchTestHelper) {
}
}
func testStemming(t *testing.T, th *SearchTestHelper) {
p1, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "great minds think", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
p2, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "mindful of what you think", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)
defer th.deleteUserPosts(th.User.Id)
testCases := []struct {
name string
terms string
orTerms bool
expectedLen int
expectedIDs []string
}{
{
name: "simple search, no stemming",
terms: `"minds think"`,
orTerms: false,
expectedLen: 1,
expectedIDs: []string{p1.Id},
},
{
name: "simple search, single word, no stemming",
terms: `"minds"`,
orTerms: false,
expectedLen: 1,
expectedIDs: []string{p1.Id},
},
{
name: "non-simple search, stemming",
terms: `minds think`,
orTerms: true,
expectedLen: 2,
expectedIDs: []string{p1.Id, p2.Id},
},
{
name: "simple search, no stemming, no results",
terms: `"mind"`,
orTerms: false,
expectedLen: 0,
expectedIDs: []string{},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
params := &model.SearchParams{Terms: tc.terms, OrTerms: tc.orTerms}
results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
require.NoError(t, err)
require.Len(t, results.Posts, tc.expectedLen)
for _, id := range tc.expectedIDs {
th.checkPostInSearchResults(t, id, results.Posts)
}
})
}
}
func testSearchEmailAddresses(t *testing.T, th *SearchTestHelper) {
p1, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "email test@test.com", "", model.PostTypeDefault, 0, false)
require.NoError(t, err)

Просмотреть файл

@@ -28,6 +28,7 @@ import (
// Regex to get quoted strings
var quotedStringsRegex = regexp.MustCompile(`("[^"]*")`)
var wildCardRegex = regexp.MustCompile(`\*($| )`)
type SqlPostStore struct {
*SqlStore
@@ -2088,12 +2089,10 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search
// we've already confirmed that we have a channel or user to search for
} else if s.DriverName() == model.DatabaseDriverPostgres {
// Parse text for wildcards
var wildcard *regexp.Regexp
if wildcard, err = regexp.Compile(`\*($| )`); err == nil {
terms = wildcard.ReplaceAllLiteralString(terms, ":* ")
excludedTerms = wildcard.ReplaceAllLiteralString(excludedTerms, ":* ")
}
terms = wildCardRegex.ReplaceAllLiteralString(terms, ":* ")
excludedTerms = wildCardRegex.ReplaceAllLiteralString(excludedTerms, ":* ")
simpleSearch := false
// Replace spaces with to_tsquery symbols
replaceSpaces := func(input string, excludedInput bool) string {
if input == "" {
@@ -2105,6 +2104,11 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search
// Replace spaces within quoted strings with '<->'
input = quotedStringsRegex.ReplaceAllStringFunc(input, func(match string) string {
// If the whole search term is a quoted string,
// we don't want to do stemming.
if input == match {
simpleSearch = true
}
return strings.Replace(match, " ", "<->", -1)
})
@@ -2124,7 +2128,12 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search
tsQueryClause += " &!(" + excludedClause + ")"
}
searchClause := fmt.Sprintf("to_tsvector('%[1]s', %[2]s) @@ to_tsquery('%[1]s', ?)", s.pgDefaultTextSearchConfig, searchType)
textSearchCfg := s.pgDefaultTextSearchConfig
if simpleSearch {
textSearchCfg = "simple"
}
searchClause := fmt.Sprintf("to_tsvector('%[1]s', %[2]s) @@ to_tsquery('%[1]s', ?)", textSearchCfg, searchType)
baseQuery = baseQuery.Where(searchClause, tsQueryClause)
} else if s.DriverName() == model.DatabaseDriverMysql {
if searchType == "Message" {