MM-60790: Prevent stemming in DB search when search term is a fully quoted string (#30214)
When a search term is a fully quoted string, we want to avoid stemming if it is supported by the search backend. For Postgres, it does provide a feature by which if we use the "simple" search config, then no stemming is performed and an exact match with the word is done without having to resort to LIKE queries. Unfortunately, for ES/OS this is not an option because the message field is a text field, which means ES/OS will analyze it, stem it and store it in its root form. Therefore, no exact match can be possible with ES/OS. The only solution here is to have yet another keyword field for message which will store it in its raw form. But this will effectively double the disk storage for post indices and not a good design choice. Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-term-query.html#avoid-term-query-text-fields https://mattermost.atlassian.net/browse/MM-60790 ```release-note NONE ``` * re-arrange the tests to run only on DB ```release-note NONE ``` --------- Co-authored-by: Mattermost Build <build@mattermost.com>
Этот коммит содержится в:
коммит произвёл
GitHub
родитель
c951a9cfac
Коммит
8eadf849bb
@@ -35,6 +35,11 @@ var searchPostStoreTests = []searchTest{
|
||||
Fn: testSearchANDORQuotesCombinations,
|
||||
Tags: []string{EnginePostgres, EngineMySQL, EngineElasticSearch},
|
||||
},
|
||||
{
|
||||
Name: "Should be able to search without stemming",
|
||||
Fn: testStemming,
|
||||
Tags: []string{EnginePostgres, EngineMySQL},
|
||||
},
|
||||
{
|
||||
// Postgres supports search with and without quotes
|
||||
Name: "Should be able to search for email addresses with or without quotes",
|
||||
@@ -462,6 +467,65 @@ func testSearchANDORQuotesCombinations(t *testing.T, th *SearchTestHelper) {
|
||||
}
|
||||
}
|
||||
|
||||
func testStemming(t *testing.T, th *SearchTestHelper) {
|
||||
p1, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "great minds think", "", model.PostTypeDefault, 0, false)
|
||||
require.NoError(t, err)
|
||||
p2, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "mindful of what you think", "", model.PostTypeDefault, 0, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer th.deleteUserPosts(th.User.Id)
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
terms string
|
||||
orTerms bool
|
||||
expectedLen int
|
||||
expectedIDs []string
|
||||
}{
|
||||
{
|
||||
name: "simple search, no stemming",
|
||||
terms: `"minds think"`,
|
||||
orTerms: false,
|
||||
expectedLen: 1,
|
||||
expectedIDs: []string{p1.Id},
|
||||
},
|
||||
{
|
||||
name: "simple search, single word, no stemming",
|
||||
terms: `"minds"`,
|
||||
orTerms: false,
|
||||
expectedLen: 1,
|
||||
expectedIDs: []string{p1.Id},
|
||||
},
|
||||
{
|
||||
name: "non-simple search, stemming",
|
||||
terms: `minds think`,
|
||||
orTerms: true,
|
||||
expectedLen: 2,
|
||||
expectedIDs: []string{p1.Id, p2.Id},
|
||||
},
|
||||
{
|
||||
name: "simple search, no stemming, no results",
|
||||
terms: `"mind"`,
|
||||
orTerms: false,
|
||||
expectedLen: 0,
|
||||
expectedIDs: []string{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
params := &model.SearchParams{Terms: tc.terms, OrTerms: tc.orTerms}
|
||||
results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, results.Posts, tc.expectedLen)
|
||||
for _, id := range tc.expectedIDs {
|
||||
th.checkPostInSearchResults(t, id, results.Posts)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testSearchEmailAddresses(t *testing.T, th *SearchTestHelper) {
|
||||
p1, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "email test@test.com", "", model.PostTypeDefault, 0, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
|
||||
// Regex to get quoted strings
|
||||
var quotedStringsRegex = regexp.MustCompile(`("[^"]*")`)
|
||||
var wildCardRegex = regexp.MustCompile(`\*($| )`)
|
||||
|
||||
type SqlPostStore struct {
|
||||
*SqlStore
|
||||
@@ -2088,12 +2089,10 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search
|
||||
// we've already confirmed that we have a channel or user to search for
|
||||
} else if s.DriverName() == model.DatabaseDriverPostgres {
|
||||
// Parse text for wildcards
|
||||
var wildcard *regexp.Regexp
|
||||
if wildcard, err = regexp.Compile(`\*($| )`); err == nil {
|
||||
terms = wildcard.ReplaceAllLiteralString(terms, ":* ")
|
||||
excludedTerms = wildcard.ReplaceAllLiteralString(excludedTerms, ":* ")
|
||||
}
|
||||
terms = wildCardRegex.ReplaceAllLiteralString(terms, ":* ")
|
||||
excludedTerms = wildCardRegex.ReplaceAllLiteralString(excludedTerms, ":* ")
|
||||
|
||||
simpleSearch := false
|
||||
// Replace spaces with to_tsquery symbols
|
||||
replaceSpaces := func(input string, excludedInput bool) string {
|
||||
if input == "" {
|
||||
@@ -2105,6 +2104,11 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search
|
||||
|
||||
// Replace spaces within quoted strings with '<->'
|
||||
input = quotedStringsRegex.ReplaceAllStringFunc(input, func(match string) string {
|
||||
// If the whole search term is a quoted string,
|
||||
// we don't want to do stemming.
|
||||
if input == match {
|
||||
simpleSearch = true
|
||||
}
|
||||
return strings.Replace(match, " ", "<->", -1)
|
||||
})
|
||||
|
||||
@@ -2124,7 +2128,12 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search
|
||||
tsQueryClause += " &!(" + excludedClause + ")"
|
||||
}
|
||||
|
||||
searchClause := fmt.Sprintf("to_tsvector('%[1]s', %[2]s) @@ to_tsquery('%[1]s', ?)", s.pgDefaultTextSearchConfig, searchType)
|
||||
textSearchCfg := s.pgDefaultTextSearchConfig
|
||||
if simpleSearch {
|
||||
textSearchCfg = "simple"
|
||||
}
|
||||
|
||||
searchClause := fmt.Sprintf("to_tsvector('%[1]s', %[2]s) @@ to_tsquery('%[1]s', ?)", textSearchCfg, searchType)
|
||||
baseQuery = baseQuery.Where(searchClause, tsQueryClause)
|
||||
} else if s.DriverName() == model.DatabaseDriverMysql {
|
||||
if searchType == "Message" {
|
||||
|
||||
Ссылка в новой задаче
Block a user