From 6a50106cd9abfbc399728b25f23b67f491ddbef9 Mon Sep 17 00:00:00 2001 From: Rodrigo Villablanca Date: Fri, 21 Aug 2020 11:58:17 -0400 Subject: [PATCH] Filter stop words when searching posts in mysql (#14509) --- i18n/en.json | 4 +++ store/searchlayer/stop_word.go | 7 +++++ store/searchtest/post_layer.go | 19 +++++++++++-- store/sqlstore/post_store.go | 32 ++++++++++++++++++++++ store/sqlstore/post_store_test.go | 45 +++++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 store/searchlayer/stop_word.go diff --git a/i18n/en.json b/i18n/en.json index 99293ab9b8..9b413cb69c 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -7358,6 +7358,10 @@ "id": "store.sql_post.populate_reply_count.app_error", "translation": "Unable to get the post replies count" }, + { + "id": "store.sql_post.search.app_error", + "translation": "Error searching posts" + }, { "id": "store.sql_post.search.disabled", "translation": "Searching has been disabled on this server. Please contact your System Administrator." diff --git a/store/searchlayer/stop_word.go b/store/searchlayer/stop_word.go new file mode 100644 index 0000000000..bbaf1c3572 --- /dev/null +++ b/store/searchlayer/stop_word.go @@ -0,0 +1,7 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package searchlayer + +var MYSQL_STOP_WORDS = []string{"a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", "from", "how", "i", "in", "is", "it", "la", "of", + "on", "or", "that", "the", "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "the", "www"} diff --git a/store/searchtest/post_layer.go b/store/searchtest/post_layer.go index 8f57573e8a..ed70663269 100644 --- a/store/searchtest/post_layer.go +++ b/store/searchtest/post_layer.go @@ -106,7 +106,7 @@ var searchPostStoreTests = []searchTest{ { Name: "Should be able to ignore stop words", Fn: testSearchIgnoringStopWords, - Tags: []string{ENGINE_ELASTICSEARCH}, + Tags: []string{ENGINE_MYSQL, ENGINE_ELASTICSEARCH}, }, { Name: "Should support search stemming", @@ -953,6 +953,8 @@ func testSearchIgnoringStopWords(t *testing.T, th *SearchTestHelper) { require.Nil(t, err) p3, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "in the a on to where you", "", model.POST_DEFAULT, 0, false) require.Nil(t, err) + p4, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "where is the car?", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) defer th.deleteUserPosts(th.User.Id) t.Run("Should avoid stop word 'the'", func(t *testing.T) { @@ -979,7 +981,7 @@ func testSearchIgnoringStopWords(t *testing.T, th *SearchTestHelper) { t.Run("Should avoid stop word 'in'", func(t *testing.T) { params := &model.SearchParams{ - Terms: "in where", + Terms: "in where you", } results, apperr := th.Store.Post().SearchPostsInTeamForUser([]*model.SearchParams{params}, th.User.Id, th.Team.Id, false, false, 0, 20) require.Nil(t, apperr) @@ -987,6 +989,19 @@ func testSearchIgnoringStopWords(t *testing.T, th *SearchTestHelper) { require.Len(t, results.Posts, 1) th.checkPostInSearchResults(t, p3.Id, results.Posts) }) + + t.Run("Should avoid stop words 'where', 'is' and 'the'", func(t *testing.T) { + results, apperr := th.Store.Post().Search(th.Team.Id, th.User.Id, &model.SearchParams{Terms: "where is the car"}) + require.Nil(t, apperr) + require.Len(t, results.Posts, 1) + th.checkPostInSearchResults(t, p4.Id, results.Posts) + }) + + t.Run("Should remove all terms and return empty list", func(t *testing.T) { + results, apperr := th.Store.Post().Search(th.Team.Id, th.User.Id, &model.SearchParams{Terms: "where is the"}) + require.Nil(t, apperr) + require.Empty(t, results.Posts) + }) } func testSupportStemming(t *testing.T, th *SearchTestHelper) { diff --git a/store/sqlstore/post_store.go b/store/sqlstore/post_store.go index d8c1262a4b..caffeed2b6 100644 --- a/store/sqlstore/post_store.go +++ b/store/sqlstore/post_store.go @@ -6,6 +6,7 @@ package sqlstore import ( "database/sql" "fmt" + "github.com/mattermost/mattermost-server/v5/store/searchlayer" "net/http" "regexp" "strconv" @@ -1273,6 +1274,18 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search searchClause := fmt.Sprintf("AND to_tsvector('english', %s) @@ to_tsquery('english', :Terms)", searchType) searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", searchClause, 1) } else if s.DriverName() == model.DATABASE_DRIVER_MYSQL { + if searchType == "Message" { + var err error + terms, err = removeMysqlStopWordsFromTerms(terms) + if err != nil { + return nil, model.NewAppError("SqlPostStore.search", "store.sql_post.search.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + if terms == "" { + return list, nil + } + } + searchClause := fmt.Sprintf("AND MATCH (%s) AGAINST (:Terms IN BOOLEAN MODE)", searchType) searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", searchClause, 1) @@ -1318,6 +1331,25 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search return list, nil } +func removeMysqlStopWordsFromTerms(terms string) (string, error) { + stopWords := make([]string, len(searchlayer.MYSQL_STOP_WORDS)) + copy(stopWords, searchlayer.MYSQL_STOP_WORDS) + re, err := regexp.Compile(fmt.Sprintf(`^(%s)$`, strings.Join(stopWords, "|"))) + if err != nil { + return "", err + } + + newTerms := make([]string, 0) + separatedTerms := strings.Fields(terms) + for _, term := range separatedTerms { + term = strings.TrimSpace(term) + if term = re.ReplaceAllString(term, ""); term != "" { + newTerms = append(newTerms, term) + } + } + return strings.Join(newTerms, " "), nil +} + func (s *SqlPostStore) AnalyticsUserCountsWithPostsByDay(teamId string) (model.AnalyticsRows, *model.AppError) { query := `SELECT DISTINCT diff --git a/store/sqlstore/post_store_test.go b/store/sqlstore/post_store_test.go index d280a24f1c..bea8bfe518 100644 --- a/store/sqlstore/post_store_test.go +++ b/store/sqlstore/post_store_test.go @@ -4,6 +4,7 @@ package sqlstore import ( + "github.com/stretchr/testify/require" "testing" "github.com/mattermost/mattermost-server/v5/store/searchtest" @@ -17,3 +18,47 @@ func TestPostStore(t *testing.T) { func TestSearchPostStore(t *testing.T) { StoreTestWithSearchTestEngine(t, searchtest.TestSearchPostStore) } + +func TestMysqlStopWords(t *testing.T) { + mysqlStopWordsTests := []struct { + Name string + Args []string + Expected []string + Empty bool + }{ + { + Name: "Should remove only the stop words", + Args: []string{"where is my car", "so this is real", "test this-and-that is awesome"}, + Expected: []string{"my car", "so real", "test this-and-that awesome"}, + }, + { + Name: "Should not remove part of a word containg stop words", + Args: []string{"whereabouts", "wherein", "tothis", "thisorthat", "waswhen", "whowas", "inthe", "whowill", "thewww"}, + Expected: []string{"whereabouts", "wherein", "tothis", "thisorthat", "waswhen", "whowas", "inthe", "whowill", "thewww"}, + }, + { + Name: "Should remove all words from terms", + Args: []string{"where about", "where in", "to this", "this or that", "was when", "who was", "in the", "who will", "the www"}, + Empty: true, + }, + { + Name: "Should not remove part of a word containg stop words separated by hyphens", + Args: []string{"where-about", "where-in", "to-this", "this-or-that", "was-when", "who-was", "in-the", "who-will", "the-www"}, + Expected: []string{"where-about", "where-in", "to-this", "this-or-that", "was-when", "who-was", "in-the", "who-will", "the-www"}, + }, + } + + for _, tc := range mysqlStopWordsTests { + t.Run(tc.Name, func(t *testing.T) { + for i, term := range tc.Args { + got, err := removeMysqlStopWordsFromTerms(term) + require.NoError(t, err) + if tc.Empty { + require.Empty(t, got) + } else { + require.Equal(t, tc.Expected[i], got) + } + } + }) + } +}