Filter stop words when searching posts in mysql (#14509)

i18n/en.json

@@ -7358,6 +7358,10 @@
     "id": "store.sql_post.populate_reply_count.app_error",
     "translation": "Unable to get the post replies count"
   },
+  {
+    "id": "store.sql_post.search.app_error",
+    "translation": "Error searching posts"
+  },
   {
     "id": "store.sql_post.search.disabled",
     "translation": "Searching has been disabled on this server. Please contact your System Administrator."

store/searchlayer/stop_word.go

@@ -0,0 +1,7 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package searchlayer
+
+var MYSQL_STOP_WORDS = []string{"a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", "from", "how", "i", "in", "is", "it", "la", "of",
+	"on", "or", "that", "the", "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "the", "www"}

store/searchtest/post_layer.go

@@ -106,7 +106,7 @@ var searchPostStoreTests = []searchTest{
 	{
 		Name: "Should be able to ignore stop words",
 		Fn:   testSearchIgnoringStopWords,
-		Tags: []string{ENGINE_ELASTICSEARCH},
+		Tags: []string{ENGINE_MYSQL, ENGINE_ELASTICSEARCH},
 	},
 	{
 		Name: "Should support search stemming",
@@ -953,6 +953,8 @@ func testSearchIgnoringStopWords(t *testing.T, th *SearchTestHelper) {
 	require.Nil(t, err)
 	p3, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "in the a on to where you", "", model.POST_DEFAULT, 0, false)
 	require.Nil(t, err)
+	p4, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "where is the car?", "", model.POST_DEFAULT, 0, false)
+	require.Nil(t, err)
 	defer th.deleteUserPosts(th.User.Id)
 
 	t.Run("Should avoid stop word 'the'", func(t *testing.T) {
@@ -979,7 +981,7 @@ func testSearchIgnoringStopWords(t *testing.T, th *SearchTestHelper) {
 
 	t.Run("Should avoid stop word 'in'", func(t *testing.T) {
 		params := &model.SearchParams{
-			Terms: "in where",
+			Terms: "in where you",
 		}
 		results, apperr := th.Store.Post().SearchPostsInTeamForUser([]*model.SearchParams{params}, th.User.Id, th.Team.Id, false, false, 0, 20)
 		require.Nil(t, apperr)
@@ -987,6 +989,19 @@ func testSearchIgnoringStopWords(t *testing.T, th *SearchTestHelper) {
 		require.Len(t, results.Posts, 1)
 		th.checkPostInSearchResults(t, p3.Id, results.Posts)
 	})
+
+	t.Run("Should avoid stop words 'where', 'is' and 'the'", func(t *testing.T) {
+		results, apperr := th.Store.Post().Search(th.Team.Id, th.User.Id, &model.SearchParams{Terms: "where is the car"})
+		require.Nil(t, apperr)
+		require.Len(t, results.Posts, 1)
+		th.checkPostInSearchResults(t, p4.Id, results.Posts)
+	})
+
+	t.Run("Should remove all terms and return empty list", func(t *testing.T) {
+		results, apperr := th.Store.Post().Search(th.Team.Id, th.User.Id, &model.SearchParams{Terms: "where is the"})
+		require.Nil(t, apperr)
+		require.Empty(t, results.Posts)
+	})
 }
 
 func testSupportStemming(t *testing.T, th *SearchTestHelper) {

store/sqlstore/post_store.go

@@ -6,6 +6,7 @@ package sqlstore
 import (
 	"database/sql"
 	"fmt"
+	"github.com/mattermost/mattermost-server/v5/store/searchlayer"
 	"net/http"
 	"regexp"
 	"strconv"
@@ -1273,6 +1274,18 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search
 		searchClause := fmt.Sprintf("AND to_tsvector('english', %s) @@  to_tsquery('english', :Terms)", searchType)
 		searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", searchClause, 1)
 	} else if s.DriverName() == model.DATABASE_DRIVER_MYSQL {
+		if searchType == "Message" {
+			var err error
+			terms, err = removeMysqlStopWordsFromTerms(terms)
+			if err != nil {
+				return nil, model.NewAppError("SqlPostStore.search", "store.sql_post.search.app_error", nil, err.Error(), http.StatusInternalServerError)
+			}
+
+			if terms == "" {
+				return list, nil
+			}
+		}
+
 		searchClause := fmt.Sprintf("AND MATCH (%s) AGAINST (:Terms IN BOOLEAN MODE)", searchType)
 		searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", searchClause, 1)
 
@@ -1318,6 +1331,25 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search
 	return list, nil
 }
 
+func removeMysqlStopWordsFromTerms(terms string) (string, error) {
+	stopWords := make([]string, len(searchlayer.MYSQL_STOP_WORDS))
+	copy(stopWords, searchlayer.MYSQL_STOP_WORDS)
+	re, err := regexp.Compile(fmt.Sprintf(`^(%s)$`, strings.Join(stopWords, "|")))
+	if err != nil {
+		return "", err
+	}
+
+	newTerms := make([]string, 0)
+	separatedTerms := strings.Fields(terms)
+	for _, term := range separatedTerms {
+		term = strings.TrimSpace(term)
+		if term = re.ReplaceAllString(term, ""); term != "" {
+			newTerms = append(newTerms, term)
+		}
+	}
+	return strings.Join(newTerms, " "), nil
+}
+
 func (s *SqlPostStore) AnalyticsUserCountsWithPostsByDay(teamId string) (model.AnalyticsRows, *model.AppError) {
 	query :=
 		`SELECT DISTINCT

store/sqlstore/post_store_test.go

@@ -4,6 +4,7 @@
 package sqlstore
 
 import (
+	"github.com/stretchr/testify/require"
 	"testing"
 
 	"github.com/mattermost/mattermost-server/v5/store/searchtest"
@@ -17,3 +18,47 @@ func TestPostStore(t *testing.T) {
 func TestSearchPostStore(t *testing.T) {
 	StoreTestWithSearchTestEngine(t, searchtest.TestSearchPostStore)
 }
+
+func TestMysqlStopWords(t *testing.T) {
+	mysqlStopWordsTests := []struct {
+		Name     string
+		Args     []string
+		Expected []string
+		Empty    bool
+	}{
+		{
+			Name:     "Should remove only the stop words",
+			Args:     []string{"where is my car", "so this is real", "test this-and-that is awesome"},
+			Expected: []string{"my car", "so real", "test this-and-that awesome"},
+		},
+		{
+			Name:     "Should not remove part of a word containg stop words",
+			Args:     []string{"whereabouts", "wherein", "tothis", "thisorthat", "waswhen", "whowas", "inthe", "whowill", "thewww"},
+			Expected: []string{"whereabouts", "wherein", "tothis", "thisorthat", "waswhen", "whowas", "inthe", "whowill", "thewww"},
+		},
+		{
+			Name:  "Should remove all words from terms",
+			Args:  []string{"where about", "where in", "to this", "this or that", "was when", "who was", "in the", "who will", "the www"},
+			Empty: true,
+		},
+		{
+			Name:     "Should not remove part of a word containg stop words separated by hyphens",
+			Args:     []string{"where-about", "where-in", "to-this", "this-or-that", "was-when", "who-was", "in-the", "who-will", "the-www"},
+			Expected: []string{"where-about", "where-in", "to-this", "this-or-that", "was-when", "who-was", "in-the", "who-will", "the-www"},
+		},
+	}
+
+	for _, tc := range mysqlStopWordsTests {
+		t.Run(tc.Name, func(t *testing.T) {
+			for i, term := range tc.Args {
+				got, err := removeMysqlStopWordsFromTerms(term)
+				require.NoError(t, err)
+				if tc.Empty {
+					require.Empty(t, got)
+				} else {
+					require.Equal(t, tc.Expected[i], got)
+				}
+			}
+		})
+	}
+}