Filter stop words when searching posts in mysql (#14509)

store/sqlstore/post_store.go

@@ -6,6 +6,7 @@ package sqlstore
 import (
 	"database/sql"
 	"fmt"
+	"github.com/mattermost/mattermost-server/v5/store/searchlayer"
 	"net/http"
 	"regexp"
 	"strconv"
@@ -1273,6 +1274,18 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search
 		searchClause := fmt.Sprintf("AND to_tsvector('english', %s) @@  to_tsquery('english', :Terms)", searchType)
 		searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", searchClause, 1)
 	} else if s.DriverName() == model.DATABASE_DRIVER_MYSQL {
+		if searchType == "Message" {
+			var err error
+			terms, err = removeMysqlStopWordsFromTerms(terms)
+			if err != nil {
+				return nil, model.NewAppError("SqlPostStore.search", "store.sql_post.search.app_error", nil, err.Error(), http.StatusInternalServerError)
+			}
+
+			if terms == "" {
+				return list, nil
+			}
+		}
+
 		searchClause := fmt.Sprintf("AND MATCH (%s) AGAINST (:Terms IN BOOLEAN MODE)", searchType)
 		searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", searchClause, 1)
 
@@ -1318,6 +1331,25 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search
 	return list, nil
 }
 
+func removeMysqlStopWordsFromTerms(terms string) (string, error) {
+	stopWords := make([]string, len(searchlayer.MYSQL_STOP_WORDS))
+	copy(stopWords, searchlayer.MYSQL_STOP_WORDS)
+	re, err := regexp.Compile(fmt.Sprintf(`^(%s)$`, strings.Join(stopWords, "|")))
+	if err != nil {
+		return "", err
+	}
+
+	newTerms := make([]string, 0)
+	separatedTerms := strings.Fields(terms)
+	for _, term := range separatedTerms {
+		term = strings.TrimSpace(term)
+		if term = re.ReplaceAllString(term, ""); term != "" {
+			newTerms = append(newTerms, term)
+		}
+	}
+	return strings.Join(newTerms, " "), nil
+}
+
 func (s *SqlPostStore) AnalyticsUserCountsWithPostsByDay(teamId string) (model.AnalyticsRows, *model.AppError) {
 	query :=
 		`SELECT DISTINCT

store/sqlstore/post_store_test.go

@@ -4,6 +4,7 @@
 package sqlstore
 
 import (
+	"github.com/stretchr/testify/require"
 	"testing"
 
 	"github.com/mattermost/mattermost-server/v5/store/searchtest"
@@ -17,3 +18,47 @@ func TestPostStore(t *testing.T) {
 func TestSearchPostStore(t *testing.T) {
 	StoreTestWithSearchTestEngine(t, searchtest.TestSearchPostStore)
 }
+
+func TestMysqlStopWords(t *testing.T) {
+	mysqlStopWordsTests := []struct {
+		Name     string
+		Args     []string
+		Expected []string
+		Empty    bool
+	}{
+		{
+			Name:     "Should remove only the stop words",
+			Args:     []string{"where is my car", "so this is real", "test this-and-that is awesome"},
+			Expected: []string{"my car", "so real", "test this-and-that awesome"},
+		},
+		{
+			Name:     "Should not remove part of a word containg stop words",
+			Args:     []string{"whereabouts", "wherein", "tothis", "thisorthat", "waswhen", "whowas", "inthe", "whowill", "thewww"},
+			Expected: []string{"whereabouts", "wherein", "tothis", "thisorthat", "waswhen", "whowas", "inthe", "whowill", "thewww"},
+		},
+		{
+			Name:  "Should remove all words from terms",
+			Args:  []string{"where about", "where in", "to this", "this or that", "was when", "who was", "in the", "who will", "the www"},
+			Empty: true,
+		},
+		{
+			Name:     "Should not remove part of a word containg stop words separated by hyphens",
+			Args:     []string{"where-about", "where-in", "to-this", "this-or-that", "was-when", "who-was", "in-the", "who-will", "the-www"},
+			Expected: []string{"where-about", "where-in", "to-this", "this-or-that", "was-when", "who-was", "in-the", "who-will", "the-www"},
+		},
+	}
+
+	for _, tc := range mysqlStopWordsTests {
+		t.Run(tc.Name, func(t *testing.T) {
+			for i, term := range tc.Args {
+				got, err := removeMysqlStopWordsFromTerms(term)
+				require.NoError(t, err)
+				if tc.Empty {
+					require.Empty(t, got)
+				} else {
+					require.Equal(t, tc.Expected[i], got)
+				}
+			}
+		})
+	}
+}