Filter stop words when searching posts in mysql (#14509)

Этот коммит содержится в:
Rodrigo Villablanca
2020-08-21 11:58:17 -04:00
коммит произвёл GitHub
родитель 2055c49dfe
Коммит 6a50106cd9
5 изменённых файлов: 105 добавлений и 2 удалений

Просмотреть файл

@@ -6,6 +6,7 @@ package sqlstore
import (
"database/sql"
"fmt"
"github.com/mattermost/mattermost-server/v5/store/searchlayer"
"net/http"
"regexp"
"strconv"
@@ -1273,6 +1274,18 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search
searchClause := fmt.Sprintf("AND to_tsvector('english', %s) @@ to_tsquery('english', :Terms)", searchType)
searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", searchClause, 1)
} else if s.DriverName() == model.DATABASE_DRIVER_MYSQL {
if searchType == "Message" {
var err error
terms, err = removeMysqlStopWordsFromTerms(terms)
if err != nil {
return nil, model.NewAppError("SqlPostStore.search", "store.sql_post.search.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if terms == "" {
return list, nil
}
}
searchClause := fmt.Sprintf("AND MATCH (%s) AGAINST (:Terms IN BOOLEAN MODE)", searchType)
searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", searchClause, 1)
@@ -1318,6 +1331,25 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search
return list, nil
}
func removeMysqlStopWordsFromTerms(terms string) (string, error) {
stopWords := make([]string, len(searchlayer.MYSQL_STOP_WORDS))
copy(stopWords, searchlayer.MYSQL_STOP_WORDS)
re, err := regexp.Compile(fmt.Sprintf(`^(%s)$`, strings.Join(stopWords, "|")))
if err != nil {
return "", err
}
newTerms := make([]string, 0)
separatedTerms := strings.Fields(terms)
for _, term := range separatedTerms {
term = strings.TrimSpace(term)
if term = re.ReplaceAllString(term, ""); term != "" {
newTerms = append(newTerms, term)
}
}
return strings.Join(newTerms, " "), nil
}
func (s *SqlPostStore) AnalyticsUserCountsWithPostsByDay(teamId string) (model.AnalyticsRows, *model.AppError) {
query :=
`SELECT DISTINCT