Adding the new search engine abstraction (#13304)
* WIP * Adding bleve to go modules * WIP * Adding missing files from searchengine implementation * WIP * WIP * WIP * WIP * WIP * WIP * User and channel indexing and searches implemented * Make bleve tests run with in-memory indexes * Implement post index and deletion tests * Initial commits for the search layer * Removing unnecesary indexing * WIP * WIP * More fixes for tests * Adding the search layer * Finishing the migration of searchers to the layer * Removing unnecesary code * Allowing multiple engines active at the same time * WIP * Add simple post search * Print information when using bleve * Adding some debugging to understand better how the searches are working * Making more dynamic config of search engines * Add post search basics * Adding the Purge API endpoint * Fixing bleve config updates * Adding missed file * Regenerating search engine mocks * Adding missed v5 to modules imports * fixing i18n * Fixing some test around search engine * Removing all bleve traces * Cleaning up the vendors directory and go.mod/go.sum files * Regenerating timer layer * Adding properly the license * Fixing govet shadow error * Fixing some tests * Fixing TestSearchPostsFromUser * Fixing another test * Fixing more tests * Fixing more tests * Removing SearchEngine redundant text from searchengine module code * Fixing some reindexing problems in members updates * Fixing tests * Addressing PR comments * Reverting go.mod and go.sum * Addressing PR comments * Fixing tests compilation * Fixing govet * Adding search engine stop method * Being more explicit on where we use includeDeleted * Adding GetSqlSupplier test helper method * Mocking elasticsearch start function * Fixing tests Co-authored-by: Miguel de la Cruz <miguel@mcrx.me> Co-authored-by: mattermod <mattermod@users.noreply.github.com>
Этот коммит содержится в:
Jesús Espino
коммит произвёл
GitHub
GitHub
родитель
e2883bfe5f
Коммит
c66e182b08
@@ -1865,9 +1865,12 @@ func (s SqlChannelStore) GetAll(teamId string) ([]*model.Channel, *model.AppErro
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (s SqlChannelStore) GetChannelsByIds(channelIds []string) ([]*model.Channel, *model.AppError) {
|
||||
func (s SqlChannelStore) GetChannelsByIds(channelIds []string, includeDeleted bool) ([]*model.Channel, *model.AppError) {
|
||||
keys, params := MapStringsToQueryParams(channelIds, "Channel")
|
||||
query := `SELECT * FROM Channels WHERE Id IN ` + keys + ` ORDER BY Name`
|
||||
if !includeDeleted {
|
||||
query = `SELECT * FROM Channels WHERE DeleteAt=0 AND Id IN ` + keys + ` ORDER BY Name`
|
||||
}
|
||||
|
||||
var channels []*model.Channel
|
||||
_, err := s.GetReplica().Select(&channels, query, params)
|
||||
|
||||
@@ -999,7 +999,7 @@ func (s *SqlPostStore) buildCreateDateFilterClause(params *model.SearchParams, q
|
||||
return searchQuery, queryParams
|
||||
}
|
||||
|
||||
func (s *SqlPostStore) buildSearchChannelFilterClause(channels []string, paramPrefix string, exclusion bool, queryParams map[string]interface{}) (string, map[string]interface{}) {
|
||||
func (s *SqlPostStore) buildSearchChannelFilterClause(channels []string, paramPrefix string, exclusion bool, queryParams map[string]interface{}, byName bool) (string, map[string]interface{}) {
|
||||
if len(channels) == 0 {
|
||||
return "", queryParams
|
||||
}
|
||||
@@ -1011,13 +1011,20 @@ func (s *SqlPostStore) buildSearchChannelFilterClause(channels []string, paramPr
|
||||
queryParams[paramName] = channel
|
||||
}
|
||||
clause := strings.Join(clauseSlice, ", ")
|
||||
if exclusion {
|
||||
return "AND Name NOT IN (" + clause + ")", queryParams
|
||||
if byName {
|
||||
if exclusion {
|
||||
return "AND Name NOT IN (" + clause + ")", queryParams
|
||||
}
|
||||
return "AND Name IN (" + clause + ")", queryParams
|
||||
}
|
||||
return "AND Name IN (" + clause + ")", queryParams
|
||||
|
||||
if exclusion {
|
||||
return "AND Id NOT IN (" + clause + ")", queryParams
|
||||
}
|
||||
return "AND Id IN (" + clause + ")", queryParams
|
||||
}
|
||||
|
||||
func (s *SqlPostStore) buildSearchUserFilterClause(users []string, paramPrefix string, exclusion bool, queryParams map[string]interface{}) (string, map[string]interface{}) {
|
||||
func (s *SqlPostStore) buildSearchUserFilterClause(users []string, paramPrefix string, exclusion bool, queryParams map[string]interface{}, byUsername bool) (string, map[string]interface{}) {
|
||||
if len(users) == 0 {
|
||||
return "", queryParams
|
||||
}
|
||||
@@ -1028,13 +1035,19 @@ func (s *SqlPostStore) buildSearchUserFilterClause(users []string, paramPrefix s
|
||||
queryParams[paramName] = user
|
||||
}
|
||||
clause := strings.Join(clauseSlice, ", ")
|
||||
if exclusion {
|
||||
return "AND Username NOT IN (" + clause + ")", queryParams
|
||||
if byUsername {
|
||||
if exclusion {
|
||||
return "AND Username NOT IN (" + clause + ")", queryParams
|
||||
}
|
||||
return "AND Username IN (" + clause + ")", queryParams
|
||||
}
|
||||
return "AND Username IN (" + clause + ")", queryParams
|
||||
if exclusion {
|
||||
return "AND Id NOT IN (" + clause + ")", queryParams
|
||||
}
|
||||
return "AND Id IN (" + clause + ")", queryParams
|
||||
}
|
||||
|
||||
func (s *SqlPostStore) buildSearchPostFilterClause(fromUsers []string, excludedUsers []string, queryParams map[string]interface{}) (string, map[string]interface{}) {
|
||||
func (s *SqlPostStore) buildSearchPostFilterClause(fromUsers []string, excludedUsers []string, queryParams map[string]interface{}, userByUsername bool) (string, map[string]interface{}) {
|
||||
if len(fromUsers) == 0 && len(excludedUsers) == 0 {
|
||||
return "", queryParams
|
||||
}
|
||||
@@ -1052,16 +1065,20 @@ func (s *SqlPostStore) buildSearchPostFilterClause(fromUsers []string, excludedU
|
||||
FROM_USER_FILTER
|
||||
EXCLUDED_USER_FILTER)`
|
||||
|
||||
fromUserClause, queryParams := s.buildSearchUserFilterClause(fromUsers, "FromUser", false, queryParams)
|
||||
fromUserClause, queryParams := s.buildSearchUserFilterClause(fromUsers, "FromUser", false, queryParams, userByUsername)
|
||||
filterQuery = strings.Replace(filterQuery, "FROM_USER_FILTER", fromUserClause, 1)
|
||||
|
||||
excludedUserClause, queryParams := s.buildSearchUserFilterClause(excludedUsers, "ExcludedUser", true, queryParams)
|
||||
excludedUserClause, queryParams := s.buildSearchUserFilterClause(excludedUsers, "ExcludedUser", true, queryParams, userByUsername)
|
||||
filterQuery = strings.Replace(filterQuery, "EXCLUDED_USER_FILTER", excludedUserClause, 1)
|
||||
|
||||
return filterQuery, queryParams
|
||||
}
|
||||
|
||||
func (s *SqlPostStore) Search(teamId string, userId string, params *model.SearchParams) (*model.PostList, *model.AppError) {
|
||||
return s.search(teamId, userId, params, true, true)
|
||||
}
|
||||
|
||||
func (s *SqlPostStore) search(teamId string, userId string, params *model.SearchParams, channelsByName bool, userByUsername bool) (*model.PostList, *model.AppError) {
|
||||
queryParams := map[string]interface{}{
|
||||
"TeamId": teamId,
|
||||
"UserId": userId,
|
||||
@@ -1114,13 +1131,13 @@ func (s *SqlPostStore) Search(teamId string, userId string, params *model.Search
|
||||
ORDER BY CreateAt DESC
|
||||
LIMIT 100`
|
||||
|
||||
inChannelClause, queryParams := s.buildSearchChannelFilterClause(params.InChannels, "InChannel", false, queryParams)
|
||||
inChannelClause, queryParams := s.buildSearchChannelFilterClause(params.InChannels, "InChannel", false, queryParams, channelsByName)
|
||||
searchQuery = strings.Replace(searchQuery, "IN_CHANNEL_FILTER", inChannelClause, 1)
|
||||
|
||||
excludedChannelClause, queryParams := s.buildSearchChannelFilterClause(params.ExcludedChannels, "ExcludedChannel", true, queryParams)
|
||||
excludedChannelClause, queryParams := s.buildSearchChannelFilterClause(params.ExcludedChannels, "ExcludedChannel", true, queryParams, channelsByName)
|
||||
searchQuery = strings.Replace(searchQuery, "EXCLUDED_CHANNEL_FILTER", excludedChannelClause, 1)
|
||||
|
||||
postFilterClause, queryParams := s.buildSearchPostFilterClause(params.FromUsers, params.ExcludedUsers, queryParams)
|
||||
postFilterClause, queryParams := s.buildSearchPostFilterClause(params.FromUsers, params.ExcludedUsers, queryParams, userByUsername)
|
||||
searchQuery = strings.Replace(searchQuery, "POST_FILTER", postFilterClause, 1)
|
||||
|
||||
createDateFilterClause, queryParams := s.buildCreateDateFilterClause(params, queryParams)
|
||||
@@ -1666,3 +1683,49 @@ func (s *SqlPostStore) GetDirectPostParentsForExportAfter(limit int, afterId str
|
||||
}
|
||||
return posts, nil
|
||||
}
|
||||
|
||||
func (s *SqlPostStore) SearchPostsInTeamForUser(paramsList []*model.SearchParams, userId, teamId string, isOrSearch, includeDeletedChannels bool, page, perPage int) (*model.PostSearchResults, *model.AppError) {
|
||||
// Since we don't support paging for DB search, we just return nothing for later pages
|
||||
if page > 0 {
|
||||
return model.MakePostSearchResults(model.NewPostList(), nil), nil
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
pchan := make(chan store.StoreResult, len(paramsList))
|
||||
|
||||
for _, params := range paramsList {
|
||||
// Don't allow users to search for everything.
|
||||
if params.Terms == "*" {
|
||||
continue
|
||||
}
|
||||
|
||||
params.IncludeDeletedChannels = includeDeletedChannels
|
||||
params.OrTerms = isOrSearch
|
||||
|
||||
wg.Add(1)
|
||||
|
||||
go func(params *model.SearchParams) {
|
||||
defer wg.Done()
|
||||
postList, err := s.search(teamId, userId, params, false, false)
|
||||
pchan <- store.StoreResult{Data: postList, Err: err}
|
||||
}(params)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(pchan)
|
||||
|
||||
posts := model.NewPostList()
|
||||
|
||||
for result := range pchan {
|
||||
if result.Err != nil {
|
||||
return nil, result.Err
|
||||
}
|
||||
data := result.Data.(*model.PostList)
|
||||
posts.Extend(data)
|
||||
}
|
||||
|
||||
posts.SortByCreateAt()
|
||||
|
||||
return model.MakePostSearchResults(posts, nil), nil
|
||||
}
|
||||
|
||||
@@ -1716,3 +1716,35 @@ func (us SqlUserStore) DemoteUserToGuest(userId string) *model.AppError {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (us SqlUserStore) AutocompleteUsersInChannel(teamId, channelId, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) {
|
||||
autocomplete := &model.UserAutocompleteInChannel{}
|
||||
uchan := make(chan store.StoreResult, 1)
|
||||
go func() {
|
||||
users, err := us.SearchInChannel(channelId, term, options)
|
||||
uchan <- store.StoreResult{Data: users, Err: err}
|
||||
close(uchan)
|
||||
}()
|
||||
|
||||
nuchan := make(chan store.StoreResult, 1)
|
||||
go func() {
|
||||
users, err := us.SearchNotInChannel(teamId, channelId, term, options)
|
||||
nuchan <- store.StoreResult{Data: users, Err: err}
|
||||
close(nuchan)
|
||||
}()
|
||||
|
||||
result := <-uchan
|
||||
if result.Err != nil {
|
||||
return nil, result.Err
|
||||
}
|
||||
users := result.Data.([]*model.User)
|
||||
autocomplete.InChannel = users
|
||||
|
||||
result = <-nuchan
|
||||
if result.Err != nil {
|
||||
return nil, result.Err
|
||||
}
|
||||
users = result.Data.([]*model.User)
|
||||
autocomplete.OutOfChannel = users
|
||||
return autocomplete, nil
|
||||
}
|
||||
|
||||
Ссылка в новой задаче
Block a user