Adding the new search engine abstraction (#13304)
* WIP * Adding bleve to go modules * WIP * Adding missing files from searchengine implementation * WIP * WIP * WIP * WIP * WIP * WIP * User and channel indexing and searches implemented * Make bleve tests run with in-memory indexes * Implement post index and deletion tests * Initial commits for the search layer * Removing unnecesary indexing * WIP * WIP * More fixes for tests * Adding the search layer * Finishing the migration of searchers to the layer * Removing unnecesary code * Allowing multiple engines active at the same time * WIP * Add simple post search * Print information when using bleve * Adding some debugging to understand better how the searches are working * Making more dynamic config of search engines * Add post search basics * Adding the Purge API endpoint * Fixing bleve config updates * Adding missed file * Regenerating search engine mocks * Adding missed v5 to modules imports * fixing i18n * Fixing some test around search engine * Removing all bleve traces * Cleaning up the vendors directory and go.mod/go.sum files * Regenerating timer layer * Adding properly the license * Fixing govet shadow error * Fixing some tests * Fixing TestSearchPostsFromUser * Fixing another test * Fixing more tests * Fixing more tests * Removing SearchEngine redundant text from searchengine module code * Fixing some reindexing problems in members updates * Fixing tests * Addressing PR comments * Reverting go.mod and go.sum * Addressing PR comments * Fixing tests compilation * Fixing govet * Adding search engine stop method * Being more explicit on where we use includeDeleted * Adding GetSqlSupplier test helper method * Mocking elasticsearch start function * Fixing tests Co-authored-by: Miguel de la Cruz <miguel@mcrx.me> Co-authored-by: mattermod <mattermod@users.noreply.github.com>
Этот коммит содержится в:
Jesús Espino
коммит произвёл
GitHub
GitHub
родитель
e2883bfe5f
Коммит
c66e182b08
122
app/post.go
122
app/post.go
@@ -297,14 +297,6 @@ func (a *App) CreatePost(post *model.Post, channel *model.Channel, triggerWebhoo
|
||||
})
|
||||
}
|
||||
|
||||
if a.IsESIndexingEnabled() {
|
||||
a.Srv().Go(func() {
|
||||
if err = a.Elasticsearch().IndexPost(rpost, channel.TeamId); err != nil {
|
||||
mlog.Error("Encountered error indexing post", mlog.String("post_id", post.Id), mlog.Err(err))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if a.Metrics() != nil {
|
||||
a.Metrics().IncrementPostCreate()
|
||||
}
|
||||
@@ -585,19 +577,6 @@ func (a *App) UpdatePost(post *model.Post, safeUpdate bool) (*model.Post, *model
|
||||
})
|
||||
}
|
||||
|
||||
if a.IsESIndexingEnabled() {
|
||||
a.Srv().Go(func() {
|
||||
channel, chanErr := a.Srv().Store.Channel().GetForPost(rpost.Id)
|
||||
if chanErr != nil {
|
||||
mlog.Error("Couldn't get channel for post for Elasticsearch indexing.", mlog.String("channel_id", rpost.ChannelId), mlog.String("post_id", rpost.Id))
|
||||
return
|
||||
}
|
||||
if err := a.Elasticsearch().IndexPost(rpost, channel.TeamId); err != nil {
|
||||
mlog.Error("Encountered error indexing post", mlog.String("post_id", post.Id), mlog.Err(err))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
rpost = a.PreparePostForClient(rpost, false, true)
|
||||
|
||||
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_EDITED, "", rpost.ChannelId, "", nil)
|
||||
@@ -870,14 +849,6 @@ func (a *App) DeletePost(postId, deleteByID string) (*model.Post, *model.AppErro
|
||||
a.DeleteFlaggedPosts(post.Id)
|
||||
})
|
||||
|
||||
if a.IsESIndexingEnabled() {
|
||||
a.Srv().Go(func() {
|
||||
if err := a.Elasticsearch().DeletePost(post); err != nil {
|
||||
mlog.Error("Encountered error deleting post", mlog.String("post_id", post.Id), mlog.Err(err))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
a.invalidateCacheForChannelPosts(post.ChannelId)
|
||||
|
||||
return post, nil
|
||||
@@ -1006,10 +977,18 @@ func (a *App) SearchPostsInTeam(teamId string, paramsList []*model.SearchParams)
|
||||
})
|
||||
}
|
||||
|
||||
func (a *App) esSearchPostsInTeamForUser(paramsList []*model.SearchParams, userId, teamId string, isOrSearch, includeDeletedChannels bool, page, perPage int) (*model.PostSearchResults, *model.AppError) {
|
||||
finalParamsList := []*model.SearchParams{}
|
||||
func (a *App) SearchPostsInTeamForUser(terms string, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, timeZoneOffset int, page, perPage int) (*model.PostSearchResults, *model.AppError) {
|
||||
var postSearchResults *model.PostSearchResults
|
||||
var err *model.AppError
|
||||
paramsList := model.ParseSearchParams(strings.TrimSpace(terms), timeZoneOffset)
|
||||
includeDeleted := includeDeletedChannels && *a.Config().TeamSettings.ExperimentalViewArchivedChannels
|
||||
|
||||
if !*a.Config().ServiceSettings.EnablePostSearch {
|
||||
return nil, model.NewAppError("SearchPostsInTeamForUser", "store.sql_post.search.disabled", nil, fmt.Sprintf("teamId=%v userId=%v", teamId, userId), http.StatusNotImplemented)
|
||||
}
|
||||
|
||||
finalParamsList := []*model.SearchParams{}
|
||||
|
||||
for _, params := range paramsList {
|
||||
params.OrTerms = isOrSearch
|
||||
// Don't allow users to search for "*"
|
||||
@@ -1031,90 +1010,11 @@ func (a *App) esSearchPostsInTeamForUser(paramsList []*model.SearchParams, userI
|
||||
return model.MakePostSearchResults(model.NewPostList(), nil), nil
|
||||
}
|
||||
|
||||
// We only allow the user to search in channels they are a member of.
|
||||
userChannels, err := a.GetChannelsForUser(teamId, userId, includeDeleted)
|
||||
if err != nil {
|
||||
mlog.Error("error getting channel for user", mlog.Err(err))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
postIds, matches, err := a.Elasticsearch().SearchPosts(userChannels, finalParamsList, page, perPage)
|
||||
postSearchResults, err = a.Srv().Store.Post().SearchPostsInTeamForUser(finalParamsList, userId, teamId, isOrSearch, includeDeleted, page, perPage)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Get the posts
|
||||
postList := model.NewPostList()
|
||||
if len(postIds) > 0 {
|
||||
posts, err := a.Srv().Store.Post().GetPostsByIds(postIds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range posts {
|
||||
if p.DeleteAt == 0 {
|
||||
postList.AddPost(p)
|
||||
postList.AddOrder(p.Id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return model.MakePostSearchResults(postList, matches), nil
|
||||
}
|
||||
|
||||
func (a *App) SearchPostsInTeamForUser(terms string, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, timeZoneOffset int, page, perPage int) (*model.PostSearchResults, *model.AppError) {
|
||||
var postSearchResults *model.PostSearchResults
|
||||
var err *model.AppError
|
||||
paramsList := model.ParseSearchParams(strings.TrimSpace(terms), timeZoneOffset)
|
||||
|
||||
if !*a.Config().ServiceSettings.EnablePostSearch {
|
||||
return nil, model.NewAppError("SearchPostsInTeamForUser", "store.sql_post.search.disabled", nil, fmt.Sprintf("teamId=%v userId=%v", teamId, userId), http.StatusNotImplemented)
|
||||
}
|
||||
|
||||
if a.IsESSearchEnabled() {
|
||||
postSearchResults, err = a.esSearchPostsInTeamForUser(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage)
|
||||
if err != nil {
|
||||
mlog.Error("Encountered error on SearchPostsInTeamForUser through Elasticsearch. Falling back to default search.", mlog.Err(err))
|
||||
}
|
||||
}
|
||||
|
||||
if !a.IsESSearchEnabled() || err != nil {
|
||||
// Since we don't support paging for DB search, we just return nothing for later pages
|
||||
if page > 0 {
|
||||
return model.MakePostSearchResults(model.NewPostList(), nil), nil
|
||||
}
|
||||
|
||||
includeDeleted := includeDeletedChannels && *a.Config().TeamSettings.ExperimentalViewArchivedChannels
|
||||
posts, err := a.searchPostsInTeam(teamId, userId, paramsList, func(params *model.SearchParams) {
|
||||
params.IncludeDeletedChannels = includeDeleted
|
||||
params.OrTerms = isOrSearch
|
||||
for idx, channelName := range params.InChannels {
|
||||
if strings.HasPrefix(channelName, "@") {
|
||||
channel, err := a.parseAndFetchChannelIdByNameFromInFilter(channelName, userId, teamId, includeDeletedChannels)
|
||||
if err != nil {
|
||||
mlog.Error("error getting channel_id by name from in filter", mlog.Err(err))
|
||||
continue
|
||||
}
|
||||
params.InChannels[idx] = channel.Name
|
||||
}
|
||||
}
|
||||
for idx, channelName := range params.ExcludedChannels {
|
||||
if strings.HasPrefix(channelName, "@") {
|
||||
channel, err := a.parseAndFetchChannelIdByNameFromInFilter(channelName, userId, teamId, includeDeletedChannels)
|
||||
if err != nil {
|
||||
mlog.Error("error getting channel_id by name from in filter", mlog.Err(err))
|
||||
continue
|
||||
}
|
||||
params.ExcludedChannels[idx] = channel.Name
|
||||
}
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
postSearchResults = model.MakePostSearchResults(posts, nil)
|
||||
}
|
||||
|
||||
return postSearchResults, nil
|
||||
}
|
||||
|
||||
|
||||
Ссылка в новой задаче
Block a user