mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

Adding the new search engine abstraction (#13304)

Просмотр исходного кода 
* WIP

* Adding bleve to go modules

* WIP

* Adding missing files from searchengine implementation

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* User and channel indexing and searches implemented

* Make bleve tests run with in-memory indexes

* Implement post index and deletion tests

* Initial commits for the search layer

* Removing unnecesary indexing

* WIP

* WIP

* More fixes for tests

* Adding the search layer

* Finishing the migration of searchers to the layer

* Removing unnecesary code

* Allowing multiple engines active at the same time

* WIP

* Add simple post search

* Print information when using bleve

* Adding some debugging to understand better how the searches are working

* Making more dynamic config of search engines

* Add post search basics

* Adding the Purge API endpoint

* Fixing bleve config updates

* Adding missed file

* Regenerating search engine mocks

* Adding missed v5 to modules imports

* fixing i18n

* Fixing some test around search engine

* Removing all bleve traces

* Cleaning up the vendors directory and go.mod/go.sum files

* Regenerating timer layer

* Adding properly the license

* Fixing govet shadow error

* Fixing some tests

* Fixing TestSearchPostsFromUser

* Fixing another test

* Fixing more tests

* Fixing more tests

* Removing SearchEngine redundant text from searchengine module code

* Fixing some reindexing problems in members updates

* Fixing tests

* Addressing PR comments

* Reverting go.mod and go.sum

* Addressing PR comments

* Fixing tests compilation

* Fixing govet

* Adding search engine stop method

* Being more explicit on where we use includeDeleted

* Adding GetSqlSupplier test helper method

* Mocking elasticsearch start function

* Fixing tests

Co-authored-by: Miguel de la Cruz <miguel@mcrx.me>
Co-authored-by: mattermod <mattermod@users.noreply.github.com>
Этот коммит содержится в:
Jesús Espino
2020-03-13 15:33:18 +01:00
коммит произвёл GitHub
родитель e2883bfe5f
Коммит c66e182b08
43 изменённых файлов: 1331 добавлений и 908 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

148
app/channel.go
Просмотреть файл

@@ -112,14 +112,6 @@ func (a *App) JoinDefaultChannels(teamId string, user *model.User, shouldBeAdmin
}
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
if err = a.indexUser(user); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
}
})
}
return err
}
@@ -189,14 +181,6 @@ func (a *App) CreateChannelWithUser(channel *model.Channel, userId string) (*mod
message.Add("team_id", channel.TeamId)
a.Publish(message)
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
if err := a.indexUser(user); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
}
})
}
return rchannel, nil
}
@@ -267,23 +251,6 @@ func (a *App) CreateChannel(channel *model.Channel, addMember bool) (*model.Chan
})
}
if a.IsESIndexingEnabled() {
if sc.Type == model.CHANNEL_OPEN {
a.Srv().Go(func() {
if err := a.Elasticsearch().IndexChannel(sc); err != nil {
mlog.Error("Encountered error indexing channel", mlog.String("channel_id", sc.Id), mlog.Err(err))
}
})
}
if addMember {
a.Srv().Go(func() {
if err := a.indexUserFromId(channel.CreatorId); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", channel.CreatorId), mlog.Err(err))
}
})
}
}
return sc, nil
}
@@ -314,16 +281,6 @@ func (a *App) GetOrCreateDirectChannel(userId, otherUserId string) (*model.Chann
})
}
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
for _, id := range []string{userId, otherUserId} {
if indexUserErr := a.indexUserFromId(id); indexUserErr != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", id), mlog.Err(indexUserErr))
}
}
})
}
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_DIRECT_ADDED, "", channel.Id, "", nil)
message.Add("teammate_id", otherUserId)
a.Publish(message)
@@ -431,16 +388,6 @@ func (a *App) CreateGroupChannel(userIds []string, creatorId string) (*model.Cha
message.Add("teammate_ids", model.ArrayToJson(userIds))
a.Publish(message)
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
for _, id := range userIds {
if err := a.indexUserFromId(id); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", id), mlog.Err(err))
}
}
})
}
return channel, nil
}
@@ -536,14 +483,6 @@ func (a *App) UpdateChannel(channel *model.Channel) (*model.Channel, *model.AppE
messageWs.Add("channel", channel.ToJson())
a.Publish(messageWs)
if a.IsESIndexingEnabled() && channel.Type == model.CHANNEL_OPEN {
a.Srv().Go(func() {
if err := a.Elasticsearch().IndexChannel(channel); err != nil {
mlog.Error("Encountered error indexing channel", mlog.String("channel_id", channel.Id), mlog.Err(err))
}
})
}
return channel, nil
}
@@ -1243,14 +1182,6 @@ func (a *App) AddChannelMember(userId string, channel *model.Channel, userReques
})
}
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
if err := a.indexUser(user); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
}
})
}
if userRequestorId == "" || userId == userRequestorId {
a.postJoinChannelMessage(user, channel)
} else {
@@ -1639,14 +1570,6 @@ func (a *App) JoinChannel(channel *model.Channel, userId string) *model.AppError
})
}
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
if err := a.indexUser(user); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
}
})
}
if err := a.postJoinChannelMessage(user, channel); err != nil {
return err
}
@@ -1927,14 +1850,6 @@ func (a *App) removeUserFromChannel(userIdToRemove string, removerUserId string,
})
}
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
if err := a.indexUserFromId(userIdToRemove); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", userIdToRemove), mlog.Err(err))
}
})
}
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_USER_REMOVED, "", channel.Id, "", nil)
message.Add("user_id", userIdToRemove)
message.Add("remover_id", removerUserId)
@@ -2056,50 +1971,11 @@ func (a *App) MarkChannelAsUnreadFromPost(postID string, userID string) (*model.
return channelUnread, nil
}
func (a *App) esAutocompleteChannels(teamId, term string, includeDeleted bool) (*model.ChannelList, *model.AppError) {
channelIds, err := a.Elasticsearch().SearchChannels(teamId, term)
if err != nil {
return nil, err
}
channelList := model.ChannelList{}
if len(channelIds) > 0 {
channels, err := a.Srv().Store.Channel().GetChannelsByIds(channelIds)
if err != nil {
return nil, err
}
for _, c := range channels {
if c.DeleteAt > 0 && !includeDeleted {
continue
}
channelList = append(channelList, c)
}
}
return &channelList, nil
}
func (a *App) AutocompleteChannels(teamId string, term string) (*model.ChannelList, *model.AppError) {
includeDeleted := *a.Config().TeamSettings.ExperimentalViewArchivedChannels
var channelList *model.ChannelList
var err *model.AppError
term = strings.TrimSpace(term)
if a.IsESAutocompletionEnabled() {
channelList, err = a.esAutocompleteChannels(teamId, term, includeDeleted)
if err != nil {
mlog.Error("Encountered error on AutocompleteChannels through Elasticsearch. Falling back to default autocompletion.", mlog.Err(err))
}
}
if !a.IsESAutocompletionEnabled() || err != nil {
channelList, err = a.Srv().Store.Channel().AutocompleteInTeam(teamId, term, includeDeleted)
if err != nil {
return nil, err
}
}
return channelList, nil
return a.Srv().Store.Channel().AutocompleteInTeam(teamId, term, includeDeleted)
}
func (a *App) AutocompleteChannelsForSearch(teamId string, userId string, term string) (*model.ChannelList, *model.AppError) {
@@ -2246,11 +2122,6 @@ func (a *App) ViewChannel(view *model.ChannelView, userId string, currentSession
}
func (a *App) PermanentDeleteChannel(channel *model.Channel) *model.AppError {
profiles, err := a.Srv().Store.User().GetAllProfilesInChannel(channel.Id, false)
if err != nil {
return err
}
if err := a.Srv().Store.Post().PermanentDeleteByChannel(channel.Id); err != nil {
return err
}
@@ -2271,23 +2142,6 @@ func (a *App) PermanentDeleteChannel(channel *model.Channel) *model.AppError {
return err
}
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
for _, user := range profiles {
if err := a.indexUser(user); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
}
}
})
if channel.Type == model.CHANNEL_OPEN {
a.Srv().Go(func() {
if err := a.Elasticsearch().DeleteChannel(channel); err != nil {
mlog.Error("Encountered error deleting channel", mlog.String("channel_id", channel.Id), mlog.Err(err))
}
})
}
}
return nil
}