From c66e182b081792524de285ddd323afa1e22a326e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Fri, 13 Mar 2020 15:33:18 +0100 Subject: [PATCH] Adding the new search engine abstraction (#13304) * WIP * Adding bleve to go modules * WIP * Adding missing files from searchengine implementation * WIP * WIP * WIP * WIP * WIP * WIP * User and channel indexing and searches implemented * Make bleve tests run with in-memory indexes * Implement post index and deletion tests * Initial commits for the search layer * Removing unnecesary indexing * WIP * WIP * More fixes for tests * Adding the search layer * Finishing the migration of searchers to the layer * Removing unnecesary code * Allowing multiple engines active at the same time * WIP * Add simple post search * Print information when using bleve * Adding some debugging to understand better how the searches are working * Making more dynamic config of search engines * Add post search basics * Adding the Purge API endpoint * Fixing bleve config updates * Adding missed file * Regenerating search engine mocks * Adding missed v5 to modules imports * fixing i18n * Fixing some test around search engine * Removing all bleve traces * Cleaning up the vendors directory and go.mod/go.sum files * Regenerating timer layer * Adding properly the license * Fixing govet shadow error * Fixing some tests * Fixing TestSearchPostsFromUser * Fixing another test * Fixing more tests * Fixing more tests * Removing SearchEngine redundant text from searchengine module code * Fixing some reindexing problems in members updates * Fixing tests * Addressing PR comments * Reverting go.mod and go.sum * Addressing PR comments * Fixing tests compilation * Fixing govet * Adding search engine stop method * Being more explicit on where we use includeDeleted * Adding GetSqlSupplier test helper method * Mocking elasticsearch start function * Fixing tests Co-authored-by: Miguel de la Cruz Co-authored-by: mattermod --- Makefile | 8 +- app/app.go | 7 +- app/app_iface.go | 10 +- app/channel.go | 148 +------- app/config.go | 16 - app/config_test.go | 4 +- app/diagnostics.go | 6 +- app/enterprise.go | 12 +- app/helper_test.go | 10 + .../opentracing_layer.go.tmpl | 9 +- app/opentracing_layer.go | 98 ++---- app/options.go | 6 +- app/post.go | 122 +------ app/post_test.go | 46 ++- app/{elasticsearch.go => searchengine.go} | 12 +- app/server.go | 98 ++++-- app/server_app_adapters.go | 16 +- app/team.go | 13 +- app/user.go | 333 +----------------- app/user_test.go | 84 ----- cmd/mattermost/commands/init.go | 2 +- cmd/mattermost/commands/server.go | 2 +- jobs/interfaces/searchengine.go | 17 + model/user_search.go | 2 + .../searchengine/interface.go | 10 +- .../mocks/SearchEngineInterface.go | 113 +++++- services/searchengine/searchengine.go | 43 +++ store/opentracing_layer.go | 40 ++- store/searchlayer/channel_layer.go | 207 +++++++++++ store/searchlayer/layer.go | 93 +++++ store/searchlayer/post_layer.go | 126 +++++++ store/searchlayer/team_layer.go | 46 +++ store/searchlayer/user_layer.go | 192 ++++++++++ store/sqlstore/channel_store.go | 5 +- store/sqlstore/post_store.go | 91 ++++- store/sqlstore/user_store.go | 32 ++ store/store.go | 4 +- store/storetest/channel_store.go | 45 ++- store/storetest/mocks/ChannelStore.go | 14 +- store/storetest/mocks/PostStore.go | 25 ++ store/storetest/mocks/UserStore.go | 25 ++ store/timer_layer.go | 36 +- testlib/helper.go | 11 +- 43 files changed, 1331 insertions(+), 908 deletions(-) rename app/{elasticsearch.go => searchengine.go} (84%) create mode 100644 jobs/interfaces/searchengine.go rename einterfaces/elasticsearch.go => services/searchengine/interface.go (85%) rename einterfaces/mocks/ElasticsearchInterface.go => services/searchengine/mocks/SearchEngineInterface.go (68%) create mode 100644 services/searchengine/searchengine.go create mode 100644 store/searchlayer/channel_layer.go create mode 100644 store/searchlayer/layer.go create mode 100644 store/searchlayer/post_layer.go create mode 100644 store/searchlayer/team_layer.go create mode 100644 store/searchlayer/user_layer.go diff --git a/Makefile b/Makefile index d2a4392f53..b1c2582f7c 100644 --- a/Makefile +++ b/Makefile @@ -201,8 +201,8 @@ endif app-layers: ## Extract interface from App struct # The following commented commands can be used to re-generate the AppIface from the App struct - # env GO111MODULE=off $(GO) get gopkg.in/reflog/struct2interface.v0 - # $(GOBIN)/struct2interface.v0 -f "app" -o "app/app_iface.go" -p "app" -s "App" -i "AppIface" -t ./app/layer_generators/app_iface.go.tmpl + #env GO111MODULE=off $(GO) get gopkg.in/reflog/struct2interface.v0 + #$(GOBIN)/struct2interface.v0 -f "app" -o "app/app_iface.go" -p "app" -s "App" -i "AppIface" -t ./app/layer_generators/app_iface.go.tmpl $(GO) run ./app/layer_generators -in ./app/app_iface.go -out ./app/opentracing_layer.go -template ./app/layer_generators/opentracing_layer.go.tmpl i18n-extract: ## Extract strings for translation from the source code @@ -234,6 +234,10 @@ einterfaces-mocks: ## Creates mock files for einterfaces. env GO111MODULE=off $(GO) get -u github.com/vektra/mockery/... $(GOBIN)/mockery -dir einterfaces -all -output einterfaces/mocks -note 'Regenerate this file using `make einterfaces-mocks`.' +searchengine-mocks: ## Creates mock files for searchengines. + env GO111MODULE=off go get -u github.com/vektra/mockery/... + $(GOPATH)/bin/mockery -dir services/searchengine -all -output services/searchengine/mocks -note 'Regenerate this file using `make searchengine-mocks`.' + pluginapi: ## Generates api and hooks glue code for plugins $(GO) generate $(GOFLAGS) ./plugin diff --git a/app/app.go b/app/app.go index 35acbfc00c..78cda50f28 100644 --- a/app/app.go +++ b/app/app.go @@ -16,6 +16,7 @@ import ( "github.com/mattermost/mattermost-server/v5/model" "github.com/mattermost/mattermost-server/v5/services/httpservice" "github.com/mattermost/mattermost-server/v5/services/imageproxy" + "github.com/mattermost/mattermost-server/v5/services/searchengine" "github.com/mattermost/mattermost-server/v5/services/timezones" "github.com/mattermost/mattermost-server/v5/utils" ) @@ -38,7 +39,7 @@ type App struct { cluster einterfaces.ClusterInterface compliance einterfaces.ComplianceInterface dataRetention einterfaces.DataRetentionInterface - elasticsearch einterfaces.ElasticsearchInterface + searchEngine *searchengine.Broker ldap einterfaces.LdapInterface messageExport einterfaces.MessageExportInterface metrics einterfaces.MetricsInterface @@ -202,8 +203,8 @@ func (a *App) Compliance() einterfaces.ComplianceInterface { func (a *App) DataRetention() einterfaces.DataRetentionInterface { return a.dataRetention } -func (a *App) Elasticsearch() einterfaces.ElasticsearchInterface { - return a.elasticsearch +func (a *App) SearchEngine() *searchengine.Broker { + return a.searchEngine } func (a *App) Ldap() einterfaces.LdapInterface { return a.ldap diff --git a/app/app_iface.go b/app/app_iface.go index 5b8566712a..c69b1f7804 100644 --- a/app/app_iface.go +++ b/app/app_iface.go @@ -29,16 +29,13 @@ import ( "github.com/mattermost/mattermost-server/v5/services/filesstore" "github.com/mattermost/mattermost-server/v5/services/httpservice" "github.com/mattermost/mattermost-server/v5/services/imageproxy" + "github.com/mattermost/mattermost-server/v5/services/searchengine" "github.com/mattermost/mattermost-server/v5/services/timezones" "github.com/mattermost/mattermost-server/v5/store" ) // AppIface is extracted from App struct and contains all it's exported methods. It's provided to allow partial interface passing and app layers creation. type AppIface interface { - // GetViewUsersRestrictionsForTeam returns a list with the channel ids that the user has permissions to view on a - // team. If the result is an empty list, the user can't view any channel; if it's - // nil, there are no restrictions for the user in the specified team. - GetViewUsersRestrictionsForTeam(userId string, teamId string) ([]string, *model.AppError) // @openTracingParams teamId // previous ListCommands now ListAutocompleteCommands ListAutocompleteCommands(teamId string, T goi18n.TranslateFunc) ([]*model.Command, *model.AppError) @@ -413,7 +410,6 @@ type AppIface interface { DoUploadFile(now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) DoUploadFileExpectModification(now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte) (*model.FileInfo, []byte, *model.AppError) DownloadFromURL(downloadURL string) ([]byte, error) - Elasticsearch() einterfaces.ElasticsearchInterface EnableUserAccessToken(token *model.UserAccessToken) *model.AppError EnsureDiagnosticId() EnvironmentConfig() map[string]interface{} @@ -677,9 +673,6 @@ type AppIface interface { InviteNewUsersToTeam(emailList []string, teamId, senderId string) *model.AppError InviteNewUsersToTeamGracefully(emailList []string, teamId, senderId string) ([]*model.EmailInviteWithError, *model.AppError) IpAddress() string - IsESAutocompletionEnabled() bool - IsESIndexingEnabled() bool - IsESSearchEnabled() bool IsFirstUserAccount() bool IsLeader() bool IsPasswordValid(password string) *model.AppError @@ -795,6 +788,7 @@ type AppIface interface { SearchChannelsForUser(userId, teamId, term string) (*model.ChannelList, *model.AppError) SearchChannelsUserNotIn(teamId string, userId string, term string) (*model.ChannelList, *model.AppError) SearchEmoji(name string, prefixOnly bool, limit int) ([]*model.Emoji, *model.AppError) + SearchEngine() *searchengine.Broker SearchGroupChannels(userId, term string) (*model.ChannelList, *model.AppError) SearchPostsInTeam(teamId string, paramsList []*model.SearchParams) (*model.PostList, *model.AppError) SearchPostsInTeamForUser(terms string, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, timeZoneOffset int, page, perPage int) (*model.PostSearchResults, *model.AppError) diff --git a/app/channel.go b/app/channel.go index 2cfd9e2077..32950de98d 100644 --- a/app/channel.go +++ b/app/channel.go @@ -112,14 +112,6 @@ func (a *App) JoinDefaultChannels(teamId string, user *model.User, shouldBeAdmin } - if a.IsESIndexingEnabled() { - a.Srv().Go(func() { - if err = a.indexUser(user); err != nil { - mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err)) - } - }) - } - return err } @@ -189,14 +181,6 @@ func (a *App) CreateChannelWithUser(channel *model.Channel, userId string) (*mod message.Add("team_id", channel.TeamId) a.Publish(message) - if a.IsESIndexingEnabled() { - a.Srv().Go(func() { - if err := a.indexUser(user); err != nil { - mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err)) - } - }) - } - return rchannel, nil } @@ -267,23 +251,6 @@ func (a *App) CreateChannel(channel *model.Channel, addMember bool) (*model.Chan }) } - if a.IsESIndexingEnabled() { - if sc.Type == model.CHANNEL_OPEN { - a.Srv().Go(func() { - if err := a.Elasticsearch().IndexChannel(sc); err != nil { - mlog.Error("Encountered error indexing channel", mlog.String("channel_id", sc.Id), mlog.Err(err)) - } - }) - } - if addMember { - a.Srv().Go(func() { - if err := a.indexUserFromId(channel.CreatorId); err != nil { - mlog.Error("Encountered error indexing user", mlog.String("user_id", channel.CreatorId), mlog.Err(err)) - } - }) - } - } - return sc, nil } @@ -314,16 +281,6 @@ func (a *App) GetOrCreateDirectChannel(userId, otherUserId string) (*model.Chann }) } - if a.IsESIndexingEnabled() { - a.Srv().Go(func() { - for _, id := range []string{userId, otherUserId} { - if indexUserErr := a.indexUserFromId(id); indexUserErr != nil { - mlog.Error("Encountered error indexing user", mlog.String("user_id", id), mlog.Err(indexUserErr)) - } - } - }) - } - message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_DIRECT_ADDED, "", channel.Id, "", nil) message.Add("teammate_id", otherUserId) a.Publish(message) @@ -431,16 +388,6 @@ func (a *App) CreateGroupChannel(userIds []string, creatorId string) (*model.Cha message.Add("teammate_ids", model.ArrayToJson(userIds)) a.Publish(message) - if a.IsESIndexingEnabled() { - a.Srv().Go(func() { - for _, id := range userIds { - if err := a.indexUserFromId(id); err != nil { - mlog.Error("Encountered error indexing user", mlog.String("user_id", id), mlog.Err(err)) - } - } - }) - } - return channel, nil } @@ -536,14 +483,6 @@ func (a *App) UpdateChannel(channel *model.Channel) (*model.Channel, *model.AppE messageWs.Add("channel", channel.ToJson()) a.Publish(messageWs) - if a.IsESIndexingEnabled() && channel.Type == model.CHANNEL_OPEN { - a.Srv().Go(func() { - if err := a.Elasticsearch().IndexChannel(channel); err != nil { - mlog.Error("Encountered error indexing channel", mlog.String("channel_id", channel.Id), mlog.Err(err)) - } - }) - } - return channel, nil } @@ -1243,14 +1182,6 @@ func (a *App) AddChannelMember(userId string, channel *model.Channel, userReques }) } - if a.IsESIndexingEnabled() { - a.Srv().Go(func() { - if err := a.indexUser(user); err != nil { - mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err)) - } - }) - } - if userRequestorId == "" || userId == userRequestorId { a.postJoinChannelMessage(user, channel) } else { @@ -1639,14 +1570,6 @@ func (a *App) JoinChannel(channel *model.Channel, userId string) *model.AppError }) } - if a.IsESIndexingEnabled() { - a.Srv().Go(func() { - if err := a.indexUser(user); err != nil { - mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err)) - } - }) - } - if err := a.postJoinChannelMessage(user, channel); err != nil { return err } @@ -1927,14 +1850,6 @@ func (a *App) removeUserFromChannel(userIdToRemove string, removerUserId string, }) } - if a.IsESIndexingEnabled() { - a.Srv().Go(func() { - if err := a.indexUserFromId(userIdToRemove); err != nil { - mlog.Error("Encountered error indexing user", mlog.String("user_id", userIdToRemove), mlog.Err(err)) - } - }) - } - message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_USER_REMOVED, "", channel.Id, "", nil) message.Add("user_id", userIdToRemove) message.Add("remover_id", removerUserId) @@ -2056,50 +1971,11 @@ func (a *App) MarkChannelAsUnreadFromPost(postID string, userID string) (*model. return channelUnread, nil } -func (a *App) esAutocompleteChannels(teamId, term string, includeDeleted bool) (*model.ChannelList, *model.AppError) { - channelIds, err := a.Elasticsearch().SearchChannels(teamId, term) - if err != nil { - return nil, err - } - - channelList := model.ChannelList{} - if len(channelIds) > 0 { - channels, err := a.Srv().Store.Channel().GetChannelsByIds(channelIds) - if err != nil { - return nil, err - } - for _, c := range channels { - if c.DeleteAt > 0 && !includeDeleted { - continue - } - channelList = append(channelList, c) - } - } - - return &channelList, nil -} - func (a *App) AutocompleteChannels(teamId string, term string) (*model.ChannelList, *model.AppError) { includeDeleted := *a.Config().TeamSettings.ExperimentalViewArchivedChannels - var channelList *model.ChannelList - var err *model.AppError term = strings.TrimSpace(term) - if a.IsESAutocompletionEnabled() { - channelList, err = a.esAutocompleteChannels(teamId, term, includeDeleted) - if err != nil { - mlog.Error("Encountered error on AutocompleteChannels through Elasticsearch. Falling back to default autocompletion.", mlog.Err(err)) - } - } - - if !a.IsESAutocompletionEnabled() || err != nil { - channelList, err = a.Srv().Store.Channel().AutocompleteInTeam(teamId, term, includeDeleted) - if err != nil { - return nil, err - } - } - - return channelList, nil + return a.Srv().Store.Channel().AutocompleteInTeam(teamId, term, includeDeleted) } func (a *App) AutocompleteChannelsForSearch(teamId string, userId string, term string) (*model.ChannelList, *model.AppError) { @@ -2246,11 +2122,6 @@ func (a *App) ViewChannel(view *model.ChannelView, userId string, currentSession } func (a *App) PermanentDeleteChannel(channel *model.Channel) *model.AppError { - profiles, err := a.Srv().Store.User().GetAllProfilesInChannel(channel.Id, false) - if err != nil { - return err - } - if err := a.Srv().Store.Post().PermanentDeleteByChannel(channel.Id); err != nil { return err } @@ -2271,23 +2142,6 @@ func (a *App) PermanentDeleteChannel(channel *model.Channel) *model.AppError { return err } - if a.IsESIndexingEnabled() { - a.Srv().Go(func() { - for _, user := range profiles { - if err := a.indexUser(user); err != nil { - mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err)) - } - } - }) - if channel.Type == model.CHANNEL_OPEN { - a.Srv().Go(func() { - if err := a.Elasticsearch().DeleteChannel(channel); err != nil { - mlog.Error("Encountered error deleting channel", mlog.String("channel_id", channel.Id), mlog.Err(err)) - } - }) - } - } - return nil } diff --git a/app/config.go b/app/config.go index 0b0548055b..4130ecfefe 100644 --- a/app/config.go +++ b/app/config.go @@ -406,22 +406,6 @@ func (a *App) SaveConfig(newCfg *model.Config, sendConfigChangeClusterMessage bo return nil } -func (a *App) IsESIndexingEnabled() bool { - return a.Elasticsearch() != nil && *a.Config().ElasticsearchSettings.EnableIndexing -} - -func (a *App) IsESSearchEnabled() bool { - esInterface := a.Elasticsearch() - license := a.License() - return esInterface != nil && *a.Config().ElasticsearchSettings.EnableSearching && license != nil && *license.Features.Elasticsearch -} - -func (a *App) IsESAutocompletionEnabled() bool { - esInterface := a.Elasticsearch() - license := a.License() - return esInterface != nil && *a.Config().ElasticsearchSettings.EnableAutocomplete && license != nil && *license.Features.Elasticsearch -} - func (a *App) HandleMessageExportConfig(cfg *model.Config, appCfg *model.Config) { // If the Message Export feature has been toggled in the System Console, rewrite the ExportFromTimestamp field to an // appropriate value. The rewriting occurs here to ensure it doesn't affect values written to the config file diff --git a/app/config_test.go b/app/config_test.go index 38fd0d6ca9..fe59c4c2d4 100644 --- a/app/config_test.go +++ b/app/config_test.go @@ -12,8 +12,6 @@ import ( "github.com/stretchr/testify/mock" "github.com/mattermost/mattermost-server/v5/model" - "github.com/mattermost/mattermost-server/v5/store/localcachelayer" - "github.com/mattermost/mattermost-server/v5/store/sqlstore" "github.com/mattermost/mattermost-server/v5/store/storetest/mocks" "github.com/mattermost/mattermost-server/v5/utils" ) @@ -126,7 +124,7 @@ func TestEnsureInstallationDate(t *testing.T) { for _, tc := range tt { t.Run(tc.Name, func(t *testing.T) { - sqlStore := th.App.Srv().Store.User().(localcachelayer.LocalCacheUserStore).UserStore.(*sqlstore.SqlUserStore) + sqlStore := th.GetSqlSupplier() sqlStore.GetMaster().Exec("DELETE FROM Users") for _, createAt := range tc.UsersCreationDates { diff --git a/app/diagnostics.go b/app/diagnostics.go index d482f84e6a..2dd56b18f6 100644 --- a/app/diagnostics.go +++ b/app/diagnostics.go @@ -914,8 +914,10 @@ func (a *App) trackPermissions() { func (a *App) trackElasticsearch() { data := map[string]interface{}{} - if a.Elasticsearch() != nil && a.Elasticsearch().GetVersion() != 0 { - data["elasticsearch_server_version"] = a.Elasticsearch().GetVersion() + for _, engine := range a.SearchEngine().GetActiveEngines() { + if engine.GetVersion() != 0 && engine.GetName() == "elasticsearch" { + data["elasticsearch_server_version"] = engine.GetVersion() + } } a.SendDiagnostic(TRACK_ELASTICSEARCH, data) diff --git a/app/enterprise.go b/app/enterprise.go index 589bbc44c0..8a6cf3a588 100644 --- a/app/enterprise.go +++ b/app/enterprise.go @@ -9,6 +9,7 @@ import ( tjobs "github.com/mattermost/mattermost-server/v5/jobs/interfaces" "github.com/mattermost/mattermost-server/v5/mlog" "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/services/searchengine" ) var accountMigrationInterface func(*Server) einterfaces.AccountMigrationInterface @@ -35,9 +36,9 @@ func RegisterDataRetentionInterface(f func(*App) einterfaces.DataRetentionInterf dataRetentionInterface = f } -var elasticsearchInterface func(*App) einterfaces.ElasticsearchInterface +var elasticsearchInterface func(*App) searchengine.SearchEngineInterface -func RegisterElasticsearchInterface(f func(*App) einterfaces.ElasticsearchInterface) { +func RegisterElasticsearchInterface(f func(*App) searchengine.SearchEngineInterface) { elasticsearchInterface = f } @@ -129,9 +130,6 @@ func (s *Server) initEnterprise() { if complianceInterface != nil { s.Compliance = complianceInterface(s.FakeApp()) } - if elasticsearchInterface != nil { - s.Elasticsearch = elasticsearchInterface(s.FakeApp()) - } if ldapInterface != nil { s.Ldap = ldapInterface(s.FakeApp()) } @@ -161,4 +159,8 @@ func (s *Server) initEnterprise() { if clusterInterface != nil { s.Cluster = clusterInterface(s) } + + if elasticsearchInterface != nil { + s.SearchEngine.RegisterElasticsearchEngine(elasticsearchInterface(s.FakeApp())) + } } diff --git a/app/helper_test.go b/app/helper_test.go index 69ff372ce8..21ceb3b79b 100644 --- a/app/helper_test.go +++ b/app/helper_test.go @@ -19,6 +19,7 @@ import ( "github.com/mattermost/mattermost-server/v5/model" "github.com/mattermost/mattermost-server/v5/store" "github.com/mattermost/mattermost-server/v5/store/localcachelayer" + "github.com/mattermost/mattermost-server/v5/store/sqlstore" "github.com/mattermost/mattermost-server/v5/store/storetest/mocks" "github.com/mattermost/mattermost-server/v5/testlib" "github.com/mattermost/mattermost-server/v5/utils" @@ -94,6 +95,11 @@ func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer boo } th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = prevListenAddress }) + + th.App.Srv().SearchEngine = mainHelper.SearchEngine + + th.App.Srv().Store.MarkSystemRanUnitTests() + th.App.UpdateConfig(func(cfg *model.Config) { *cfg.TeamSettings.EnableOpenServer = true }) // Disable strict password requirements for test @@ -569,6 +575,10 @@ func (me *TestHelper) TearDown() { } } +func (me *TestHelper) GetSqlSupplier() *sqlstore.SqlSupplier { + return mainHelper.GetSQLSupplier() +} + func (me *TestHelper) ResetRoleMigration() { sqlSupplier := mainHelper.GetSQLSupplier() if _, err := sqlSupplier.GetMaster().Exec("DELETE from Roles"); err != nil { diff --git a/app/layer_generators/opentracing_layer.go.tmpl b/app/layer_generators/opentracing_layer.go.tmpl index 33e367975a..90a9e2ab5a 100644 --- a/app/layer_generators/opentracing_layer.go.tmpl +++ b/app/layer_generators/opentracing_layer.go.tmpl @@ -31,7 +31,7 @@ type {{.Name}} struct { cluster einterfaces.ClusterInterface compliance einterfaces.ComplianceInterface dataRetention einterfaces.DataRetentionInterface - elasticsearch einterfaces.ElasticsearchInterface + searchEngine *searchengine.Broker ldap einterfaces.LdapInterface messageExport einterfaces.MessageExportInterface metrics einterfaces.MetricsInterface @@ -99,7 +99,7 @@ func NewOpenTracingAppLayer(childApp AppIface, ctx context.Context) *{{.Name}} { newApp.cluster = childApp.Cluster() newApp.compliance = childApp.Compliance() newApp.dataRetention = childApp.DataRetention() - newApp.elasticsearch = childApp.Elasticsearch() + newApp.searchEngine = childApp.SearchEngine() newApp.ldap = childApp.Ldap() newApp.messageExport = childApp.MessageExport() newApp.metrics = childApp.Metrics() @@ -156,9 +156,6 @@ func (a *{{.Name}}) Compliance() einterfaces.ComplianceInterface { func (a *{{.Name}}) DataRetention() einterfaces.DataRetentionInterface { return a.dataRetention } -func (a *{{.Name}}) Elasticsearch() einterfaces.ElasticsearchInterface { - return a.elasticsearch -} func (a *{{.Name}}) Ldap() einterfaces.LdapInterface { return a.ldap } @@ -215,4 +212,4 @@ func (a *{{.Name}}) SetServer(srv *Server) { } func (a *{{.Name}}) GetT() goi18n.TranslateFunc { return a.t -} \ No newline at end of file +} diff --git a/app/opentracing_layer.go b/app/opentracing_layer.go index 3b9188fd65..aa5c911af7 100644 --- a/app/opentracing_layer.go +++ b/app/opentracing_layer.go @@ -29,6 +29,7 @@ import ( "github.com/mattermost/mattermost-server/v5/services/filesstore" "github.com/mattermost/mattermost-server/v5/services/httpservice" "github.com/mattermost/mattermost-server/v5/services/imageproxy" + "github.com/mattermost/mattermost-server/v5/services/searchengine" "github.com/mattermost/mattermost-server/v5/services/timezones" "github.com/mattermost/mattermost-server/v5/services/tracing" "github.com/mattermost/mattermost-server/v5/store" @@ -56,7 +57,7 @@ type OpenTracingAppLayer struct { cluster einterfaces.ClusterInterface compliance einterfaces.ComplianceInterface dataRetention einterfaces.DataRetentionInterface - elasticsearch einterfaces.ElasticsearchInterface + searchEngine *searchengine.Broker ldap einterfaces.LdapInterface messageExport einterfaces.MessageExportInterface metrics einterfaces.MetricsInterface @@ -8588,28 +8589,6 @@ func (a *OpenTracingAppLayer) GetViewUsersRestrictions(userId string) (*model.Vi return resultVar0, resultVar1 } -func (a *OpenTracingAppLayer) GetViewUsersRestrictionsForTeam(userId string, teamId string) ([]string, *model.AppError) { - origCtx := a.ctx - span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetViewUsersRestrictionsForTeam") - - a.ctx = newCtx - a.app.Srv().Store.SetContext(newCtx) - defer func() { - a.app.Srv().Store.SetContext(origCtx) - a.ctx = origCtx - }() - - defer span.Finish() - resultVar0, resultVar1 := a.app.GetViewUsersRestrictionsForTeam(userId, teamId) - - if resultVar1 != nil { - span.LogFields(spanlog.Error(resultVar1)) - ext.Error.Set(span, true) - } - - return resultVar0, resultVar1 -} - func (a *OpenTracingAppLayer) HTMLTemplates() *template.Template { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.HTMLTemplates") @@ -9229,57 +9208,6 @@ func (a *OpenTracingAppLayer) InviteNewUsersToTeamGracefully(emailList []string, return resultVar0, resultVar1 } -func (a *OpenTracingAppLayer) IsESAutocompletionEnabled() bool { - origCtx := a.ctx - span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.IsESAutocompletionEnabled") - - a.ctx = newCtx - a.app.Srv().Store.SetContext(newCtx) - defer func() { - a.app.Srv().Store.SetContext(origCtx) - a.ctx = origCtx - }() - - defer span.Finish() - resultVar0 := a.app.IsESAutocompletionEnabled() - - return resultVar0 -} - -func (a *OpenTracingAppLayer) IsESIndexingEnabled() bool { - origCtx := a.ctx - span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.IsESIndexingEnabled") - - a.ctx = newCtx - a.app.Srv().Store.SetContext(newCtx) - defer func() { - a.app.Srv().Store.SetContext(origCtx) - a.ctx = origCtx - }() - - defer span.Finish() - resultVar0 := a.app.IsESIndexingEnabled() - - return resultVar0 -} - -func (a *OpenTracingAppLayer) IsESSearchEnabled() bool { - origCtx := a.ctx - span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.IsESSearchEnabled") - - a.ctx = newCtx - a.app.Srv().Store.SetContext(newCtx) - defer func() { - a.app.Srv().Store.SetContext(origCtx) - a.ctx = origCtx - }() - - defer span.Finish() - resultVar0 := a.app.IsESSearchEnabled() - - return resultVar0 -} - func (a *OpenTracingAppLayer) IsFirstUserAccount() bool { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.IsFirstUserAccount") @@ -11790,6 +11718,23 @@ func (a *OpenTracingAppLayer) SearchEmoji(name string, prefixOnly bool, limit in return resultVar0, resultVar1 } +func (a *OpenTracingAppLayer) SearchEngine() *searchengine.Broker { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SearchEngine") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0 := a.app.SearchEngine() + + return resultVar0 +} + func (a *OpenTracingAppLayer) SearchGroupChannels(userId string, term string) (*model.ChannelList, *model.AppError) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SearchGroupChannels") @@ -14914,7 +14859,7 @@ func NewOpenTracingAppLayer(childApp AppIface, ctx context.Context) *OpenTracing newApp.cluster = childApp.Cluster() newApp.compliance = childApp.Compliance() newApp.dataRetention = childApp.DataRetention() - newApp.elasticsearch = childApp.Elasticsearch() + newApp.searchEngine = childApp.SearchEngine() newApp.ldap = childApp.Ldap() newApp.messageExport = childApp.MessageExport() newApp.metrics = childApp.Metrics() @@ -14970,9 +14915,6 @@ func (a *OpenTracingAppLayer) Compliance() einterfaces.ComplianceInterface { func (a *OpenTracingAppLayer) DataRetention() einterfaces.DataRetentionInterface { return a.dataRetention } -func (a *OpenTracingAppLayer) Elasticsearch() einterfaces.ElasticsearchInterface { - return a.elasticsearch -} func (a *OpenTracingAppLayer) Ldap() einterfaces.LdapInterface { return a.ldap } diff --git a/app/options.go b/app/options.go index eeb7818e76..560009c76d 100644 --- a/app/options.go +++ b/app/options.go @@ -78,8 +78,8 @@ func StartMetrics(s *Server) error { return nil } -func StartElasticsearch(s *Server) error { - s.startElasticsearch = true +func StartSearchEngine(s *Server) error { + s.startSearchEngine = true return nil } @@ -104,7 +104,7 @@ func ServerConnector(s *Server) AppOption { a.cluster = s.Cluster a.compliance = s.Compliance a.dataRetention = s.DataRetention - a.elasticsearch = s.Elasticsearch + a.searchEngine = s.SearchEngine a.ldap = s.Ldap a.messageExport = s.MessageExport a.metrics = s.Metrics diff --git a/app/post.go b/app/post.go index 424090834e..915a74a4b1 100644 --- a/app/post.go +++ b/app/post.go @@ -297,14 +297,6 @@ func (a *App) CreatePost(post *model.Post, channel *model.Channel, triggerWebhoo }) } - if a.IsESIndexingEnabled() { - a.Srv().Go(func() { - if err = a.Elasticsearch().IndexPost(rpost, channel.TeamId); err != nil { - mlog.Error("Encountered error indexing post", mlog.String("post_id", post.Id), mlog.Err(err)) - } - }) - } - if a.Metrics() != nil { a.Metrics().IncrementPostCreate() } @@ -585,19 +577,6 @@ func (a *App) UpdatePost(post *model.Post, safeUpdate bool) (*model.Post, *model }) } - if a.IsESIndexingEnabled() { - a.Srv().Go(func() { - channel, chanErr := a.Srv().Store.Channel().GetForPost(rpost.Id) - if chanErr != nil { - mlog.Error("Couldn't get channel for post for Elasticsearch indexing.", mlog.String("channel_id", rpost.ChannelId), mlog.String("post_id", rpost.Id)) - return - } - if err := a.Elasticsearch().IndexPost(rpost, channel.TeamId); err != nil { - mlog.Error("Encountered error indexing post", mlog.String("post_id", post.Id), mlog.Err(err)) - } - }) - } - rpost = a.PreparePostForClient(rpost, false, true) message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_EDITED, "", rpost.ChannelId, "", nil) @@ -870,14 +849,6 @@ func (a *App) DeletePost(postId, deleteByID string) (*model.Post, *model.AppErro a.DeleteFlaggedPosts(post.Id) }) - if a.IsESIndexingEnabled() { - a.Srv().Go(func() { - if err := a.Elasticsearch().DeletePost(post); err != nil { - mlog.Error("Encountered error deleting post", mlog.String("post_id", post.Id), mlog.Err(err)) - } - }) - } - a.invalidateCacheForChannelPosts(post.ChannelId) return post, nil @@ -1006,10 +977,18 @@ func (a *App) SearchPostsInTeam(teamId string, paramsList []*model.SearchParams) }) } -func (a *App) esSearchPostsInTeamForUser(paramsList []*model.SearchParams, userId, teamId string, isOrSearch, includeDeletedChannels bool, page, perPage int) (*model.PostSearchResults, *model.AppError) { - finalParamsList := []*model.SearchParams{} +func (a *App) SearchPostsInTeamForUser(terms string, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, timeZoneOffset int, page, perPage int) (*model.PostSearchResults, *model.AppError) { + var postSearchResults *model.PostSearchResults + var err *model.AppError + paramsList := model.ParseSearchParams(strings.TrimSpace(terms), timeZoneOffset) includeDeleted := includeDeletedChannels && *a.Config().TeamSettings.ExperimentalViewArchivedChannels + if !*a.Config().ServiceSettings.EnablePostSearch { + return nil, model.NewAppError("SearchPostsInTeamForUser", "store.sql_post.search.disabled", nil, fmt.Sprintf("teamId=%v userId=%v", teamId, userId), http.StatusNotImplemented) + } + + finalParamsList := []*model.SearchParams{} + for _, params := range paramsList { params.OrTerms = isOrSearch // Don't allow users to search for "*" @@ -1031,90 +1010,11 @@ func (a *App) esSearchPostsInTeamForUser(paramsList []*model.SearchParams, userI return model.MakePostSearchResults(model.NewPostList(), nil), nil } - // We only allow the user to search in channels they are a member of. - userChannels, err := a.GetChannelsForUser(teamId, userId, includeDeleted) - if err != nil { - mlog.Error("error getting channel for user", mlog.Err(err)) - return nil, err - } - - postIds, matches, err := a.Elasticsearch().SearchPosts(userChannels, finalParamsList, page, perPage) + postSearchResults, err = a.Srv().Store.Post().SearchPostsInTeamForUser(finalParamsList, userId, teamId, isOrSearch, includeDeleted, page, perPage) if err != nil { return nil, err } - // Get the posts - postList := model.NewPostList() - if len(postIds) > 0 { - posts, err := a.Srv().Store.Post().GetPostsByIds(postIds) - if err != nil { - return nil, err - } - for _, p := range posts { - if p.DeleteAt == 0 { - postList.AddPost(p) - postList.AddOrder(p.Id) - } - } - } - - return model.MakePostSearchResults(postList, matches), nil -} - -func (a *App) SearchPostsInTeamForUser(terms string, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, timeZoneOffset int, page, perPage int) (*model.PostSearchResults, *model.AppError) { - var postSearchResults *model.PostSearchResults - var err *model.AppError - paramsList := model.ParseSearchParams(strings.TrimSpace(terms), timeZoneOffset) - - if !*a.Config().ServiceSettings.EnablePostSearch { - return nil, model.NewAppError("SearchPostsInTeamForUser", "store.sql_post.search.disabled", nil, fmt.Sprintf("teamId=%v userId=%v", teamId, userId), http.StatusNotImplemented) - } - - if a.IsESSearchEnabled() { - postSearchResults, err = a.esSearchPostsInTeamForUser(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage) - if err != nil { - mlog.Error("Encountered error on SearchPostsInTeamForUser through Elasticsearch. Falling back to default search.", mlog.Err(err)) - } - } - - if !a.IsESSearchEnabled() || err != nil { - // Since we don't support paging for DB search, we just return nothing for later pages - if page > 0 { - return model.MakePostSearchResults(model.NewPostList(), nil), nil - } - - includeDeleted := includeDeletedChannels && *a.Config().TeamSettings.ExperimentalViewArchivedChannels - posts, err := a.searchPostsInTeam(teamId, userId, paramsList, func(params *model.SearchParams) { - params.IncludeDeletedChannels = includeDeleted - params.OrTerms = isOrSearch - for idx, channelName := range params.InChannels { - if strings.HasPrefix(channelName, "@") { - channel, err := a.parseAndFetchChannelIdByNameFromInFilter(channelName, userId, teamId, includeDeletedChannels) - if err != nil { - mlog.Error("error getting channel_id by name from in filter", mlog.Err(err)) - continue - } - params.InChannels[idx] = channel.Name - } - } - for idx, channelName := range params.ExcludedChannels { - if strings.HasPrefix(channelName, "@") { - channel, err := a.parseAndFetchChannelIdByNameFromInFilter(channelName, userId, teamId, includeDeletedChannels) - if err != nil { - mlog.Error("error getting channel_id by name from in filter", mlog.Err(err)) - continue - } - params.ExcludedChannels[idx] = channel.Name - } - } - }) - if err != nil { - return nil, err - } - - postSearchResults = model.MakePostSearchResults(posts, nil) - } - return postSearchResults, nil } diff --git a/app/post_test.go b/app/post_test.go index fc97e84936..df82dc2f28 100644 --- a/app/post_test.go +++ b/app/post_test.go @@ -13,9 +13,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/mattermost/mattermost-server/v5/einterfaces/mocks" "github.com/mattermost/mattermost-server/v5/model" "github.com/mattermost/mattermost-server/v5/plugin/plugintest/mock" + "github.com/mattermost/mattermost-server/v5/services/searchengine/mocks" "github.com/mattermost/mattermost-server/v5/store/storetest" storemocks "github.com/mattermost/mattermost-server/v5/store/storetest/mocks" ) @@ -944,9 +944,16 @@ func TestSearchPostsInTeamForUser(t *testing.T) { posts[2].Id, } - es := &mocks.ElasticsearchInterface{} + es := &mocks.SearchEngineInterface{} es.On("SearchPosts", mock.Anything, mock.Anything, page, perPage).Return(resultsPage, nil, nil) - th.App.elasticsearch = es + es.On("GetName").Return("mock") + es.On("Start").Return(nil).Maybe() + es.On("IsActive").Return(true) + es.On("IsSearchEnabled").Return(true) + th.App.Srv().SearchEngine.ElasticsearchEngine = es + defer func() { + th.App.Srv().SearchEngine.ElasticsearchEngine = nil + }() results, err := th.App.SearchPostsInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage) @@ -965,9 +972,16 @@ func TestSearchPostsInTeamForUser(t *testing.T) { posts[0].Id, } - es := &mocks.ElasticsearchInterface{} + es := &mocks.SearchEngineInterface{} es.On("SearchPosts", mock.Anything, mock.Anything, page, perPage).Return(resultsPage, nil, nil) - th.App.elasticsearch = es + es.On("GetName").Return("mock") + es.On("Start").Return(nil).Maybe() + es.On("IsActive").Return(true) + es.On("IsSearchEnabled").Return(true) + th.App.Srv().SearchEngine.ElasticsearchEngine = es + defer func() { + th.App.Srv().SearchEngine.ElasticsearchEngine = nil + }() results, err := th.App.SearchPostsInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage) @@ -982,9 +996,16 @@ func TestSearchPostsInTeamForUser(t *testing.T) { page := 0 - es := &mocks.ElasticsearchInterface{} + es := &mocks.SearchEngineInterface{} es.On("SearchPosts", mock.Anything, mock.Anything, page, perPage).Return(nil, nil, &model.AppError{}) - th.App.elasticsearch = es + es.On("GetName").Return("mock") + es.On("Start").Return(nil).Maybe() + es.On("IsActive").Return(true) + es.On("IsSearchEnabled").Return(true) + th.App.Srv().SearchEngine.ElasticsearchEngine = es + defer func() { + th.App.Srv().SearchEngine.ElasticsearchEngine = nil + }() results, err := th.App.SearchPostsInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage) @@ -1007,9 +1028,16 @@ func TestSearchPostsInTeamForUser(t *testing.T) { page := 1 - es := &mocks.ElasticsearchInterface{} + es := &mocks.SearchEngineInterface{} es.On("SearchPosts", mock.Anything, mock.Anything, page, perPage).Return(nil, nil, &model.AppError{}) - th.App.elasticsearch = es + es.On("GetName").Return("mock") + es.On("Start").Return(nil).Maybe() + es.On("IsActive").Return(true) + es.On("IsSearchEnabled").Return(true) + th.App.Srv().SearchEngine.ElasticsearchEngine = es + defer func() { + th.App.Srv().SearchEngine.ElasticsearchEngine = nil + }() results, err := th.App.SearchPostsInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage) diff --git a/app/elasticsearch.go b/app/searchengine.go similarity index 84% rename from app/elasticsearch.go rename to app/searchengine.go index b1906098d6..d1eae5123c 100644 --- a/app/elasticsearch.go +++ b/app/searchengine.go @@ -18,12 +18,12 @@ func (a *App) TestElasticsearch(cfg *model.Config) *model.AppError { } } - esI := a.Elasticsearch() - if esI == nil { + seI := a.SearchEngine().ElasticsearchEngine + if seI == nil { err := model.NewAppError("TestElasticsearch", "ent.elasticsearch.test_config.license.error", nil, "", http.StatusNotImplemented) return err } - if err := esI.TestConfig(cfg); err != nil { + if err := seI.TestConfig(cfg); err != nil { return err } @@ -31,13 +31,13 @@ func (a *App) TestElasticsearch(cfg *model.Config) *model.AppError { } func (a *App) PurgeElasticsearchIndexes() *model.AppError { - esI := a.Elasticsearch() - if esI == nil { + seI := a.SearchEngine().ElasticsearchEngine + if seI == nil { err := model.NewAppError("PurgeElasticsearchIndexes", "ent.elasticsearch.test_config.license.error", nil, "", http.StatusNotImplemented) return err } - if err := esI.PurgeIndexes(); err != nil { + if err := seI.PurgeIndexes(); err != nil { return err } diff --git a/app/server.go b/app/server.go index e08ba2fdeb..34448acc11 100644 --- a/app/server.go +++ b/app/server.go @@ -35,6 +35,7 @@ import ( "github.com/mattermost/mattermost-server/v5/services/cache/lru" "github.com/mattermost/mattermost-server/v5/services/httpservice" "github.com/mattermost/mattermost-server/v5/services/imageproxy" + "github.com/mattermost/mattermost-server/v5/services/searchengine" "github.com/mattermost/mattermost-server/v5/services/timezones" "github.com/mattermost/mattermost-server/v5/services/tracing" "github.com/mattermost/mattermost-server/v5/store" @@ -98,6 +99,8 @@ type Server struct { licenseListenerId string logListenerId string clusterLeaderListenerId string + searchConfigListenerId string + searchLicenseListenerId string configStore config.Store asymmetricSigningKey *ecdsa.PrivateKey postActionCookieSecret []byte @@ -122,15 +125,16 @@ type Server struct { Log *mlog.Logger NotificationsLog *mlog.Logger - joinCluster bool - startMetrics bool - startElasticsearch bool + joinCluster bool + startMetrics bool + startSearchEngine bool + + SearchEngine *searchengine.Broker AccountMigration einterfaces.AccountMigrationInterface Cluster einterfaces.ClusterInterface Compliance einterfaces.ComplianceInterface DataRetention einterfaces.DataRetentionInterface - Elasticsearch einterfaces.ElasticsearchInterface Ldap einterfaces.LdapInterface MessageExport einterfaces.MessageExportInterface Metrics einterfaces.MetricsInterface @@ -206,6 +210,8 @@ func NewServer(options ...Option) (*Server, error) { return nil, errors.Wrapf(err, "unable to load Mattermost translation files") } + s.SearchEngine = searchengine.NewBroker(s.Config(), s.Jobs) + // at the moment we only have this implementation // in the future the cache provider will be built based on the loaded config s.CacheProvider = new(lru.CacheProvider) @@ -304,10 +310,6 @@ func NewServer(options ...Option) (*Server, error) { s.Metrics.StartServer() } - if s.startElasticsearch && s.Elasticsearch != nil { - s.StartElasticsearch() - } - s.AddConfigListener(func(oldConfig *model.Config, newConfig *model.Config) { if *oldConfig.GuestAccountsSettings.Enable && !*newConfig.GuestAccountsSettings.Enable { if appErr := s.FakeApp().DeactivateGuests(); appErr != nil { @@ -354,6 +356,11 @@ func NewServer(options ...Option) (*Server, error) { } } + s.SearchEngine.UpdateConfig(s.Config()) + searchConfigListenerId, searchLicenseListenerId := s.StartSearchEngine() + s.searchConfigListenerId = searchConfigListenerId + s.searchLicenseListenerId = searchLicenseListenerId + return s, nil } @@ -413,6 +420,7 @@ func (s *Server) Shutdown() error { s.RemoveConfigListener(s.configListenerId) s.RemoveConfigListener(s.logListenerId) + s.stopSearchEngine() s.Audit.Shutdown() @@ -769,33 +777,40 @@ func doSessionCleanup(s *Server) { s.Store.Session().Cleanup(model.GetMillis(), SESSIONS_CLEANUP_BATCH_SIZE) } -func (s *Server) StartElasticsearch() { - s.Go(func() { - if err := s.Elasticsearch.Start(); err != nil { - s.Log.Error(err.Error()) - } - }) +func (s *Server) StartSearchEngine() (string, string) { + if s.SearchEngine.ElasticsearchEngine != nil && s.SearchEngine.ElasticsearchEngine.IsActive() { + s.Go(func() { + if err := s.SearchEngine.ElasticsearchEngine.Start(); err != nil { + s.Log.Error(err.Error()) + } + }) + } - s.AddConfigListener(func(oldConfig *model.Config, newConfig *model.Config) { - if !*oldConfig.ElasticsearchSettings.EnableIndexing && *newConfig.ElasticsearchSettings.EnableIndexing { + configListenerId := s.AddConfigListener(func(oldConfig *model.Config, newConfig *model.Config) { + if s.SearchEngine == nil { + return + } + s.SearchEngine.UpdateConfig(newConfig) + + if s.SearchEngine.ElasticsearchEngine != nil && !*oldConfig.ElasticsearchSettings.EnableIndexing && *newConfig.ElasticsearchSettings.EnableIndexing { s.Go(func() { - if err := s.Elasticsearch.Start(); err != nil { + if err := s.SearchEngine.ElasticsearchEngine.Start(); err != nil { mlog.Error(err.Error()) } }) - } else if *oldConfig.ElasticsearchSettings.EnableIndexing && !*newConfig.ElasticsearchSettings.EnableIndexing { + } else if s.SearchEngine.ElasticsearchEngine != nil && *oldConfig.ElasticsearchSettings.EnableIndexing && !*newConfig.ElasticsearchSettings.EnableIndexing { s.Go(func() { - if err := s.Elasticsearch.Stop(); err != nil { + if err := s.SearchEngine.ElasticsearchEngine.Stop(); err != nil { mlog.Error(err.Error()) } }) - } else if *oldConfig.ElasticsearchSettings.Password != *newConfig.ElasticsearchSettings.Password || *oldConfig.ElasticsearchSettings.Username != *newConfig.ElasticsearchSettings.Username || *oldConfig.ElasticsearchSettings.ConnectionUrl != *newConfig.ElasticsearchSettings.ConnectionUrl || *oldConfig.ElasticsearchSettings.Sniff != *newConfig.ElasticsearchSettings.Sniff { + } else if s.SearchEngine.ElasticsearchEngine != nil && *oldConfig.ElasticsearchSettings.Password != *newConfig.ElasticsearchSettings.Password || *oldConfig.ElasticsearchSettings.Username != *newConfig.ElasticsearchSettings.Username || *oldConfig.ElasticsearchSettings.ConnectionUrl != *newConfig.ElasticsearchSettings.ConnectionUrl || *oldConfig.ElasticsearchSettings.Sniff != *newConfig.ElasticsearchSettings.Sniff { s.Go(func() { if *oldConfig.ElasticsearchSettings.EnableIndexing { - if err := s.Elasticsearch.Stop(); err != nil { + if err := s.SearchEngine.ElasticsearchEngine.Stop(); err != nil { mlog.Error(err.Error()) } - if err := s.Elasticsearch.Start(); err != nil { + if err := s.SearchEngine.ElasticsearchEngine.Start(); err != nil { mlog.Error(err.Error()) } } @@ -803,21 +818,38 @@ func (s *Server) StartElasticsearch() { } }) - s.AddLicenseListener(func(oldLicense, newLicense *model.License) { + licenseListenerId := s.AddLicenseListener(func(oldLicense, newLicense *model.License) { + if s.SearchEngine == nil { + return + } if oldLicense == nil && newLicense != nil { - s.Go(func() { - if err := s.Elasticsearch.Start(); err != nil { - mlog.Error(err.Error()) - } - }) + if s.SearchEngine.ElasticsearchEngine != nil && s.SearchEngine.ElasticsearchEngine.IsActive() { + s.Go(func() { + if err := s.SearchEngine.ElasticsearchEngine.Start(); err != nil { + mlog.Error(err.Error()) + } + }) + } } else if oldLicense != nil && newLicense == nil { - s.Go(func() { - if err := s.Elasticsearch.Stop(); err != nil { - mlog.Error(err.Error()) - } - }) + if s.SearchEngine.ElasticsearchEngine != nil { + s.Go(func() { + if err := s.SearchEngine.ElasticsearchEngine.Stop(); err != nil { + mlog.Error(err.Error()) + } + }) + } } }) + + return configListenerId, licenseListenerId +} + +func (s *Server) stopSearchEngine() { + s.RemoveConfigListener(s.searchConfigListenerId) + s.RemoveLicenseListener(s.searchLicenseListenerId) + if s.SearchEngine != nil && s.SearchEngine.ElasticsearchEngine != nil && s.SearchEngine.ElasticsearchEngine.IsActive() { + s.SearchEngine.ElasticsearchEngine.Stop() + } } func (s *Server) initDiagnostics(endpoint string) { diff --git a/app/server_app_adapters.go b/app/server_app_adapters.go index f28ef26bb5..04a689913c 100644 --- a/app/server_app_adapters.go +++ b/app/server_app_adapters.go @@ -13,6 +13,7 @@ import ( "github.com/mattermost/mattermost-server/v5/services/mailservice" "github.com/mattermost/mattermost-server/v5/store" "github.com/mattermost/mattermost-server/v5/store/localcachelayer" + "github.com/mattermost/mattermost-server/v5/store/searchlayer" "github.com/mattermost/mattermost-server/v5/store/sqlstore" "github.com/mattermost/mattermost-server/v5/utils" "github.com/pkg/errors" @@ -61,10 +62,17 @@ func (s *Server) RunOldAppInitialization() error { if s.FakeApp().Srv().newStore == nil { s.FakeApp().Srv().newStore = func() store.Store { return store.NewTimerLayer( - localcachelayer.NewLocalCacheLayer( - sqlstore.NewSqlSupplier(s.FakeApp().Config().SqlSettings, s.Metrics), - s.Metrics, s.Cluster, s.CacheProvider), - s.Metrics) + searchlayer.NewSearchLayer( + localcachelayer.NewLocalCacheLayer( + sqlstore.NewSqlSupplier(s.FakeApp().Config().SqlSettings, s.Metrics), + s.Metrics, + s.Cluster, + s.CacheProvider, + ), + s.SearchEngine, + ), + s.Metrics, + ) } } diff --git a/app/team.go b/app/team.go index bab9bb4cd5..b490f07ac3 100644 --- a/app/team.go +++ b/app/team.go @@ -489,7 +489,7 @@ func (a *App) AddUserToTeamByToken(userId string, tokenId string) (*model.Team, } if token.Type == TOKEN_TYPE_GUEST_INVITATION { - channels, err := a.Srv().Store.Channel().GetChannelsByIds(strings.Split(tokenData["channels"], " ")) + channels, err := a.Srv().Store.Channel().GetChannelsByIds(strings.Split(tokenData["channels"], " "), false) if err != nil { return nil, err } @@ -944,15 +944,6 @@ func (a *App) RemoveTeamMemberFromTeam(teamMember *model.TeamMember, requestorId }) } - esInterface := a.Elasticsearch() - if esInterface != nil && *a.Config().ElasticsearchSettings.EnableIndexing { - a.Srv().Go(func() { - if err := a.indexUser(user); err != nil { - mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err)) - } - }) - } - if _, err := a.Srv().Store.User().UpdateUpdateAt(user.Id); err != nil { return err } @@ -1135,7 +1126,7 @@ func (a *App) prepareInviteGuestsToChannels(teamId string, guestsInvite *model.G }() cchan := make(chan store.StoreResult, 1) go func() { - channels, err := a.Srv().Store.Channel().GetChannelsByIds(guestsInvite.Channels) + channels, err := a.Srv().Store.Channel().GetChannelsByIds(guestsInvite.Channels, false) cchan <- store.StoreResult{Data: channels, Err: err} close(cchan) }() diff --git a/app/user.go b/app/user.go index 03f7294c25..885efa7582 100644 --- a/app/user.go +++ b/app/user.go @@ -67,7 +67,7 @@ func (a *App) CreateUserWithToken(user *model.User, token *model.Token) (*model. return nil, err } - channels, err := a.Srv().Store.Channel().GetChannelsByIds(strings.Split(tokenData["channels"], " ")) + channels, err := a.Srv().Store.Channel().GetChannelsByIds(strings.Split(tokenData["channels"], " "), false) if err != nil { return nil, err } @@ -201,40 +201,6 @@ func (a *App) IsFirstUserAccount() bool { return false } -// indexUser fetches the required information to index a user from the database and -// calls the elasticsearch interface method -func (a *App) indexUser(user *model.User) *model.AppError { - userTeams, err := a.Srv().Store.Team().GetTeamsByUserId(user.Id) - if err != nil { - return err - } - - userTeamsIds := []string{} - for _, team := range userTeams { - userTeamsIds = append(userTeamsIds, team.Id) - } - - userChannelMembers, err := a.Srv().Store.Channel().GetAllChannelMembersForUser(user.Id, false, true) - if err != nil { - return err - } - - userChannelsIds := []string{} - for channelId := range userChannelMembers { - userChannelsIds = append(userChannelsIds, channelId) - } - - return a.Elasticsearch().IndexUser(user, userTeamsIds, userChannelsIds) -} - -func (a *App) indexUserFromId(userId string) *model.AppError { - user, err := a.GetUser(userId) - if err != nil { - return err - } - return a.indexUser(user) -} - // CreateUser creates a user and sets several fields of the returned User struct to // their zero values. func (a *App) CreateUser(user *model.User) (*model.User, *model.AppError) { @@ -294,14 +260,6 @@ func (a *App) createUserOrGuest(user *model.User, guest bool) (*model.User, *mod }) } - if a.IsESIndexingEnabled() { - a.Srv().Go(func() { - if err := a.indexUser(user); err != nil { - mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err)) - } - }) - } - return ruser, nil } @@ -1193,14 +1151,6 @@ func (a *App) UpdateUser(user *model.User, sendNotifications bool) (*model.User, a.InvalidateCacheForUser(user.Id) - if a.IsESIndexingEnabled() { - a.Srv().Go(func() { - if err := a.indexUser(user); err != nil { - mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err)) - } - }) - } - return userUpdate.New, nil } @@ -1554,14 +1504,6 @@ func (a *App) PermanentDeleteUser(user *model.User) *model.AppError { mlog.Warn("Permanently deleted account", mlog.String("user_email", user.Email), mlog.String("user_id", user.Id)) - if a.IsESIndexingEnabled() { - a.Srv().Go(func() { - if err := a.Elasticsearch().DeleteUser(user); err != nil { - mlog.Error("Encountered error deleting user", mlog.String("user_id", user.Id), mlog.Err(err)) - } - }) - } - return nil } @@ -1743,21 +1685,12 @@ func (a *App) SearchUsersNotInChannel(teamId string, channelId string, term stri return users, nil } -func (a *App) esSearchUsersInTeam(teamId, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError) { - listOfAllowedChannels, err := a.GetViewUsersRestrictionsForTeam(a.Session().UserId, teamId) - if err != nil { - return nil, err - } - if listOfAllowedChannels != nil && len(listOfAllowedChannels) == 0 { - return []*model.User{}, nil - } +func (a *App) SearchUsersInTeam(teamId, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError) { + var users []*model.User + var err *model.AppError + term = strings.TrimSpace(term) - usersIds, err := a.Elasticsearch().SearchUsersInTeam(teamId, listOfAllowedChannels, term, options) - if err != nil { - return nil, err - } - - users, err := a.Srv().Store.User().GetProfileByIds(usersIds, nil, false) + users, err = a.Srv().Store.User().Search(teamId, term, options) if err != nil { return nil, err } @@ -1769,32 +1702,6 @@ func (a *App) esSearchUsersInTeam(teamId, term string, options *model.UserSearch return users, nil } -func (a *App) SearchUsersInTeam(teamId, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError) { - var users []*model.User - var err *model.AppError - term = strings.TrimSpace(term) - - if a.IsESAutocompletionEnabled() { - users, err = a.esSearchUsersInTeam(teamId, term, options) - if err != nil { - mlog.Error("Encountered error on SearchUsersInTeam through Elasticsearch. Falling back to default search.", mlog.Err(err)) - } - } - - if !a.IsESAutocompletionEnabled() || err != nil { - users, err = a.Srv().Store.User().Search(teamId, term, options) - if err != nil { - return nil, err - } - - for _, user := range users { - a.SanitizeProfile(user, options.IsAdmin) - } - } - - return users, nil -} - func (a *App) SearchUsersNotInTeam(notInTeamId string, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError) { term = strings.TrimSpace(term) users, err := a.Srv().Store.User().SearchNotInTeam(notInTeamId, term, options) @@ -1823,141 +1730,31 @@ func (a *App) SearchUsersWithoutTeam(term string, options *model.UserSearchOptio return users, nil } -func (a *App) esAutocompleteUsersInChannel(teamId, channelId, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) { - listOfAllowedChannels, err := a.getListOfAllowedChannelsForTeam(teamId, options.ViewRestrictions) - if err != nil { - return nil, err - } - if len(listOfAllowedChannels) == 0 { - return &model.UserAutocompleteInChannel{}, nil - } - uchanIds := []string{} - nuchanIds := []string{} - if !strings.Contains(strings.Join(listOfAllowedChannels, "."), channelId) { - nuchanIds, err = a.Elasticsearch().SearchUsersInTeam(teamId, listOfAllowedChannels, term, options) - } else { - uchanIds, nuchanIds, err = a.Elasticsearch().SearchUsersInChannel(teamId, channelId, listOfAllowedChannels, term, options) - } - if err != nil { - return nil, err - } - - uchan := make(chan store.StoreResult, 1) - go func() { - users, err := a.Srv().Store.User().GetProfileByIds(uchanIds, nil, false) - uchan <- store.StoreResult{Data: users, Err: err} - close(uchan) - }() - - nuchan := make(chan store.StoreResult, 1) - go func() { - users, err := a.Srv().Store.User().GetProfileByIds(nuchanIds, nil, false) - nuchan <- store.StoreResult{Data: users, Err: err} - close(nuchan) - }() - - autocomplete := &model.UserAutocompleteInChannel{} - - result := <-uchan - if result.Err != nil { - return nil, result.Err - } - users := result.Data.([]*model.User) - - for _, user := range users { - a.SanitizeProfile(user, options.IsAdmin) - } - - autocomplete.InChannel = users - - result = <-nuchan - if result.Err != nil { - return nil, result.Err - } - users = result.Data.([]*model.User) - - for _, user := range users { - a.SanitizeProfile(user, options.IsAdmin) - } - - autocomplete.OutOfChannel = users - - return autocomplete, nil -} - func (a *App) AutocompleteUsersInChannel(teamId string, channelId string, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) { - var autocomplete *model.UserAutocompleteInChannel - var err *model.AppError term = strings.TrimSpace(term) - if a.IsESAutocompletionEnabled() { - autocomplete, err = a.esAutocompleteUsersInChannel(teamId, channelId, term, options) - if err != nil { - mlog.Error("Encountered error on AutocompleteUsersInChannel through Elasticsearch. Falling back to default autocompletion.", mlog.Err(err)) - } + autocomplete, err := a.Srv().Store.User().AutocompleteUsersInChannel(teamId, channelId, term, options) + if err != nil { + return nil, err } - if !a.IsESAutocompletionEnabled() || err != nil { - autocomplete = &model.UserAutocompleteInChannel{} + for _, user := range autocomplete.InChannel { + a.SanitizeProfile(user, options.IsAdmin) + } - uchan := make(chan store.StoreResult, 1) - go func() { - users, err := a.Srv().Store.User().SearchInChannel(channelId, term, options) - uchan <- store.StoreResult{Data: users, Err: err} - close(uchan) - }() - - nuchan := make(chan store.StoreResult, 1) - go func() { - users, err := a.Srv().Store.User().SearchNotInChannel(teamId, channelId, term, options) - nuchan <- store.StoreResult{Data: users, Err: err} - close(nuchan) - }() - - result := <-uchan - if result.Err != nil { - return nil, result.Err - } - - users := result.Data.([]*model.User) - - for _, user := range users { - a.SanitizeProfile(user, options.IsAdmin) - } - - autocomplete.InChannel = users - - result = <-nuchan - if result.Err != nil { - return nil, result.Err - } - users = result.Data.([]*model.User) - - for _, user := range users { - a.SanitizeProfile(user, options.IsAdmin) - } - - autocomplete.OutOfChannel = users + for _, user := range autocomplete.OutOfChannel { + a.SanitizeProfile(user, options.IsAdmin) } return autocomplete, nil } -func (a *App) esAutocompleteUsersInTeam(teamId, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInTeam, *model.AppError) { - listOfAllowedChannels, err := a.getListOfAllowedChannelsForTeam(teamId, options.ViewRestrictions) - if err != nil { - return nil, err - } - if len(listOfAllowedChannels) == 0 { - return &model.UserAutocompleteInTeam{}, nil - } +func (a *App) AutocompleteUsersInTeam(teamId string, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInTeam, *model.AppError) { + var err *model.AppError - usersIds, err := a.Elasticsearch().SearchUsersInTeam(teamId, listOfAllowedChannels, term, options) - if err != nil { - return nil, err - } + term = strings.TrimSpace(term) - users, err := a.Srv().Store.User().GetProfileByIds(usersIds, nil, false) + users, err := a.Srv().Store.User().Search(teamId, term, options) if err != nil { return nil, err } @@ -1968,37 +1765,6 @@ func (a *App) esAutocompleteUsersInTeam(teamId, term string, options *model.User autocomplete := &model.UserAutocompleteInTeam{} autocomplete.InTeam = users - - return autocomplete, nil -} - -func (a *App) AutocompleteUsersInTeam(teamId string, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInTeam, *model.AppError) { - var autocomplete *model.UserAutocompleteInTeam - var err *model.AppError - - term = strings.TrimSpace(term) - - if a.IsESAutocompletionEnabled() { - autocomplete, err = a.esAutocompleteUsersInTeam(teamId, term, options) - if err != nil { - mlog.Error("Encountered error on AutocompleteUsersInTeam through Elasticsearch. Falling back to default autocompletion.", mlog.Err(err)) - } - } - - if !a.IsESAutocompletionEnabled() || err != nil { - autocomplete = &model.UserAutocompleteInTeam{} - users, err := a.Srv().Store.User().Search(teamId, term, options) - if err != nil { - return nil, err - } - - for _, user := range users { - a.SanitizeProfile(user, options.IsAdmin) - } - - autocomplete.InTeam = users - } - return autocomplete, nil } @@ -2044,14 +1810,6 @@ func (a *App) UpdateOAuthUserAttrs(userData io.Reader, user *model.User, provide user = users.New a.InvalidateCacheForUser(user.Id) - - if a.IsESIndexingEnabled() { - a.Srv().Go(func() { - if err := a.indexUser(user); err != nil { - mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err)) - } - }) - } } return nil @@ -2194,61 +1952,6 @@ func (a *App) GetViewUsersRestrictions(userId string) (*model.ViewUsersRestricti return &model.ViewUsersRestrictions{Teams: teamIdsWithPermission, Channels: channelIds}, nil } -/** - * Returns a list with the channel ids that the user has permissions to view on a - * team. If the result is an empty list, the user can't view any channel; if it's - * nil, there are no restrictions for the user in the specified team. - */ -func (a *App) GetViewUsersRestrictionsForTeam(userId string, teamId string) ([]string, *model.AppError) { - if a.HasPermissionTo(userId, model.PERMISSION_VIEW_MEMBERS) { - return nil, nil - } - - if a.HasPermissionToTeam(userId, teamId, model.PERMISSION_VIEW_MEMBERS) { - return nil, nil - } - - members, err := a.Srv().Store.Channel().GetMembersForUser(teamId, userId) - if err != nil { - return nil, err - } - - channelIds := []string{} - for _, membership := range *members { - channelIds = append(channelIds, membership.ChannelId) - } - - return channelIds, nil -} - -func (a *App) getListOfAllowedChannelsForTeam(teamId string, viewRestrictions *model.ViewUsersRestrictions) ([]string, *model.AppError) { - var listOfAllowedChannels []string - if viewRestrictions == nil || strings.Contains(strings.Join(viewRestrictions.Teams, "."), teamId) { - channels, err := a.Srv().Store.Channel().GetTeamChannels(teamId) - if err != nil { - return nil, err - } - channelIds := []string{} - for _, channel := range *channels { - channelIds = append(channelIds, channel.Id) - } - - return channelIds, nil - } - - channels, err := a.Srv().Store.Channel().GetChannelsByIds(viewRestrictions.Channels) - if err != nil { - return nil, err - } - for _, c := range channels { - if c.TeamId == teamId { - listOfAllowedChannels = append(listOfAllowedChannels, c.Id) - } - } - - return listOfAllowedChannels, nil -} - // PromoteGuestToUser Convert user's roles and all his mermbership's roles from // guest roles to regular user roles. func (a *App) PromoteGuestToUser(user *model.User, requestorId string) *model.AppError { diff --git a/app/user_test.go b/app/user_test.go index bb66c076a9..7775d336dd 100644 --- a/app/user_test.go +++ b/app/user_test.go @@ -942,90 +942,6 @@ func TestGetViewUsersRestrictions(t *testing.T) { }) } -func TestGetViewUsersRestrictionsForTeam(t *testing.T) { - th := Setup(t).InitBasic() - defer th.TearDown() - - team1 := th.CreateTeam() - team2 := th.CreateTeam() - th.CreateTeam() // Another team - - user1 := th.CreateUser() - - th.LinkUserToTeam(user1, team1) - th.LinkUserToTeam(user1, team2) - - th.App.UpdateTeamMemberRoles(team1.Id, user1.Id, "team_user team_admin") - - team1channel1 := th.CreateChannel(team1) - team1channel2 := th.CreateChannel(team1) - th.CreateChannel(team1) // Another channel - team1offtopic, err := th.App.GetChannelByName("off-topic", team1.Id, false) - require.Nil(t, err) - team1townsquare, err := th.App.GetChannelByName("town-square", team1.Id, false) - require.Nil(t, err) - - team2channel1 := th.CreateChannel(team2) - th.CreateChannel(team2) // Another channel - - th.App.AddUserToChannel(user1, team1channel1) - th.App.AddUserToChannel(user1, team1channel2) - th.App.AddUserToChannel(user1, team2channel1) - - addPermission := func(role *model.Role, permission string) *model.AppError { - newPermissions := append(role.Permissions, permission) - _, err := th.App.PatchRole(role, &model.RolePatch{Permissions: &newPermissions}) - return err - } - - removePermission := func(role *model.Role, permission string) *model.AppError { - newPermissions := []string{} - for _, oldPermission := range role.Permissions { - if permission != oldPermission { - newPermissions = append(newPermissions, oldPermission) - } - } - _, err := th.App.PatchRole(role, &model.RolePatch{Permissions: &newPermissions}) - return err - } - - t.Run("VIEW_MEMBERS permission granted at system level", func(t *testing.T) { - restrictions, err := th.App.GetViewUsersRestrictionsForTeam(user1.Id, team1.Id) - require.Nil(t, err) - - assert.Nil(t, restrictions) - }) - - t.Run("VIEW_MEMBERS permission granted at team level", func(t *testing.T) { - systemUserRole, err := th.App.GetRoleByName(model.SYSTEM_USER_ROLE_ID) - require.Nil(t, err) - teamUserRole, err := th.App.GetRoleByName(model.TEAM_USER_ROLE_ID) - require.Nil(t, err) - - require.Nil(t, removePermission(systemUserRole, model.PERMISSION_VIEW_MEMBERS.Id)) - defer addPermission(systemUserRole, model.PERMISSION_VIEW_MEMBERS.Id) - require.Nil(t, addPermission(teamUserRole, model.PERMISSION_VIEW_MEMBERS.Id)) - defer removePermission(teamUserRole, model.PERMISSION_VIEW_MEMBERS.Id) - - restrictions, err := th.App.GetViewUsersRestrictionsForTeam(user1.Id, team1.Id) - require.Nil(t, err) - assert.Nil(t, restrictions) - }) - - t.Run("VIEW_MEMBERS permission not granted at any level", func(t *testing.T) { - systemUserRole, err := th.App.GetRoleByName(model.SYSTEM_USER_ROLE_ID) - require.Nil(t, err) - require.Nil(t, removePermission(systemUserRole, model.PERMISSION_VIEW_MEMBERS.Id)) - defer addPermission(systemUserRole, model.PERMISSION_VIEW_MEMBERS.Id) - - restrictions, err := th.App.GetViewUsersRestrictionsForTeam(user1.Id, team1.Id) - require.Nil(t, err) - - assert.NotNil(t, restrictions) - assert.ElementsMatch(t, []string{team1townsquare.Id, team1offtopic.Id, team1channel1.Id, team1channel2.Id}, restrictions) - }) -} - func TestPromoteGuestToUser(t *testing.T) { th := Setup(t).InitBasic() defer th.TearDown() diff --git a/cmd/mattermost/commands/init.go b/cmd/mattermost/commands/init.go index 94e56434e9..e4bec66fb4 100644 --- a/cmd/mattermost/commands/init.go +++ b/cmd/mattermost/commands/init.go @@ -35,7 +35,7 @@ func InitDBCommandContext(configDSN string) (*app.App, error) { s, err := app.NewServer( app.Config(configDSN, false), - app.StartElasticsearch, + app.StartSearchEngine, ) if err != nil { return nil, err diff --git a/cmd/mattermost/commands/server.go b/cmd/mattermost/commands/server.go index 75dcd3aed4..4dd296461b 100644 --- a/cmd/mattermost/commands/server.go +++ b/cmd/mattermost/commands/server.go @@ -58,7 +58,7 @@ func runServer(configStore config.Store, disableConfigWatch bool, usedPlatform b app.ConfigStore(configStore), app.RunJobs, app.JoinCluster, - app.StartElasticsearch, + app.StartSearchEngine, app.StartMetrics, } server, err := app.NewServer(options...) diff --git a/jobs/interfaces/searchengine.go b/jobs/interfaces/searchengine.go new file mode 100644 index 0000000000..670218ad84 --- /dev/null +++ b/jobs/interfaces/searchengine.go @@ -0,0 +1,17 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package interfaces + +import ( + "github.com/mattermost/mattermost-server/v5/model" +) + +type SearchEngineIndexerInterface interface { + MakeWorker() model.Worker +} + +type SearchEngineAggregatorInterface interface { + MakeWorker() model.Worker + MakeScheduler() model.Scheduler +} diff --git a/model/user_search.go b/model/user_search.go index 5e4ed621c3..fa9fa8a283 100644 --- a/model/user_search.go +++ b/model/user_search.go @@ -62,4 +62,6 @@ type UserSearchOptions struct { Role string // Restrict to search in a list of teams and channels ViewRestrictions *ViewUsersRestrictions + // List of allowed channels + ListOfAllowedChannels []string } diff --git a/einterfaces/elasticsearch.go b/services/searchengine/interface.go similarity index 85% rename from einterfaces/elasticsearch.go rename to services/searchengine/interface.go index e77cb70e88..3f15c58de6 100644 --- a/einterfaces/elasticsearch.go +++ b/services/searchengine/interface.go @@ -1,7 +1,7 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -package einterfaces +package searchengine import ( "time" @@ -9,10 +9,16 @@ import ( "github.com/mattermost/mattermost-server/v5/model" ) -type ElasticsearchInterface interface { +type SearchEngineInterface interface { Start() *model.AppError Stop() *model.AppError GetVersion() int + UpdateConfig(cfg *model.Config) + GetName() string + IsActive() bool + IsIndexingEnabled() bool + IsSearchEnabled() bool + IsAutocompletionEnabled() bool IndexPost(post *model.Post, teamId string) *model.AppError SearchPosts(channels *model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, model.PostSearchMatches, *model.AppError) DeletePost(post *model.Post) *model.AppError diff --git a/einterfaces/mocks/ElasticsearchInterface.go b/services/searchengine/mocks/SearchEngineInterface.go similarity index 68% rename from einterfaces/mocks/ElasticsearchInterface.go rename to services/searchengine/mocks/SearchEngineInterface.go index 59b699d077..a04edd64a1 100644 --- a/einterfaces/mocks/ElasticsearchInterface.go +++ b/services/searchengine/mocks/SearchEngineInterface.go @@ -1,6 +1,6 @@ // Code generated by mockery v1.0.0. DO NOT EDIT. -// Regenerate this file using `make einterfaces-mocks`. +// Regenerate this file using `make searchengine-mocks`. package mocks @@ -11,13 +11,13 @@ import ( mock "github.com/stretchr/testify/mock" ) -// ElasticsearchInterface is an autogenerated mock type for the ElasticsearchInterface type -type ElasticsearchInterface struct { +// SearchEngineInterface is an autogenerated mock type for the SearchEngineInterface type +type SearchEngineInterface struct { mock.Mock } // DataRetentionDeleteIndexes provides a mock function with given fields: cutoff -func (_m *ElasticsearchInterface) DataRetentionDeleteIndexes(cutoff time.Time) *model.AppError { +func (_m *SearchEngineInterface) DataRetentionDeleteIndexes(cutoff time.Time) *model.AppError { ret := _m.Called(cutoff) var r0 *model.AppError @@ -33,7 +33,7 @@ func (_m *ElasticsearchInterface) DataRetentionDeleteIndexes(cutoff time.Time) * } // DeleteChannel provides a mock function with given fields: channel -func (_m *ElasticsearchInterface) DeleteChannel(channel *model.Channel) *model.AppError { +func (_m *SearchEngineInterface) DeleteChannel(channel *model.Channel) *model.AppError { ret := _m.Called(channel) var r0 *model.AppError @@ -49,7 +49,7 @@ func (_m *ElasticsearchInterface) DeleteChannel(channel *model.Channel) *model.A } // DeletePost provides a mock function with given fields: post -func (_m *ElasticsearchInterface) DeletePost(post *model.Post) *model.AppError { +func (_m *SearchEngineInterface) DeletePost(post *model.Post) *model.AppError { ret := _m.Called(post) var r0 *model.AppError @@ -65,7 +65,7 @@ func (_m *ElasticsearchInterface) DeletePost(post *model.Post) *model.AppError { } // DeleteUser provides a mock function with given fields: user -func (_m *ElasticsearchInterface) DeleteUser(user *model.User) *model.AppError { +func (_m *SearchEngineInterface) DeleteUser(user *model.User) *model.AppError { ret := _m.Called(user) var r0 *model.AppError @@ -80,8 +80,22 @@ func (_m *ElasticsearchInterface) DeleteUser(user *model.User) *model.AppError { return r0 } +// GetName provides a mock function with given fields: +func (_m *SearchEngineInterface) GetName() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + // GetVersion provides a mock function with given fields: -func (_m *ElasticsearchInterface) GetVersion() int { +func (_m *SearchEngineInterface) GetVersion() int { ret := _m.Called() var r0 int @@ -95,7 +109,7 @@ func (_m *ElasticsearchInterface) GetVersion() int { } // IndexChannel provides a mock function with given fields: channel -func (_m *ElasticsearchInterface) IndexChannel(channel *model.Channel) *model.AppError { +func (_m *SearchEngineInterface) IndexChannel(channel *model.Channel) *model.AppError { ret := _m.Called(channel) var r0 *model.AppError @@ -111,7 +125,7 @@ func (_m *ElasticsearchInterface) IndexChannel(channel *model.Channel) *model.Ap } // IndexPost provides a mock function with given fields: post, teamId -func (_m *ElasticsearchInterface) IndexPost(post *model.Post, teamId string) *model.AppError { +func (_m *SearchEngineInterface) IndexPost(post *model.Post, teamId string) *model.AppError { ret := _m.Called(post, teamId) var r0 *model.AppError @@ -127,7 +141,7 @@ func (_m *ElasticsearchInterface) IndexPost(post *model.Post, teamId string) *mo } // IndexUser provides a mock function with given fields: user, teamsIds, channelsIds -func (_m *ElasticsearchInterface) IndexUser(user *model.User, teamsIds []string, channelsIds []string) *model.AppError { +func (_m *SearchEngineInterface) IndexUser(user *model.User, teamsIds []string, channelsIds []string) *model.AppError { ret := _m.Called(user, teamsIds, channelsIds) var r0 *model.AppError @@ -142,8 +156,64 @@ func (_m *ElasticsearchInterface) IndexUser(user *model.User, teamsIds []string, return r0 } +// IsActive provides a mock function with given fields: +func (_m *SearchEngineInterface) IsActive() bool { + ret := _m.Called() + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// IsAutocompletionEnabled provides a mock function with given fields: +func (_m *SearchEngineInterface) IsAutocompletionEnabled() bool { + ret := _m.Called() + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// IsIndexingEnabled provides a mock function with given fields: +func (_m *SearchEngineInterface) IsIndexingEnabled() bool { + ret := _m.Called() + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// IsSearchEnabled provides a mock function with given fields: +func (_m *SearchEngineInterface) IsSearchEnabled() bool { + ret := _m.Called() + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + // PurgeIndexes provides a mock function with given fields: -func (_m *ElasticsearchInterface) PurgeIndexes() *model.AppError { +func (_m *SearchEngineInterface) PurgeIndexes() *model.AppError { ret := _m.Called() var r0 *model.AppError @@ -159,7 +229,7 @@ func (_m *ElasticsearchInterface) PurgeIndexes() *model.AppError { } // SearchChannels provides a mock function with given fields: teamId, term -func (_m *ElasticsearchInterface) SearchChannels(teamId string, term string) ([]string, *model.AppError) { +func (_m *SearchEngineInterface) SearchChannels(teamId string, term string) ([]string, *model.AppError) { ret := _m.Called(teamId, term) var r0 []string @@ -184,7 +254,7 @@ func (_m *ElasticsearchInterface) SearchChannels(teamId string, term string) ([] } // SearchPosts provides a mock function with given fields: channels, searchParams, page, perPage -func (_m *ElasticsearchInterface) SearchPosts(channels *model.ChannelList, searchParams []*model.SearchParams, page int, perPage int) ([]string, model.PostSearchMatches, *model.AppError) { +func (_m *SearchEngineInterface) SearchPosts(channels *model.ChannelList, searchParams []*model.SearchParams, page int, perPage int) ([]string, model.PostSearchMatches, *model.AppError) { ret := _m.Called(channels, searchParams, page, perPage) var r0 []string @@ -218,7 +288,7 @@ func (_m *ElasticsearchInterface) SearchPosts(channels *model.ChannelList, searc } // SearchUsersInChannel provides a mock function with given fields: teamId, channelId, restrictedToChannels, term, options -func (_m *ElasticsearchInterface) SearchUsersInChannel(teamId string, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError) { +func (_m *SearchEngineInterface) SearchUsersInChannel(teamId string, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError) { ret := _m.Called(teamId, channelId, restrictedToChannels, term, options) var r0 []string @@ -252,7 +322,7 @@ func (_m *ElasticsearchInterface) SearchUsersInChannel(teamId string, channelId } // SearchUsersInTeam provides a mock function with given fields: teamId, restrictedToChannels, term, options -func (_m *ElasticsearchInterface) SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError) { +func (_m *SearchEngineInterface) SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError) { ret := _m.Called(teamId, restrictedToChannels, term, options) var r0 []string @@ -277,7 +347,7 @@ func (_m *ElasticsearchInterface) SearchUsersInTeam(teamId string, restrictedToC } // Start provides a mock function with given fields: -func (_m *ElasticsearchInterface) Start() *model.AppError { +func (_m *SearchEngineInterface) Start() *model.AppError { ret := _m.Called() var r0 *model.AppError @@ -293,7 +363,7 @@ func (_m *ElasticsearchInterface) Start() *model.AppError { } // Stop provides a mock function with given fields: -func (_m *ElasticsearchInterface) Stop() *model.AppError { +func (_m *SearchEngineInterface) Stop() *model.AppError { ret := _m.Called() var r0 *model.AppError @@ -309,7 +379,7 @@ func (_m *ElasticsearchInterface) Stop() *model.AppError { } // TestConfig provides a mock function with given fields: cfg -func (_m *ElasticsearchInterface) TestConfig(cfg *model.Config) *model.AppError { +func (_m *SearchEngineInterface) TestConfig(cfg *model.Config) *model.AppError { ret := _m.Called(cfg) var r0 *model.AppError @@ -323,3 +393,8 @@ func (_m *ElasticsearchInterface) TestConfig(cfg *model.Config) *model.AppError return r0 } + +// UpdateConfig provides a mock function with given fields: cfg +func (_m *SearchEngineInterface) UpdateConfig(cfg *model.Config) { + _m.Called(cfg) +} diff --git a/services/searchengine/searchengine.go b/services/searchengine/searchengine.go new file mode 100644 index 0000000000..9093da4f83 --- /dev/null +++ b/services/searchengine/searchengine.go @@ -0,0 +1,43 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package searchengine + +import ( + "github.com/mattermost/mattermost-server/v5/jobs" + "github.com/mattermost/mattermost-server/v5/model" +) + +func NewBroker(cfg *model.Config, jobServer *jobs.JobServer) *Broker { + return &Broker{ + cfg: cfg, + jobServer: jobServer, + } +} + +func (seb *Broker) RegisterElasticsearchEngine(es SearchEngineInterface) { + seb.ElasticsearchEngine = es +} + +type Broker struct { + cfg *model.Config + jobServer *jobs.JobServer + ElasticsearchEngine SearchEngineInterface +} + +func (seb *Broker) UpdateConfig(cfg *model.Config) *model.AppError { + seb.cfg = cfg + if seb.ElasticsearchEngine != nil { + seb.ElasticsearchEngine.UpdateConfig(cfg) + } + + return nil +} + +func (seb *Broker) GetActiveEngines() []SearchEngineInterface { + engines := []SearchEngineInterface{} + if seb.ElasticsearchEngine != nil && seb.ElasticsearchEngine.IsActive() { + engines = append(engines, seb.ElasticsearchEngine) + } + return engines +} diff --git a/store/opentracing_layer.go b/store/opentracing_layer.go index 8998cbc614..8976af7112 100644 --- a/store/opentracing_layer.go +++ b/store/opentracing_layer.go @@ -936,7 +936,7 @@ func (s *OpenTracingLayerChannelStore) GetChannelsBatchForIndexing(startTime int return resultVar0, resultVar1 } -func (s *OpenTracingLayerChannelStore) GetChannelsByIds(channelIds []string) ([]*model.Channel, *model.AppError) { +func (s *OpenTracingLayerChannelStore) GetChannelsByIds(channelIds []string, includeDeleted bool) ([]*model.Channel, *model.AppError) { origCtx := s.Root.Store.Context() span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "ChannelStore.GetChannelsByIds") s.Root.Store.SetContext(newCtx) @@ -945,7 +945,7 @@ func (s *OpenTracingLayerChannelStore) GetChannelsByIds(channelIds []string) ([] }() defer span.Finish() - resultVar0, resultVar1 := s.ChannelStore.GetChannelsByIds(channelIds) + resultVar0, resultVar1 := s.ChannelStore.GetChannelsByIds(channelIds, includeDeleted) if resultVar1 != nil { span.LogFields(spanlog.Error(resultVar1)) ext.Error.Set(span, true) @@ -4873,6 +4873,24 @@ func (s *OpenTracingLayerPostStore) Search(teamId string, userId string, params return resultVar0, resultVar1 } +func (s *OpenTracingLayerPostStore) SearchPostsInTeamForUser(paramsList []*model.SearchParams, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, page int, perPage int) (*model.PostSearchResults, *model.AppError) { + origCtx := s.Root.Store.Context() + span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "PostStore.SearchPostsInTeamForUser") + s.Root.Store.SetContext(newCtx) + defer func() { + s.Root.Store.SetContext(origCtx) + }() + + defer span.Finish() + resultVar0, resultVar1 := s.PostStore.SearchPostsInTeamForUser(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage) + if resultVar1 != nil { + span.LogFields(spanlog.Error(resultVar1)) + ext.Error.Set(span, true) + } + + return resultVar0, resultVar1 +} + func (s *OpenTracingLayerPostStore) Update(newPost *model.Post, oldPost *model.Post) (*model.Post, *model.AppError) { origCtx := s.Root.Store.Context() span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "PostStore.Update") @@ -6905,6 +6923,24 @@ func (s *OpenTracingLayerUserStore) AnalyticsGetSystemAdminCount() (int64, *mode return resultVar0, resultVar1 } +func (s *OpenTracingLayerUserStore) AutocompleteUsersInChannel(teamId string, channelId string, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) { + origCtx := s.Root.Store.Context() + span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "UserStore.AutocompleteUsersInChannel") + s.Root.Store.SetContext(newCtx) + defer func() { + s.Root.Store.SetContext(origCtx) + }() + + defer span.Finish() + resultVar0, resultVar1 := s.UserStore.AutocompleteUsersInChannel(teamId, channelId, term, options) + if resultVar1 != nil { + span.LogFields(spanlog.Error(resultVar1)) + ext.Error.Set(span, true) + } + + return resultVar0, resultVar1 +} + func (s *OpenTracingLayerUserStore) ClearAllCustomRoleAssignments() *model.AppError { origCtx := s.Root.Store.Context() span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "UserStore.ClearAllCustomRoleAssignments") diff --git a/store/searchlayer/channel_layer.go b/store/searchlayer/channel_layer.go new file mode 100644 index 0000000000..d5214a3929 --- /dev/null +++ b/store/searchlayer/channel_layer.go @@ -0,0 +1,207 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package searchlayer + +import ( + "github.com/mattermost/mattermost-server/v5/mlog" + "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/services/searchengine" + "github.com/mattermost/mattermost-server/v5/store" +) + +type SearchChannelStore struct { + store.ChannelStore + rootStore *SearchStore +} + +func (c *SearchChannelStore) deleteChannelIndex(channel *model.Channel) { + if channel.Type == model.CHANNEL_OPEN { + for _, engine := range c.rootStore.searchEngine.GetActiveEngines() { + if engine.IsIndexingEnabled() { + go (func(engineCopy searchengine.SearchEngineInterface) { + if err := engineCopy.DeleteChannel(channel); err != nil { + mlog.Error("Encountered error deleting channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + } + mlog.Debug("Removed channel from index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id)) + })(engine) + } + } + } +} + +func (c *SearchChannelStore) indexChannel(channel *model.Channel) { + if channel.Type == model.CHANNEL_OPEN { + for _, engine := range c.rootStore.searchEngine.GetActiveEngines() { + if engine.IsIndexingEnabled() { + go (func(engineCopy searchengine.SearchEngineInterface) { + if err := engineCopy.IndexChannel(channel); err != nil { + mlog.Error("Encountered error indexing channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + } + mlog.Debug("Indexed channel in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id)) + })(engine) + } + } + } +} + +func (c *SearchChannelStore) Save(channel *model.Channel, maxChannels int64) (*model.Channel, *model.AppError) { + newChannel, err := c.ChannelStore.Save(channel, maxChannels) + if err == nil { + c.indexChannel(newChannel) + } + return newChannel, err +} + +func (c *SearchChannelStore) Update(channel *model.Channel) (*model.Channel, *model.AppError) { + updatedChannel, err := c.ChannelStore.Update(channel) + if err == nil { + c.indexChannel(updatedChannel) + } + return updatedChannel, err +} + +func (c *SearchChannelStore) UpdateMember(cm *model.ChannelMember) (*model.ChannelMember, *model.AppError) { + member, err := c.ChannelStore.UpdateMember(cm) + if err == nil { + c.rootStore.indexUserFromID(cm.UserId) + channel, channelErr := c.ChannelStore.Get(member.ChannelId, true) + if channelErr != nil { + mlog.Error("Encountered error indexing user in channel", mlog.String("channel_id", member.ChannelId), mlog.Err(err)) + } else { + c.rootStore.indexUserFromID(channel.CreatorId) + } + } + return member, err +} + +func (c *SearchChannelStore) SaveMember(cm *model.ChannelMember) (*model.ChannelMember, *model.AppError) { + member, err := c.ChannelStore.SaveMember(cm) + if err == nil { + c.rootStore.indexUserFromID(cm.UserId) + channel, channelErr := c.ChannelStore.Get(member.ChannelId, true) + if channelErr != nil { + mlog.Error("Encountered error indexing user in channel", mlog.String("channel_id", member.ChannelId), mlog.Err(err)) + } else { + c.rootStore.indexUserFromID(channel.CreatorId) + } + } + return member, err +} + +func (c *SearchChannelStore) RemoveMember(channelId, userIdToRemove string) *model.AppError { + err := c.ChannelStore.RemoveMember(channelId, userIdToRemove) + if err == nil { + c.rootStore.indexUserFromID(userIdToRemove) + } + return err +} + +func (c *SearchChannelStore) CreateDirectChannel(user *model.User, otherUser *model.User) (*model.Channel, *model.AppError) { + channel, err := c.ChannelStore.CreateDirectChannel(user, otherUser) + if err == nil { + c.rootStore.indexUserFromID(user.Id) + c.rootStore.indexUserFromID(otherUser.Id) + } + return channel, err +} + +func (c *SearchChannelStore) AutocompleteInTeam(teamId string, term string, includeDeleted bool) (*model.ChannelList, *model.AppError) { + var channelList *model.ChannelList + var err *model.AppError + + allFailed := true + for _, engine := range c.rootStore.searchEngine.GetActiveEngines() { + if engine.IsAutocompletionEnabled() { + channelList, err = c.searchAutocompleteChannels(engine, teamId, term, includeDeleted) + if err != nil { + mlog.Error("Encountered error on AutocompleteChannels through SearchEngine. Falling back to default autocompletion.", mlog.String("search_engine", engine.GetName()), mlog.Err(err)) + continue + } + allFailed = false + mlog.Debug("Using the first available search engine", mlog.String("search_engine", engine.GetName())) + break + } + } + + if allFailed { + mlog.Debug("Using database search because no other search engine is available") + channelList, err = c.ChannelStore.AutocompleteInTeam(teamId, term, includeDeleted) + if err != nil { + return nil, err + } + } + return channelList, err +} + +func (c *SearchChannelStore) searchAutocompleteChannels(engine searchengine.SearchEngineInterface, teamId, term string, includeDeleted bool) (*model.ChannelList, *model.AppError) { + channelIds, err := engine.SearchChannels(teamId, term) + if err != nil { + return nil, err + } + + channelList := model.ChannelList{} + if len(channelIds) > 0 { + channels, err := c.ChannelStore.GetChannelsByIds(channelIds, includeDeleted) + if err != nil { + return nil, err + } + for _, ch := range channels { + channelList = append(channelList, ch) + } + } + + return &channelList, nil +} + +func (c *SearchChannelStore) PermanentDeleteMembersByUser(userId string) *model.AppError { + err := c.ChannelStore.PermanentDeleteMembersByUser(userId) + if err == nil { + c.rootStore.indexUserFromID(userId) + } + return err +} + +func (c *SearchChannelStore) RemoveAllDeactivatedMembers(channelId string) *model.AppError { + profiles, errProfiles := c.rootStore.User().GetAllProfilesInChannel(channelId, true) + if errProfiles != nil { + mlog.Error("Encountered error indexing users for channel", mlog.String("channel_id", channelId), mlog.Err(errProfiles)) + } + + err := c.ChannelStore.RemoveAllDeactivatedMembers(channelId) + if err == nil && errProfiles == nil { + for _, user := range profiles { + if user.DeleteAt != 0 { + c.rootStore.indexUser(user) + } + } + } + return err +} + +func (c *SearchChannelStore) PermanentDeleteMembersByChannel(channelId string) *model.AppError { + profiles, errProfiles := c.rootStore.User().GetAllProfilesInChannel(channelId, true) + if errProfiles != nil { + mlog.Error("Encountered error indexing users for channel", mlog.String("channel_id", channelId), mlog.Err(errProfiles)) + } + + err := c.ChannelStore.PermanentDeleteMembersByChannel(channelId) + if err == nil && errProfiles == nil { + for _, user := range profiles { + c.rootStore.indexUser(user) + } + } + return err +} + +func (c *SearchChannelStore) PermanentDelete(channelId string) *model.AppError { + channel, channelErr := c.ChannelStore.Get(channelId, true) + if channelErr != nil { + mlog.Error("Encountered error deleting channel", mlog.String("channel_id", channelId), mlog.Err(channelErr)) + } + err := c.ChannelStore.PermanentDelete(channelId) + if err == nil && channelErr == nil { + c.deleteChannelIndex(channel) + } + return err +} diff --git a/store/searchlayer/layer.go b/store/searchlayer/layer.go new file mode 100644 index 0000000000..b843f5c1a8 --- /dev/null +++ b/store/searchlayer/layer.go @@ -0,0 +1,93 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package searchlayer + +import ( + "github.com/mattermost/mattermost-server/v5/mlog" + "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/services/searchengine" + "github.com/mattermost/mattermost-server/v5/store" +) + +type SearchStore struct { + store.Store + searchEngine *searchengine.Broker + user *SearchUserStore + team *SearchTeamStore + channel *SearchChannelStore + post *SearchPostStore +} + +func NewSearchLayer(baseStore store.Store, searchEngine *searchengine.Broker) SearchStore { + searchStore := SearchStore{ + Store: baseStore, + searchEngine: searchEngine, + } + searchStore.channel = &SearchChannelStore{ChannelStore: baseStore.Channel(), rootStore: &searchStore} + searchStore.post = &SearchPostStore{PostStore: baseStore.Post(), rootStore: &searchStore} + searchStore.team = &SearchTeamStore{TeamStore: baseStore.Team(), rootStore: &searchStore} + searchStore.user = &SearchUserStore{UserStore: baseStore.User(), rootStore: &searchStore} + + return searchStore +} + +func (s SearchStore) Channel() store.ChannelStore { + return s.channel +} + +func (s SearchStore) Post() store.PostStore { + return s.post +} + +func (s SearchStore) Team() store.TeamStore { + return s.team +} + +func (s SearchStore) User() store.UserStore { + return s.user +} + +func (s SearchStore) indexUserFromID(userId string) { + user, err := s.User().Get(userId) + if err != nil { + return + } + s.indexUser(user) +} + +func (s SearchStore) indexUser(user *model.User) { + for _, engine := range s.searchEngine.GetActiveEngines() { + if engine.IsIndexingEnabled() { + go (func(engineCopy searchengine.SearchEngineInterface) { + userTeams, err := s.Team().GetTeamsByUserId(user.Id) + if err != nil { + mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + return + } + + userTeamsIds := []string{} + for _, team := range userTeams { + userTeamsIds = append(userTeamsIds, team.Id) + } + + userChannelMembers, err := s.Channel().GetAllChannelMembersForUser(user.Id, false, true) + if err != nil { + mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + return + } + + userChannelsIds := []string{} + for channelId := range userChannelMembers { + userChannelsIds = append(userChannelsIds, channelId) + } + + if err := engineCopy.IndexUser(user, userTeamsIds, userChannelsIds); err != nil { + mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + return + } + mlog.Debug("Indexed user in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("user_id", user.Id)) + })(engine) + } + } +} diff --git a/store/searchlayer/post_layer.go b/store/searchlayer/post_layer.go new file mode 100644 index 0000000000..4e53e5d40e --- /dev/null +++ b/store/searchlayer/post_layer.go @@ -0,0 +1,126 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package searchlayer + +import ( + "github.com/mattermost/mattermost-server/v5/mlog" + "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/services/searchengine" + "github.com/mattermost/mattermost-server/v5/store" +) + +type SearchPostStore struct { + store.PostStore + rootStore *SearchStore +} + +func (s SearchPostStore) indexPost(post *model.Post) { + for _, engine := range s.rootStore.searchEngine.GetActiveEngines() { + if engine.IsIndexingEnabled() { + go (func(engineCopy searchengine.SearchEngineInterface) { + channel, chanErr := s.rootStore.Channel().GetForPost(post.Id) + if chanErr != nil { + mlog.Error("Couldn't get channel for post for SearchEngine indexing.", mlog.String("channel_id", post.ChannelId), mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id)) + return + } + if err := engineCopy.IndexPost(post, channel.TeamId); err != nil { + mlog.Error("Encountered error indexing post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + } + mlog.Debug("Indexed post in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id)) + })(engine) + } + } +} + +func (s SearchPostStore) deletePostIndex(post *model.Post) { + for _, engine := range s.rootStore.searchEngine.GetActiveEngines() { + if engine.IsIndexingEnabled() { + go (func(engineCopy searchengine.SearchEngineInterface) { + if err := engineCopy.DeletePost(post); err != nil { + mlog.Error("Encountered error deleting post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + } + mlog.Debug("Removed post from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id)) + })(engine) + } + } +} + +func (s SearchPostStore) Update(newPost, oldPost *model.Post) (*model.Post, *model.AppError) { + post, err := s.PostStore.Update(newPost, oldPost) + + if err == nil { + s.indexPost(post) + } + return post, err +} + +func (s SearchPostStore) Save(post *model.Post) (*model.Post, *model.AppError) { + npost, err := s.PostStore.Save(post) + + if err == nil { + s.indexPost(npost) + } + return npost, err +} + +func (s SearchPostStore) Delete(postId string, date int64, deletedByID string) *model.AppError { + err := s.PostStore.Delete(postId, date, deletedByID) + + if err == nil { + postList, err2 := s.PostStore.Get(postId, true) + if postList != nil && len(postList.Order) > 0 { + if err2 != nil { + s.deletePostIndex(postList.Posts[postList.Order[0]]) + } + } + } + return err +} + +func (s SearchPostStore) searchPostsInTeamForUserByEngine(engine searchengine.SearchEngineInterface, paramsList []*model.SearchParams, userId, teamId string, isOrSearch, includeDeletedChannels bool, page, perPage int) (*model.PostSearchResults, *model.AppError) { + // We only allow the user to search in channels they are a member of. + userChannels, err := s.rootStore.Channel().GetChannels(teamId, userId, includeDeletedChannels) + if err != nil { + mlog.Error("error getting channel for user", mlog.Err(err)) + return nil, err + } + + postIds, matches, err := engine.SearchPosts(userChannels, paramsList, page, perPage) + if err != nil { + return nil, err + } + + // Get the posts + postList := model.NewPostList() + if len(postIds) > 0 { + posts, err := s.PostStore.GetPostsByIds(postIds) + if err != nil { + return nil, err + } + for _, p := range posts { + if p.DeleteAt == 0 { + postList.AddPost(p) + postList.AddOrder(p.Id) + } + } + } + + return model.MakePostSearchResults(postList, matches), nil +} + +func (s SearchPostStore) SearchPostsInTeamForUser(paramsList []*model.SearchParams, userId, teamId string, isOrSearch, includeDeletedChannels bool, page, perPage int) (*model.PostSearchResults, *model.AppError) { + for _, engine := range s.rootStore.searchEngine.GetActiveEngines() { + if engine.IsSearchEnabled() { + results, err := s.searchPostsInTeamForUserByEngine(engine, paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage) + if err != nil { + mlog.Error("Encountered error on SearchPostsInTeamForUser.", mlog.String("search_engine", engine.GetName()), mlog.Err(err)) + continue + } + mlog.Debug("Using the first available search engine", mlog.String("search_engine", engine.GetName())) + return results, err + } + } + mlog.Debug("Using database search because no other search engine is available") + return s.PostStore.SearchPostsInTeamForUser(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage) +} diff --git a/store/searchlayer/team_layer.go b/store/searchlayer/team_layer.go new file mode 100644 index 0000000000..0408b7b11a --- /dev/null +++ b/store/searchlayer/team_layer.go @@ -0,0 +1,46 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package searchlayer + +import ( + model "github.com/mattermost/mattermost-server/v5/model" + store "github.com/mattermost/mattermost-server/v5/store" +) + +type SearchTeamStore struct { + store.TeamStore + rootStore *SearchStore +} + +func (s SearchTeamStore) SaveMember(teamMember *model.TeamMember, maxUsersPerTeam int) (*model.TeamMember, *model.AppError) { + member, err := s.TeamStore.SaveMember(teamMember, maxUsersPerTeam) + if err == nil { + s.rootStore.indexUserFromID(member.UserId) + } + return member, err +} + +func (s SearchTeamStore) UpdateMember(teamMember *model.TeamMember) (*model.TeamMember, *model.AppError) { + member, err := s.TeamStore.UpdateMember(teamMember) + if err == nil { + s.rootStore.indexUserFromID(member.UserId) + } + return member, err +} + +func (s SearchTeamStore) RemoveMember(teamId string, userId string) *model.AppError { + err := s.TeamStore.RemoveMember(teamId, userId) + if err == nil { + s.rootStore.indexUserFromID(userId) + } + return err +} + +func (s SearchTeamStore) RemoveAllMembersByUser(userId string) *model.AppError { + err := s.TeamStore.RemoveAllMembersByUser(userId) + if err == nil { + s.rootStore.indexUserFromID(userId) + } + return err +} diff --git a/store/searchlayer/user_layer.go b/store/searchlayer/user_layer.go new file mode 100644 index 0000000000..95c4e903f8 --- /dev/null +++ b/store/searchlayer/user_layer.go @@ -0,0 +1,192 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package searchlayer + +import ( + "strings" + + "github.com/mattermost/mattermost-server/v5/mlog" + "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/services/searchengine" + "github.com/mattermost/mattermost-server/v5/store" +) + +type SearchUserStore struct { + store.UserStore + rootStore *SearchStore +} + +func (s *SearchUserStore) deleteUserIndex(user *model.User) { + for _, engine := range s.rootStore.searchEngine.GetActiveEngines() { + if engine.IsIndexingEnabled() { + go (func(engineCopy searchengine.SearchEngineInterface) { + if err := engineCopy.DeleteUser(user); err != nil { + mlog.Error("Encountered error deleting user", mlog.String("user_id", user.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + return + } + mlog.Debug("Removed user from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("user_id", user.Id)) + })(engine) + } + } +} + +func (s *SearchUserStore) Search(teamId, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError) { + for _, engine := range s.rootStore.searchEngine.GetActiveEngines() { + if engine.IsSearchEnabled() { + listOfAllowedChannels, err := s.getListOfAllowedChannelsForTeam(teamId, options.ViewRestrictions) + if err != nil { + mlog.Error("Encountered error on Search.", mlog.String("search_engine", engine.GetName()), mlog.Err(err)) + continue + } + if len(listOfAllowedChannels) == 0 { + return []*model.User{}, nil + } + + usersIds, err := engine.SearchUsersInTeam(teamId, listOfAllowedChannels, term, options) + if err != nil { + continue + } + + users, err := s.UserStore.GetProfileByIds(usersIds, nil, false) + if err != nil { + continue + } + + mlog.Debug("Using the first available search engine", mlog.String("search_engine", engine.GetName())) + return users, nil + } + } + mlog.Debug("Using database search because no other search engine is available") + return s.UserStore.Search(teamId, term, options) +} + +func (s *SearchUserStore) Update(user *model.User, trustedUpdateData bool) (*model.UserUpdate, *model.AppError) { + userUpdate, err := s.UserStore.Update(user, trustedUpdateData) + + if err == nil { + s.rootStore.indexUser(userUpdate.New) + } + return userUpdate, err +} + +func (s *SearchUserStore) Save(user *model.User) (*model.User, *model.AppError) { + nuser, err := s.UserStore.Save(user) + + if err == nil { + s.rootStore.indexUser(nuser) + } + return nuser, err +} + +func (s *SearchUserStore) PermanentDelete(userId string) *model.AppError { + user, userErr := s.UserStore.Get(userId) + if userErr != nil { + mlog.Error("Encountered error deleting user", mlog.String("user_id", userId), mlog.Err(userErr)) + } + err := s.UserStore.PermanentDelete(userId) + if err == nil && userErr == nil { + s.deleteUserIndex(user) + } + return err +} + +func (s *SearchUserStore) autocompleteUsersInChannelByEngine(engine searchengine.SearchEngineInterface, teamId, channelId, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) { + var err *model.AppError + uchanIds := []string{} + nuchanIds := []string{} + if options.ListOfAllowedChannels != nil && !strings.Contains(strings.Join(options.ListOfAllowedChannels, "."), channelId) { + nuchanIds, err = engine.SearchUsersInTeam(teamId, options.ListOfAllowedChannels, term, options) + } else { + uchanIds, nuchanIds, err = engine.SearchUsersInChannel(teamId, channelId, options.ListOfAllowedChannels, term, options) + } + if err != nil { + return nil, err + } + + uchan := make(chan store.StoreResult, 1) + go func() { + users, err := s.UserStore.GetProfileByIds(uchanIds, nil, false) + uchan <- store.StoreResult{Data: users, Err: err} + close(uchan) + }() + + nuchan := make(chan store.StoreResult, 1) + go func() { + users, err := s.UserStore.GetProfileByIds(nuchanIds, nil, false) + nuchan <- store.StoreResult{Data: users, Err: err} + close(nuchan) + }() + + autocomplete := &model.UserAutocompleteInChannel{} + + result := <-uchan + if result.Err != nil { + return nil, result.Err + } + inUsers := result.Data.([]*model.User) + autocomplete.InChannel = inUsers + + result = <-nuchan + if result.Err != nil { + return nil, result.Err + } + outUsers := result.Data.([]*model.User) + autocomplete.OutOfChannel = outUsers + + return autocomplete, nil +} + +func (s *SearchUserStore) getListOfAllowedChannelsForTeam(teamId string, viewRestrictions *model.ViewUsersRestrictions) ([]string, *model.AppError) { + var listOfAllowedChannels []string + if viewRestrictions == nil || strings.Contains(strings.Join(viewRestrictions.Teams, "."), teamId) { + channels, err := s.rootStore.Channel().GetTeamChannels(teamId) + if err != nil { + return nil, err + } + channelIds := []string{} + for _, channel := range *channels { + channelIds = append(channelIds, channel.Id) + } + + return channelIds, nil + } + + channels, err := s.rootStore.Channel().GetChannelsByIds(viewRestrictions.Channels, false) + if err != nil { + return nil, err + } + for _, c := range channels { + if c.TeamId == teamId { + listOfAllowedChannels = append(listOfAllowedChannels, c.Id) + } + } + + return listOfAllowedChannels, nil +} + +func (s *SearchUserStore) AutocompleteUsersInChannel(teamId, channelId, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) { + for _, engine := range s.rootStore.searchEngine.GetActiveEngines() { + if engine.IsAutocompletionEnabled() { + listOfAllowedChannels, err := s.getListOfAllowedChannelsForTeam(teamId, options.ViewRestrictions) + if err != nil { + mlog.Error("Encountered error on AutocompleteUsersInChannel.", mlog.String("search_engine", engine.GetName()), mlog.Err(err)) + continue + } + if len(listOfAllowedChannels) == 0 { + return &model.UserAutocompleteInChannel{}, nil + } + options.ListOfAllowedChannels = listOfAllowedChannels + autocomplete, err := s.autocompleteUsersInChannelByEngine(engine, teamId, channelId, term, options) + if err != nil { + mlog.Error("Encountered error on AutocompleteUsersInChannel.", mlog.String("search_engine", engine.GetName()), mlog.Err(err)) + continue + } + mlog.Debug("Using the first available search engine", mlog.String("search_engine", engine.GetName())) + return autocomplete, err + } + } + + mlog.Debug("Using database search because no other search engine is available") + return s.UserStore.AutocompleteUsersInChannel(teamId, channelId, term, options) +} diff --git a/store/sqlstore/channel_store.go b/store/sqlstore/channel_store.go index 0a9c6d7745..717dccf8e7 100644 --- a/store/sqlstore/channel_store.go +++ b/store/sqlstore/channel_store.go @@ -1865,9 +1865,12 @@ func (s SqlChannelStore) GetAll(teamId string) ([]*model.Channel, *model.AppErro return data, nil } -func (s SqlChannelStore) GetChannelsByIds(channelIds []string) ([]*model.Channel, *model.AppError) { +func (s SqlChannelStore) GetChannelsByIds(channelIds []string, includeDeleted bool) ([]*model.Channel, *model.AppError) { keys, params := MapStringsToQueryParams(channelIds, "Channel") query := `SELECT * FROM Channels WHERE Id IN ` + keys + ` ORDER BY Name` + if !includeDeleted { + query = `SELECT * FROM Channels WHERE DeleteAt=0 AND Id IN ` + keys + ` ORDER BY Name` + } var channels []*model.Channel _, err := s.GetReplica().Select(&channels, query, params) diff --git a/store/sqlstore/post_store.go b/store/sqlstore/post_store.go index 78c6d82172..f1750bb0aa 100644 --- a/store/sqlstore/post_store.go +++ b/store/sqlstore/post_store.go @@ -999,7 +999,7 @@ func (s *SqlPostStore) buildCreateDateFilterClause(params *model.SearchParams, q return searchQuery, queryParams } -func (s *SqlPostStore) buildSearchChannelFilterClause(channels []string, paramPrefix string, exclusion bool, queryParams map[string]interface{}) (string, map[string]interface{}) { +func (s *SqlPostStore) buildSearchChannelFilterClause(channels []string, paramPrefix string, exclusion bool, queryParams map[string]interface{}, byName bool) (string, map[string]interface{}) { if len(channels) == 0 { return "", queryParams } @@ -1011,13 +1011,20 @@ func (s *SqlPostStore) buildSearchChannelFilterClause(channels []string, paramPr queryParams[paramName] = channel } clause := strings.Join(clauseSlice, ", ") - if exclusion { - return "AND Name NOT IN (" + clause + ")", queryParams + if byName { + if exclusion { + return "AND Name NOT IN (" + clause + ")", queryParams + } + return "AND Name IN (" + clause + ")", queryParams } - return "AND Name IN (" + clause + ")", queryParams + + if exclusion { + return "AND Id NOT IN (" + clause + ")", queryParams + } + return "AND Id IN (" + clause + ")", queryParams } -func (s *SqlPostStore) buildSearchUserFilterClause(users []string, paramPrefix string, exclusion bool, queryParams map[string]interface{}) (string, map[string]interface{}) { +func (s *SqlPostStore) buildSearchUserFilterClause(users []string, paramPrefix string, exclusion bool, queryParams map[string]interface{}, byUsername bool) (string, map[string]interface{}) { if len(users) == 0 { return "", queryParams } @@ -1028,13 +1035,19 @@ func (s *SqlPostStore) buildSearchUserFilterClause(users []string, paramPrefix s queryParams[paramName] = user } clause := strings.Join(clauseSlice, ", ") - if exclusion { - return "AND Username NOT IN (" + clause + ")", queryParams + if byUsername { + if exclusion { + return "AND Username NOT IN (" + clause + ")", queryParams + } + return "AND Username IN (" + clause + ")", queryParams } - return "AND Username IN (" + clause + ")", queryParams + if exclusion { + return "AND Id NOT IN (" + clause + ")", queryParams + } + return "AND Id IN (" + clause + ")", queryParams } -func (s *SqlPostStore) buildSearchPostFilterClause(fromUsers []string, excludedUsers []string, queryParams map[string]interface{}) (string, map[string]interface{}) { +func (s *SqlPostStore) buildSearchPostFilterClause(fromUsers []string, excludedUsers []string, queryParams map[string]interface{}, userByUsername bool) (string, map[string]interface{}) { if len(fromUsers) == 0 && len(excludedUsers) == 0 { return "", queryParams } @@ -1052,16 +1065,20 @@ func (s *SqlPostStore) buildSearchPostFilterClause(fromUsers []string, excludedU FROM_USER_FILTER EXCLUDED_USER_FILTER)` - fromUserClause, queryParams := s.buildSearchUserFilterClause(fromUsers, "FromUser", false, queryParams) + fromUserClause, queryParams := s.buildSearchUserFilterClause(fromUsers, "FromUser", false, queryParams, userByUsername) filterQuery = strings.Replace(filterQuery, "FROM_USER_FILTER", fromUserClause, 1) - excludedUserClause, queryParams := s.buildSearchUserFilterClause(excludedUsers, "ExcludedUser", true, queryParams) + excludedUserClause, queryParams := s.buildSearchUserFilterClause(excludedUsers, "ExcludedUser", true, queryParams, userByUsername) filterQuery = strings.Replace(filterQuery, "EXCLUDED_USER_FILTER", excludedUserClause, 1) return filterQuery, queryParams } func (s *SqlPostStore) Search(teamId string, userId string, params *model.SearchParams) (*model.PostList, *model.AppError) { + return s.search(teamId, userId, params, true, true) +} + +func (s *SqlPostStore) search(teamId string, userId string, params *model.SearchParams, channelsByName bool, userByUsername bool) (*model.PostList, *model.AppError) { queryParams := map[string]interface{}{ "TeamId": teamId, "UserId": userId, @@ -1114,13 +1131,13 @@ func (s *SqlPostStore) Search(teamId string, userId string, params *model.Search ORDER BY CreateAt DESC LIMIT 100` - inChannelClause, queryParams := s.buildSearchChannelFilterClause(params.InChannels, "InChannel", false, queryParams) + inChannelClause, queryParams := s.buildSearchChannelFilterClause(params.InChannels, "InChannel", false, queryParams, channelsByName) searchQuery = strings.Replace(searchQuery, "IN_CHANNEL_FILTER", inChannelClause, 1) - excludedChannelClause, queryParams := s.buildSearchChannelFilterClause(params.ExcludedChannels, "ExcludedChannel", true, queryParams) + excludedChannelClause, queryParams := s.buildSearchChannelFilterClause(params.ExcludedChannels, "ExcludedChannel", true, queryParams, channelsByName) searchQuery = strings.Replace(searchQuery, "EXCLUDED_CHANNEL_FILTER", excludedChannelClause, 1) - postFilterClause, queryParams := s.buildSearchPostFilterClause(params.FromUsers, params.ExcludedUsers, queryParams) + postFilterClause, queryParams := s.buildSearchPostFilterClause(params.FromUsers, params.ExcludedUsers, queryParams, userByUsername) searchQuery = strings.Replace(searchQuery, "POST_FILTER", postFilterClause, 1) createDateFilterClause, queryParams := s.buildCreateDateFilterClause(params, queryParams) @@ -1666,3 +1683,49 @@ func (s *SqlPostStore) GetDirectPostParentsForExportAfter(limit int, afterId str } return posts, nil } + +func (s *SqlPostStore) SearchPostsInTeamForUser(paramsList []*model.SearchParams, userId, teamId string, isOrSearch, includeDeletedChannels bool, page, perPage int) (*model.PostSearchResults, *model.AppError) { + // Since we don't support paging for DB search, we just return nothing for later pages + if page > 0 { + return model.MakePostSearchResults(model.NewPostList(), nil), nil + } + + var wg sync.WaitGroup + + pchan := make(chan store.StoreResult, len(paramsList)) + + for _, params := range paramsList { + // Don't allow users to search for everything. + if params.Terms == "*" { + continue + } + + params.IncludeDeletedChannels = includeDeletedChannels + params.OrTerms = isOrSearch + + wg.Add(1) + + go func(params *model.SearchParams) { + defer wg.Done() + postList, err := s.search(teamId, userId, params, false, false) + pchan <- store.StoreResult{Data: postList, Err: err} + }(params) + } + + wg.Wait() + close(pchan) + + posts := model.NewPostList() + + for result := range pchan { + if result.Err != nil { + return nil, result.Err + } + data := result.Data.(*model.PostList) + posts.Extend(data) + } + + posts.SortByCreateAt() + + return model.MakePostSearchResults(posts, nil), nil +} diff --git a/store/sqlstore/user_store.go b/store/sqlstore/user_store.go index 5e3695d9da..f8d4e5ac85 100644 --- a/store/sqlstore/user_store.go +++ b/store/sqlstore/user_store.go @@ -1716,3 +1716,35 @@ func (us SqlUserStore) DemoteUserToGuest(userId string) *model.AppError { } return nil } + +func (us SqlUserStore) AutocompleteUsersInChannel(teamId, channelId, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) { + autocomplete := &model.UserAutocompleteInChannel{} + uchan := make(chan store.StoreResult, 1) + go func() { + users, err := us.SearchInChannel(channelId, term, options) + uchan <- store.StoreResult{Data: users, Err: err} + close(uchan) + }() + + nuchan := make(chan store.StoreResult, 1) + go func() { + users, err := us.SearchNotInChannel(teamId, channelId, term, options) + nuchan <- store.StoreResult{Data: users, Err: err} + close(nuchan) + }() + + result := <-uchan + if result.Err != nil { + return nil, result.Err + } + users := result.Data.([]*model.User) + autocomplete.InChannel = users + + result = <-nuchan + if result.Err != nil { + return nil, result.Err + } + users = result.Data.([]*model.User) + autocomplete.OutOfChannel = users + return autocomplete, nil +} diff --git a/store/store.go b/store/store.go index c4612d7008..26e61a1d18 100644 --- a/store/store.go +++ b/store/store.go @@ -148,7 +148,7 @@ type ChannelStore interface { GetChannelCounts(teamId string, userId string) (*model.ChannelCounts, *model.AppError) GetTeamChannels(teamId string) (*model.ChannelList, *model.AppError) GetAll(teamId string) ([]*model.Channel, *model.AppError) - GetChannelsByIds(channelIds []string) ([]*model.Channel, *model.AppError) + GetChannelsByIds(channelIds []string, includeDeleted bool) ([]*model.Channel, *model.AppError) GetForPost(postId string) (*model.Channel, *model.AppError) SaveMember(member *model.ChannelMember) (*model.ChannelMember, *model.AppError) UpdateMember(member *model.ChannelMember) (*model.ChannelMember, *model.AppError) @@ -256,6 +256,7 @@ type PostStore interface { GetParentsForExportAfter(limit int, afterId string) ([]*model.PostForExport, *model.AppError) GetRepliesForExport(parentId string) ([]*model.ReplyForExport, *model.AppError) GetDirectPostParentsForExportAfter(limit int, afterId string) ([]*model.DirectPostForExport, *model.AppError) + SearchPostsInTeamForUser(paramsList []*model.SearchParams, userId, teamId string, isOrSearch, includeDeletedChannels bool, page, perPage int) (*model.PostSearchResults, *model.AppError) } type UserStore interface { @@ -320,6 +321,7 @@ type UserStore interface { PromoteGuestToUser(userID string) *model.AppError DemoteUserToGuest(userID string) *model.AppError DeactivateGuests() ([]string, *model.AppError) + AutocompleteUsersInChannel(teamId, channelId, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) } type BotStore interface { diff --git a/store/storetest/channel_store.go b/store/storetest/channel_store.go index bc0f8ca451..5f206fbbc2 100644 --- a/store/storetest/channel_store.go +++ b/store/storetest/channel_store.go @@ -447,6 +447,18 @@ func testChannelStoreGetChannelsByIds(t *testing.T, ss store.Store) { o2.Name = "bb" + model.NewId() + "b" o2.Type = model.CHANNEL_DIRECT + o3 := model.Channel{} + o3.TeamId = model.NewId() + o3.DisplayName = "Deleted channel" + o3.Name = "cc" + model.NewId() + "b" + o3.Type = model.CHANNEL_OPEN + _, err = ss.Channel().Save(&o3, -1) + require.Nil(t, err) + err = ss.Channel().Delete(o3.Id, 123) + require.Nil(t, err) + o3.DeleteAt = 123 + o3.UpdateAt = 123 + m1 := model.ChannelMember{} m1.ChannelId = o2.Id m1.UserId = u1.Id @@ -460,17 +472,30 @@ func testChannelStoreGetChannelsByIds(t *testing.T, ss store.Store) { _, err = ss.Channel().SaveDirectChannel(&o2, &m1, &m2) require.Nil(t, err) - r1, err := ss.Channel().GetChannelsByIds([]string{o1.Id, o2.Id}) - require.Nil(t, err, err) - require.Len(t, r1, 2, "invalid returned channels, exepected 2 and got "+strconv.Itoa(len(r1))) - require.Equal(t, o1.ToJson(), r1[0].ToJson()) - require.Equal(t, o2.ToJson(), r1[1].ToJson()) + t.Run("Get 2 existing channels", func(t *testing.T) { + r1, err := ss.Channel().GetChannelsByIds([]string{o1.Id, o2.Id}, false) + require.Nil(t, err, err) + require.Len(t, r1, 2, "invalid returned channels, exepected 2 and got "+strconv.Itoa(len(r1))) + require.Equal(t, o1.ToJson(), r1[0].ToJson()) + require.Equal(t, o2.ToJson(), r1[1].ToJson()) + }) - nonexistentId := "abcd1234" - r2, err := ss.Channel().GetChannelsByIds([]string{o1.Id, nonexistentId}) - require.Nil(t, err, err) - require.Len(t, r2, 1, "invalid returned channels, expected 1 and got "+strconv.Itoa(len(r2))) - require.Equal(t, o1.ToJson(), r2[0].ToJson(), "invalid returned channel") + t.Run("Get 1 existing and 1 not existing channel", func(t *testing.T) { + nonexistentId := "abcd1234" + r2, err := ss.Channel().GetChannelsByIds([]string{o1.Id, nonexistentId}, false) + require.Nil(t, err, err) + require.Len(t, r2, 1, "invalid returned channels, expected 1 and got "+strconv.Itoa(len(r2))) + require.Equal(t, o1.ToJson(), r2[0].ToJson(), "invalid returned channel") + }) + + t.Run("Get 2 existing and 1 deleted channel", func(t *testing.T) { + r1, err := ss.Channel().GetChannelsByIds([]string{o1.Id, o2.Id, o3.Id}, true) + require.Nil(t, err, err) + require.Len(t, r1, 3, "invalid returned channels, exepected 3 and got "+strconv.Itoa(len(r1))) + require.Equal(t, o1.ToJson(), r1[0].ToJson()) + require.Equal(t, o2.ToJson(), r1[1].ToJson()) + require.Equal(t, o3.ToJson(), r1[2].ToJson()) + }) } func testChannelStoreGetForPost(t *testing.T, ss store.Store) { diff --git a/store/storetest/mocks/ChannelStore.go b/store/storetest/mocks/ChannelStore.go index 23029d9cc2..c96e38754b 100644 --- a/store/storetest/mocks/ChannelStore.go +++ b/store/storetest/mocks/ChannelStore.go @@ -619,13 +619,13 @@ func (_m *ChannelStore) GetChannelsBatchForIndexing(startTime int64, endTime int return r0, r1 } -// GetChannelsByIds provides a mock function with given fields: channelIds -func (_m *ChannelStore) GetChannelsByIds(channelIds []string) ([]*model.Channel, *model.AppError) { - ret := _m.Called(channelIds) +// GetChannelsByIds provides a mock function with given fields: channelIds, includeDeleted +func (_m *ChannelStore) GetChannelsByIds(channelIds []string, includeDeleted bool) ([]*model.Channel, *model.AppError) { + ret := _m.Called(channelIds, includeDeleted) var r0 []*model.Channel - if rf, ok := ret.Get(0).(func([]string) []*model.Channel); ok { - r0 = rf(channelIds) + if rf, ok := ret.Get(0).(func([]string, bool) []*model.Channel); ok { + r0 = rf(channelIds, includeDeleted) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*model.Channel) @@ -633,8 +633,8 @@ func (_m *ChannelStore) GetChannelsByIds(channelIds []string) ([]*model.Channel, } var r1 *model.AppError - if rf, ok := ret.Get(1).(func([]string) *model.AppError); ok { - r1 = rf(channelIds) + if rf, ok := ret.Get(1).(func([]string, bool) *model.AppError); ok { + r1 = rf(channelIds, includeDeleted) } else { if ret.Get(1) != nil { r1 = ret.Get(1).(*model.AppError) diff --git a/store/storetest/mocks/PostStore.go b/store/storetest/mocks/PostStore.go index d8442ab6fa..2a6a06e8a7 100644 --- a/store/storetest/mocks/PostStore.go +++ b/store/storetest/mocks/PostStore.go @@ -792,6 +792,31 @@ func (_m *PostStore) Search(teamId string, userId string, params *model.SearchPa return r0, r1 } +// SearchPostsInTeamForUser provides a mock function with given fields: paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage +func (_m *PostStore) SearchPostsInTeamForUser(paramsList []*model.SearchParams, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, page int, perPage int) (*model.PostSearchResults, *model.AppError) { + ret := _m.Called(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage) + + var r0 *model.PostSearchResults + if rf, ok := ret.Get(0).(func([]*model.SearchParams, string, string, bool, bool, int, int) *model.PostSearchResults); ok { + r0 = rf(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.PostSearchResults) + } + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func([]*model.SearchParams, string, string, bool, bool, int, int) *model.AppError); ok { + r1 = rf(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} + // Update provides a mock function with given fields: newPost, oldPost func (_m *PostStore) Update(newPost *model.Post, oldPost *model.Post) (*model.Post, *model.AppError) { ret := _m.Called(newPost, oldPost) diff --git a/store/storetest/mocks/UserStore.go b/store/storetest/mocks/UserStore.go index 3c3cd98a25..5e3268f8d0 100644 --- a/store/storetest/mocks/UserStore.go +++ b/store/storetest/mocks/UserStore.go @@ -84,6 +84,31 @@ func (_m *UserStore) AnalyticsGetSystemAdminCount() (int64, *model.AppError) { return r0, r1 } +// AutocompleteUsersInChannel provides a mock function with given fields: teamId, channelId, term, options +func (_m *UserStore) AutocompleteUsersInChannel(teamId string, channelId string, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) { + ret := _m.Called(teamId, channelId, term, options) + + var r0 *model.UserAutocompleteInChannel + if rf, ok := ret.Get(0).(func(string, string, string, *model.UserSearchOptions) *model.UserAutocompleteInChannel); ok { + r0 = rf(teamId, channelId, term, options) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.UserAutocompleteInChannel) + } + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func(string, string, string, *model.UserSearchOptions) *model.AppError); ok { + r1 = rf(teamId, channelId, term, options) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} + // ClearAllCustomRoleAssignments provides a mock function with given fields: func (_m *UserStore) ClearAllCustomRoleAssignments() *model.AppError { ret := _m.Called() diff --git a/store/timer_layer.go b/store/timer_layer.go index cda63f05c0..817cc81421 100644 --- a/store/timer_layer.go +++ b/store/timer_layer.go @@ -872,10 +872,10 @@ func (s *TimerLayerChannelStore) GetChannelsBatchForIndexing(startTime int64, en return resultVar0, resultVar1 } -func (s *TimerLayerChannelStore) GetChannelsByIds(channelIds []string) ([]*model.Channel, *model.AppError) { +func (s *TimerLayerChannelStore) GetChannelsByIds(channelIds []string, includeDeleted bool) ([]*model.Channel, *model.AppError) { start := timemodule.Now() - resultVar0, resultVar1 := s.ChannelStore.GetChannelsByIds(channelIds) + resultVar0, resultVar1 := s.ChannelStore.GetChannelsByIds(channelIds, includeDeleted) elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second) if s.Root.Metrics != nil { @@ -4444,6 +4444,22 @@ func (s *TimerLayerPostStore) Search(teamId string, userId string, params *model return resultVar0, resultVar1 } +func (s *TimerLayerPostStore) SearchPostsInTeamForUser(paramsList []*model.SearchParams, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, page int, perPage int) (*model.PostSearchResults, *model.AppError) { + start := timemodule.Now() + + resultVar0, resultVar1 := s.PostStore.SearchPostsInTeamForUser(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage) + + elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second) + if s.Root.Metrics != nil { + success := "false" + if resultVar1 == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("PostStore.SearchPostsInTeamForUser", success, elapsed) + } + return resultVar0, resultVar1 +} + func (s *TimerLayerPostStore) Update(newPost *model.Post, oldPost *model.Post) (*model.Post, *model.AppError) { start := timemodule.Now() @@ -6264,6 +6280,22 @@ func (s *TimerLayerUserStore) AnalyticsGetSystemAdminCount() (int64, *model.AppE return resultVar0, resultVar1 } +func (s *TimerLayerUserStore) AutocompleteUsersInChannel(teamId string, channelId string, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) { + start := timemodule.Now() + + resultVar0, resultVar1 := s.UserStore.AutocompleteUsersInChannel(teamId, channelId, term, options) + + elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second) + if s.Root.Metrics != nil { + success := "false" + if resultVar1 == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("UserStore.AutocompleteUsersInChannel", success, elapsed) + } + return resultVar0, resultVar1 +} + func (s *TimerLayerUserStore) ClearAllCustomRoleAssignments() *model.AppError { start := timemodule.Now() diff --git a/testlib/helper.go b/testlib/helper.go index 715e63baf9..0ac9e847ef 100644 --- a/testlib/helper.go +++ b/testlib/helper.go @@ -12,7 +12,9 @@ import ( "github.com/mattermost/mattermost-server/v5/mlog" "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/services/searchengine" "github.com/mattermost/mattermost-server/v5/store" + "github.com/mattermost/mattermost-server/v5/store/searchlayer" "github.com/mattermost/mattermost-server/v5/store/sqlstore" "github.com/mattermost/mattermost-server/v5/store/storetest" "github.com/mattermost/mattermost-server/v5/utils" @@ -21,6 +23,7 @@ import ( type MainHelper struct { Settings *model.SqlSettings Store store.Store + SearchEngine *searchengine.Broker SQLSupplier *sqlstore.SqlSupplier ClusterInterface *FakeClusterInterface @@ -100,11 +103,15 @@ func (h *MainHelper) setupStore() { h.Settings = storetest.MakeSqlSettings(driverName) + config := &model.Config{} + config.SetDefaults() + + h.SearchEngine = searchengine.NewBroker(config, nil) h.ClusterInterface = &FakeClusterInterface{} h.SQLSupplier = sqlstore.NewSqlSupplier(*h.Settings, nil) - h.Store = &TestStore{ + h.Store = searchlayer.NewSearchLayer(&TestStore{ h.SQLSupplier, - } + }, h.SearchEngine) } func (h *MainHelper) setupResources() {