Adding the new search engine abstraction (#13304)

* WIP

* Adding bleve to go modules

* WIP

* Adding missing files from searchengine implementation

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* User and channel indexing and searches implemented

* Make bleve tests run with in-memory indexes

* Implement post index and deletion tests

* Initial commits for the search layer

* Removing unnecesary indexing

* WIP

* WIP

* More fixes for tests

* Adding the search layer

* Finishing the migration of searchers to the layer

* Removing unnecesary code

* Allowing multiple engines active at the same time

* WIP

* Add simple post search

* Print information when using bleve

* Adding some debugging to understand better how the searches are working

* Making more dynamic config of search engines

* Add post search basics

* Adding the Purge API endpoint

* Fixing bleve config updates

* Adding missed file

* Regenerating search engine mocks

* Adding missed v5 to modules imports

* fixing i18n

* Fixing some test around search engine

* Removing all bleve traces

* Cleaning up the vendors directory and go.mod/go.sum files

* Regenerating timer layer

* Adding properly the license

* Fixing govet shadow error

* Fixing some tests

* Fixing TestSearchPostsFromUser

* Fixing another test

* Fixing more tests

* Fixing more tests

* Removing SearchEngine redundant text from searchengine module code

* Fixing some reindexing problems in members updates

* Fixing tests

* Addressing PR comments

* Reverting go.mod and go.sum

* Addressing PR comments

* Fixing tests compilation

* Fixing govet

* Adding search engine stop method

* Being more explicit on where we use includeDeleted

* Adding GetSqlSupplier test helper method

* Mocking elasticsearch start function

* Fixing tests

Co-authored-by: Miguel de la Cruz <miguel@mcrx.me>
Co-authored-by: mattermod <mattermod@users.noreply.github.com>

Makefile

@@ -201,8 +201,8 @@ endif
 
 app-layers: ## Extract interface from App struct
     # The following commented commands can be used to re-generate the AppIface from the App struct
-	# env GO111MODULE=off $(GO) get gopkg.in/reflog/struct2interface.v0
+	#env GO111MODULE=off $(GO) get gopkg.in/reflog/struct2interface.v0
-	# $(GOBIN)/struct2interface.v0 -f "app" -o "app/app_iface.go" -p "app" -s "App" -i "AppIface" -t ./app/layer_generators/app_iface.go.tmpl
+	#$(GOBIN)/struct2interface.v0 -f "app" -o "app/app_iface.go" -p "app" -s "App" -i "AppIface" -t ./app/layer_generators/app_iface.go.tmpl
 	$(GO) run ./app/layer_generators -in ./app/app_iface.go -out ./app/opentracing_layer.go -template ./app/layer_generators/opentracing_layer.go.tmpl
 
 i18n-extract: ## Extract strings for translation from the source code
@@ -234,6 +234,10 @@ einterfaces-mocks: ## Creates mock files for einterfaces.
 	env GO111MODULE=off $(GO) get -u github.com/vektra/mockery/...
 	$(GOBIN)/mockery -dir einterfaces -all -output einterfaces/mocks -note 'Regenerate this file using `make einterfaces-mocks`.'
 
+searchengine-mocks: ## Creates mock files for searchengines.
+	env GO111MODULE=off go get -u github.com/vektra/mockery/...
+	$(GOPATH)/bin/mockery -dir services/searchengine -all -output services/searchengine/mocks -note 'Regenerate this file using `make searchengine-mocks`.'
+
 pluginapi: ## Generates api and hooks glue code for plugins
 	$(GO) generate $(GOFLAGS) ./plugin
 

app/app.go

@@ -16,6 +16,7 @@ import (
 	"github.com/mattermost/mattermost-server/v5/model"
 	"github.com/mattermost/mattermost-server/v5/services/httpservice"
 	"github.com/mattermost/mattermost-server/v5/services/imageproxy"
+	"github.com/mattermost/mattermost-server/v5/services/searchengine"
 	"github.com/mattermost/mattermost-server/v5/services/timezones"
 	"github.com/mattermost/mattermost-server/v5/utils"
 )
@@ -38,7 +39,7 @@ type App struct {
 	cluster          einterfaces.ClusterInterface
 	compliance       einterfaces.ComplianceInterface
 	dataRetention    einterfaces.DataRetentionInterface
-	elasticsearch    einterfaces.ElasticsearchInterface
+	searchEngine     *searchengine.Broker
 	ldap             einterfaces.LdapInterface
 	messageExport    einterfaces.MessageExportInterface
 	metrics          einterfaces.MetricsInterface
@@ -202,8 +203,8 @@ func (a *App) Compliance() einterfaces.ComplianceInterface {
 func (a *App) DataRetention() einterfaces.DataRetentionInterface {
 	return a.dataRetention
 }
-func (a *App) Elasticsearch() einterfaces.ElasticsearchInterface {
+func (a *App) SearchEngine() *searchengine.Broker {
-	return a.elasticsearch
+	return a.searchEngine
 }
 func (a *App) Ldap() einterfaces.LdapInterface {
 	return a.ldap

app/app_iface.go

@@ -29,16 +29,13 @@ import (
 	"github.com/mattermost/mattermost-server/v5/services/filesstore"
 	"github.com/mattermost/mattermost-server/v5/services/httpservice"
 	"github.com/mattermost/mattermost-server/v5/services/imageproxy"
+	"github.com/mattermost/mattermost-server/v5/services/searchengine"
 	"github.com/mattermost/mattermost-server/v5/services/timezones"
 	"github.com/mattermost/mattermost-server/v5/store"
 )
 
 // AppIface is extracted from App struct and contains all it's exported methods. It's provided to allow partial interface passing and app layers creation.
 type AppIface interface {
-	// GetViewUsersRestrictionsForTeam returns a list with the channel ids that the user has permissions to view on a
-	// team. If the result is an empty list, the user can't view any channel; if it's
-	// nil, there are no restrictions for the user in the specified team.
-	GetViewUsersRestrictionsForTeam(userId string, teamId string) ([]string, *model.AppError)
 	// @openTracingParams teamId
 	// previous ListCommands now ListAutocompleteCommands
 	ListAutocompleteCommands(teamId string, T goi18n.TranslateFunc) ([]*model.Command, *model.AppError)
@@ -413,7 +410,6 @@ type AppIface interface {
 	DoUploadFile(now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError)
 	DoUploadFileExpectModification(now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte) (*model.FileInfo, []byte, *model.AppError)
 	DownloadFromURL(downloadURL string) ([]byte, error)
-	Elasticsearch() einterfaces.ElasticsearchInterface
 	EnableUserAccessToken(token *model.UserAccessToken) *model.AppError
 	EnsureDiagnosticId()
 	EnvironmentConfig() map[string]interface{}
@@ -677,9 +673,6 @@ type AppIface interface {
 	InviteNewUsersToTeam(emailList []string, teamId, senderId string) *model.AppError
 	InviteNewUsersToTeamGracefully(emailList []string, teamId, senderId string) ([]*model.EmailInviteWithError, *model.AppError)
 	IpAddress() string
-	IsESAutocompletionEnabled() bool
-	IsESIndexingEnabled() bool
-	IsESSearchEnabled() bool
 	IsFirstUserAccount() bool
 	IsLeader() bool
 	IsPasswordValid(password string) *model.AppError
@@ -795,6 +788,7 @@ type AppIface interface {
 	SearchChannelsForUser(userId, teamId, term string) (*model.ChannelList, *model.AppError)
 	SearchChannelsUserNotIn(teamId string, userId string, term string) (*model.ChannelList, *model.AppError)
 	SearchEmoji(name string, prefixOnly bool, limit int) ([]*model.Emoji, *model.AppError)
+	SearchEngine() *searchengine.Broker
 	SearchGroupChannels(userId, term string) (*model.ChannelList, *model.AppError)
 	SearchPostsInTeam(teamId string, paramsList []*model.SearchParams) (*model.PostList, *model.AppError)
 	SearchPostsInTeamForUser(terms string, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, timeZoneOffset int, page, perPage int) (*model.PostSearchResults, *model.AppError)

app/channel.go

@@ -112,14 +112,6 @@ func (a *App) JoinDefaultChannels(teamId string, user *model.User, shouldBeAdmin
 
 	}
 
-	if a.IsESIndexingEnabled() {
-		a.Srv().Go(func() {
-			if err = a.indexUser(user); err != nil {
-				mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
-			}
-		})
-	}
-
 	return err
 }
 
@@ -189,14 +181,6 @@ func (a *App) CreateChannelWithUser(channel *model.Channel, userId string) (*mod
 	message.Add("team_id", channel.TeamId)
 	a.Publish(message)
 
-	if a.IsESIndexingEnabled() {
-		a.Srv().Go(func() {
-			if err := a.indexUser(user); err != nil {
-				mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
-			}
-		})
-	}
-
 	return rchannel, nil
 }
 
@@ -267,23 +251,6 @@ func (a *App) CreateChannel(channel *model.Channel, addMember bool) (*model.Chan
 		})
 	}
 
-	if a.IsESIndexingEnabled() {
-		if sc.Type == model.CHANNEL_OPEN {
-			a.Srv().Go(func() {
-				if err := a.Elasticsearch().IndexChannel(sc); err != nil {
-					mlog.Error("Encountered error indexing channel", mlog.String("channel_id", sc.Id), mlog.Err(err))
-				}
-			})
-		}
-		if addMember {
-			a.Srv().Go(func() {
-				if err := a.indexUserFromId(channel.CreatorId); err != nil {
-					mlog.Error("Encountered error indexing user", mlog.String("user_id", channel.CreatorId), mlog.Err(err))
-				}
-			})
-		}
-	}
-
 	return sc, nil
 }
 
@@ -314,16 +281,6 @@ func (a *App) GetOrCreateDirectChannel(userId, otherUserId string) (*model.Chann
 				})
 			}
 
-			if a.IsESIndexingEnabled() {
-				a.Srv().Go(func() {
-					for _, id := range []string{userId, otherUserId} {
-						if indexUserErr := a.indexUserFromId(id); indexUserErr != nil {
-							mlog.Error("Encountered error indexing user", mlog.String("user_id", id), mlog.Err(indexUserErr))
-						}
-					}
-				})
-			}
-
 			message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_DIRECT_ADDED, "", channel.Id, "", nil)
 			message.Add("teammate_id", otherUserId)
 			a.Publish(message)
@@ -431,16 +388,6 @@ func (a *App) CreateGroupChannel(userIds []string, creatorId string) (*model.Cha
 	message.Add("teammate_ids", model.ArrayToJson(userIds))
 	a.Publish(message)
 
-	if a.IsESIndexingEnabled() {
-		a.Srv().Go(func() {
-			for _, id := range userIds {
-				if err := a.indexUserFromId(id); err != nil {
-					mlog.Error("Encountered error indexing user", mlog.String("user_id", id), mlog.Err(err))
-				}
-			}
-		})
-	}
-
 	return channel, nil
 }
 
@@ -536,14 +483,6 @@ func (a *App) UpdateChannel(channel *model.Channel) (*model.Channel, *model.AppE
 	messageWs.Add("channel", channel.ToJson())
 	a.Publish(messageWs)
 
-	if a.IsESIndexingEnabled() && channel.Type == model.CHANNEL_OPEN {
-		a.Srv().Go(func() {
-			if err := a.Elasticsearch().IndexChannel(channel); err != nil {
-				mlog.Error("Encountered error indexing channel", mlog.String("channel_id", channel.Id), mlog.Err(err))
-			}
-		})
-	}
-
 	return channel, nil
 }
 
@@ -1243,14 +1182,6 @@ func (a *App) AddChannelMember(userId string, channel *model.Channel, userReques
 		})
 	}
 
-	if a.IsESIndexingEnabled() {
-		a.Srv().Go(func() {
-			if err := a.indexUser(user); err != nil {
-				mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
-			}
-		})
-	}
-
 	if userRequestorId == "" || userId == userRequestorId {
 		a.postJoinChannelMessage(user, channel)
 	} else {
@@ -1639,14 +1570,6 @@ func (a *App) JoinChannel(channel *model.Channel, userId string) *model.AppError
 		})
 	}
 
-	if a.IsESIndexingEnabled() {
-		a.Srv().Go(func() {
-			if err := a.indexUser(user); err != nil {
-				mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
-			}
-		})
-	}
-
 	if err := a.postJoinChannelMessage(user, channel); err != nil {
 		return err
 	}
@@ -1927,14 +1850,6 @@ func (a *App) removeUserFromChannel(userIdToRemove string, removerUserId string,
 		})
 	}
 
-	if a.IsESIndexingEnabled() {
-		a.Srv().Go(func() {
-			if err := a.indexUserFromId(userIdToRemove); err != nil {
-				mlog.Error("Encountered error indexing user", mlog.String("user_id", userIdToRemove), mlog.Err(err))
-			}
-		})
-	}
-
 	message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_USER_REMOVED, "", channel.Id, "", nil)
 	message.Add("user_id", userIdToRemove)
 	message.Add("remover_id", removerUserId)
@@ -2056,50 +1971,11 @@ func (a *App) MarkChannelAsUnreadFromPost(postID string, userID string) (*model.
 	return channelUnread, nil
 }
 
-func (a *App) esAutocompleteChannels(teamId, term string, includeDeleted bool) (*model.ChannelList, *model.AppError) {
-	channelIds, err := a.Elasticsearch().SearchChannels(teamId, term)
-	if err != nil {
-		return nil, err
-	}
-
-	channelList := model.ChannelList{}
-	if len(channelIds) > 0 {
-		channels, err := a.Srv().Store.Channel().GetChannelsByIds(channelIds)
-		if err != nil {
-			return nil, err
-		}
-		for _, c := range channels {
-			if c.DeleteAt > 0 && !includeDeleted {
-				continue
-			}
-			channelList = append(channelList, c)
-		}
-	}
-
-	return &channelList, nil
-}
-
 func (a *App) AutocompleteChannels(teamId string, term string) (*model.ChannelList, *model.AppError) {
 	includeDeleted := *a.Config().TeamSettings.ExperimentalViewArchivedChannels
-	var channelList *model.ChannelList
-	var err *model.AppError
 	term = strings.TrimSpace(term)
 
-	if a.IsESAutocompletionEnabled() {
+	return a.Srv().Store.Channel().AutocompleteInTeam(teamId, term, includeDeleted)
-		channelList, err = a.esAutocompleteChannels(teamId, term, includeDeleted)
-		if err != nil {
-			mlog.Error("Encountered error on AutocompleteChannels through Elasticsearch. Falling back to default autocompletion.", mlog.Err(err))
-		}
-	}
-
-	if !a.IsESAutocompletionEnabled() || err != nil {
-		channelList, err = a.Srv().Store.Channel().AutocompleteInTeam(teamId, term, includeDeleted)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return channelList, nil
 }
 
 func (a *App) AutocompleteChannelsForSearch(teamId string, userId string, term string) (*model.ChannelList, *model.AppError) {
@@ -2246,11 +2122,6 @@ func (a *App) ViewChannel(view *model.ChannelView, userId string, currentSession
 }
 
 func (a *App) PermanentDeleteChannel(channel *model.Channel) *model.AppError {
-	profiles, err := a.Srv().Store.User().GetAllProfilesInChannel(channel.Id, false)
-	if err != nil {
-		return err
-	}
-
 	if err := a.Srv().Store.Post().PermanentDeleteByChannel(channel.Id); err != nil {
 		return err
 	}
@@ -2271,23 +2142,6 @@ func (a *App) PermanentDeleteChannel(channel *model.Channel) *model.AppError {
 		return err
 	}
 
-	if a.IsESIndexingEnabled() {
-		a.Srv().Go(func() {
-			for _, user := range profiles {
-				if err := a.indexUser(user); err != nil {
-					mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
-				}
-			}
-		})
-		if channel.Type == model.CHANNEL_OPEN {
-			a.Srv().Go(func() {
-				if err := a.Elasticsearch().DeleteChannel(channel); err != nil {
-					mlog.Error("Encountered error deleting channel", mlog.String("channel_id", channel.Id), mlog.Err(err))
-				}
-			})
-		}
-	}
-
 	return nil
 }
 

app/config.go

@@ -406,22 +406,6 @@ func (a *App) SaveConfig(newCfg *model.Config, sendConfigChangeClusterMessage bo
 	return nil
 }
 
-func (a *App) IsESIndexingEnabled() bool {
-	return a.Elasticsearch() != nil && *a.Config().ElasticsearchSettings.EnableIndexing
-}
-
-func (a *App) IsESSearchEnabled() bool {
-	esInterface := a.Elasticsearch()
-	license := a.License()
-	return esInterface != nil && *a.Config().ElasticsearchSettings.EnableSearching && license != nil && *license.Features.Elasticsearch
-}
-
-func (a *App) IsESAutocompletionEnabled() bool {
-	esInterface := a.Elasticsearch()
-	license := a.License()
-	return esInterface != nil && *a.Config().ElasticsearchSettings.EnableAutocomplete && license != nil && *license.Features.Elasticsearch
-}
-
 func (a *App) HandleMessageExportConfig(cfg *model.Config, appCfg *model.Config) {
 	// If the Message Export feature has been toggled in the System Console, rewrite the ExportFromTimestamp field to an
 	// appropriate value. The rewriting occurs here to ensure it doesn't affect values written to the config file

app/config_test.go

@@ -12,8 +12,6 @@ import (
 	"github.com/stretchr/testify/mock"
 
 	"github.com/mattermost/mattermost-server/v5/model"
-	"github.com/mattermost/mattermost-server/v5/store/localcachelayer"
-	"github.com/mattermost/mattermost-server/v5/store/sqlstore"
 	"github.com/mattermost/mattermost-server/v5/store/storetest/mocks"
 	"github.com/mattermost/mattermost-server/v5/utils"
 )
@@ -126,7 +124,7 @@ func TestEnsureInstallationDate(t *testing.T) {
 
 	for _, tc := range tt {
 		t.Run(tc.Name, func(t *testing.T) {
-			sqlStore := th.App.Srv().Store.User().(localcachelayer.LocalCacheUserStore).UserStore.(*sqlstore.SqlUserStore)
+			sqlStore := th.GetSqlSupplier()
 			sqlStore.GetMaster().Exec("DELETE FROM Users")
 
 			for _, createAt := range tc.UsersCreationDates {

app/diagnostics.go

@@ -914,8 +914,10 @@ func (a *App) trackPermissions() {
 func (a *App) trackElasticsearch() {
 	data := map[string]interface{}{}
 
-	if a.Elasticsearch() != nil && a.Elasticsearch().GetVersion() != 0 {
+	for _, engine := range a.SearchEngine().GetActiveEngines() {
-		data["elasticsearch_server_version"] = a.Elasticsearch().GetVersion()
+		if engine.GetVersion() != 0 && engine.GetName() == "elasticsearch" {
+			data["elasticsearch_server_version"] = engine.GetVersion()
+		}
 	}
 
 	a.SendDiagnostic(TRACK_ELASTICSEARCH, data)

app/enterprise.go

@@ -9,6 +9,7 @@ import (
 	tjobs "github.com/mattermost/mattermost-server/v5/jobs/interfaces"
 	"github.com/mattermost/mattermost-server/v5/mlog"
 	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/services/searchengine"
 )
 
 var accountMigrationInterface func(*Server) einterfaces.AccountMigrationInterface
@@ -35,9 +36,9 @@ func RegisterDataRetentionInterface(f func(*App) einterfaces.DataRetentionInterf
 	dataRetentionInterface = f
 }
 
-var elasticsearchInterface func(*App) einterfaces.ElasticsearchInterface
+var elasticsearchInterface func(*App) searchengine.SearchEngineInterface
 
-func RegisterElasticsearchInterface(f func(*App) einterfaces.ElasticsearchInterface) {
+func RegisterElasticsearchInterface(f func(*App) searchengine.SearchEngineInterface) {
 	elasticsearchInterface = f
 }
 
@@ -129,9 +130,6 @@ func (s *Server) initEnterprise() {
 	if complianceInterface != nil {
 		s.Compliance = complianceInterface(s.FakeApp())
 	}
-	if elasticsearchInterface != nil {
-		s.Elasticsearch = elasticsearchInterface(s.FakeApp())
-	}
 	if ldapInterface != nil {
 		s.Ldap = ldapInterface(s.FakeApp())
 	}
@@ -161,4 +159,8 @@ func (s *Server) initEnterprise() {
 	if clusterInterface != nil {
 		s.Cluster = clusterInterface(s)
 	}
+
+	if elasticsearchInterface != nil {
+		s.SearchEngine.RegisterElasticsearchEngine(elasticsearchInterface(s.FakeApp()))
+	}
 }

app/helper_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/mattermost/mattermost-server/v5/model"
 	"github.com/mattermost/mattermost-server/v5/store"
 	"github.com/mattermost/mattermost-server/v5/store/localcachelayer"
+	"github.com/mattermost/mattermost-server/v5/store/sqlstore"
 	"github.com/mattermost/mattermost-server/v5/store/storetest/mocks"
 	"github.com/mattermost/mattermost-server/v5/testlib"
 	"github.com/mattermost/mattermost-server/v5/utils"
@@ -94,6 +95,11 @@ func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer boo
 	}
 
 	th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = prevListenAddress })
+
+	th.App.Srv().SearchEngine = mainHelper.SearchEngine
+
+	th.App.Srv().Store.MarkSystemRanUnitTests()
+
 	th.App.UpdateConfig(func(cfg *model.Config) { *cfg.TeamSettings.EnableOpenServer = true })
 
 	// Disable strict password requirements for test
@@ -569,6 +575,10 @@ func (me *TestHelper) TearDown() {
 	}
 }
 
+func (me *TestHelper) GetSqlSupplier() *sqlstore.SqlSupplier {
+	return mainHelper.GetSQLSupplier()
+}
+
 func (me *TestHelper) ResetRoleMigration() {
 	sqlSupplier := mainHelper.GetSQLSupplier()
 	if _, err := sqlSupplier.GetMaster().Exec("DELETE from Roles"); err != nil {

app/layer_generators/opentracing_layer.go.tmpl

@@ -31,7 +31,7 @@ type {{.Name}} struct {
 	cluster          einterfaces.ClusterInterface
 	compliance       einterfaces.ComplianceInterface
 	dataRetention    einterfaces.DataRetentionInterface
-	elasticsearch    einterfaces.ElasticsearchInterface
+	searchEngine     *searchengine.Broker
 	ldap             einterfaces.LdapInterface
 	messageExport    einterfaces.MessageExportInterface
 	metrics          einterfaces.MetricsInterface
@@ -99,7 +99,7 @@ func NewOpenTracingAppLayer(childApp AppIface, ctx context.Context) *{{.Name}} {
 	newApp.cluster = childApp.Cluster()
 	newApp.compliance = childApp.Compliance()
 	newApp.dataRetention = childApp.DataRetention()
-	newApp.elasticsearch = childApp.Elasticsearch()
+	newApp.searchEngine = childApp.SearchEngine()
 	newApp.ldap = childApp.Ldap()
 	newApp.messageExport = childApp.MessageExport()
 	newApp.metrics = childApp.Metrics()
@@ -156,9 +156,6 @@ func (a *{{.Name}}) Compliance() einterfaces.ComplianceInterface {
 func (a *{{.Name}}) DataRetention() einterfaces.DataRetentionInterface {
 	return a.dataRetention
 }
-func (a *{{.Name}}) Elasticsearch() einterfaces.ElasticsearchInterface {
-	return a.elasticsearch
-}
 func (a *{{.Name}}) Ldap() einterfaces.LdapInterface {
 	return a.ldap
 }
@@ -215,4 +212,4 @@ func (a *{{.Name}}) SetServer(srv *Server) {
 }
 func (a *{{.Name}}) GetT() goi18n.TranslateFunc {
 	return a.t
-}
+}

app/opentracing_layer.go

@@ -29,6 +29,7 @@ import (
 	"github.com/mattermost/mattermost-server/v5/services/filesstore"
 	"github.com/mattermost/mattermost-server/v5/services/httpservice"
 	"github.com/mattermost/mattermost-server/v5/services/imageproxy"
+	"github.com/mattermost/mattermost-server/v5/services/searchengine"
 	"github.com/mattermost/mattermost-server/v5/services/timezones"
 	"github.com/mattermost/mattermost-server/v5/services/tracing"
 	"github.com/mattermost/mattermost-server/v5/store"
@@ -56,7 +57,7 @@ type OpenTracingAppLayer struct {
 	cluster          einterfaces.ClusterInterface
 	compliance       einterfaces.ComplianceInterface
 	dataRetention    einterfaces.DataRetentionInterface
-	elasticsearch    einterfaces.ElasticsearchInterface
+	searchEngine     *searchengine.Broker
 	ldap             einterfaces.LdapInterface
 	messageExport    einterfaces.MessageExportInterface
 	metrics          einterfaces.MetricsInterface
@@ -8588,28 +8589,6 @@ func (a *OpenTracingAppLayer) GetViewUsersRestrictions(userId string) (*model.Vi
 	return resultVar0, resultVar1
 }
 
-func (a *OpenTracingAppLayer) GetViewUsersRestrictionsForTeam(userId string, teamId string) ([]string, *model.AppError) {
-	origCtx := a.ctx
-	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetViewUsersRestrictionsForTeam")
-
-	a.ctx = newCtx
-	a.app.Srv().Store.SetContext(newCtx)
-	defer func() {
-		a.app.Srv().Store.SetContext(origCtx)
-		a.ctx = origCtx
-	}()
-
-	defer span.Finish()
-	resultVar0, resultVar1 := a.app.GetViewUsersRestrictionsForTeam(userId, teamId)
-
-	if resultVar1 != nil {
-		span.LogFields(spanlog.Error(resultVar1))
-		ext.Error.Set(span, true)
-	}
-
-	return resultVar0, resultVar1
-}
-
 func (a *OpenTracingAppLayer) HTMLTemplates() *template.Template {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.HTMLTemplates")
@@ -9229,57 +9208,6 @@ func (a *OpenTracingAppLayer) InviteNewUsersToTeamGracefully(emailList []string,
 	return resultVar0, resultVar1
 }
 
-func (a *OpenTracingAppLayer) IsESAutocompletionEnabled() bool {
-	origCtx := a.ctx
-	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.IsESAutocompletionEnabled")
-
-	a.ctx = newCtx
-	a.app.Srv().Store.SetContext(newCtx)
-	defer func() {
-		a.app.Srv().Store.SetContext(origCtx)
-		a.ctx = origCtx
-	}()
-
-	defer span.Finish()
-	resultVar0 := a.app.IsESAutocompletionEnabled()
-
-	return resultVar0
-}
-
-func (a *OpenTracingAppLayer) IsESIndexingEnabled() bool {
-	origCtx := a.ctx
-	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.IsESIndexingEnabled")
-
-	a.ctx = newCtx
-	a.app.Srv().Store.SetContext(newCtx)
-	defer func() {
-		a.app.Srv().Store.SetContext(origCtx)
-		a.ctx = origCtx
-	}()
-
-	defer span.Finish()
-	resultVar0 := a.app.IsESIndexingEnabled()
-
-	return resultVar0
-}
-
-func (a *OpenTracingAppLayer) IsESSearchEnabled() bool {
-	origCtx := a.ctx
-	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.IsESSearchEnabled")
-
-	a.ctx = newCtx
-	a.app.Srv().Store.SetContext(newCtx)
-	defer func() {
-		a.app.Srv().Store.SetContext(origCtx)
-		a.ctx = origCtx
-	}()
-
-	defer span.Finish()
-	resultVar0 := a.app.IsESSearchEnabled()
-
-	return resultVar0
-}
-
 func (a *OpenTracingAppLayer) IsFirstUserAccount() bool {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.IsFirstUserAccount")
@@ -11790,6 +11718,23 @@ func (a *OpenTracingAppLayer) SearchEmoji(name string, prefixOnly bool, limit in
 	return resultVar0, resultVar1
 }
 
+func (a *OpenTracingAppLayer) SearchEngine() *searchengine.Broker {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SearchEngine")
+
+	a.ctx = newCtx
+	a.app.Srv().Store.SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store.SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0 := a.app.SearchEngine()
+
+	return resultVar0
+}
+
 func (a *OpenTracingAppLayer) SearchGroupChannels(userId string, term string) (*model.ChannelList, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SearchGroupChannels")
@@ -14914,7 +14859,7 @@ func NewOpenTracingAppLayer(childApp AppIface, ctx context.Context) *OpenTracing
 	newApp.cluster = childApp.Cluster()
 	newApp.compliance = childApp.Compliance()
 	newApp.dataRetention = childApp.DataRetention()
-	newApp.elasticsearch = childApp.Elasticsearch()
+	newApp.searchEngine = childApp.SearchEngine()
 	newApp.ldap = childApp.Ldap()
 	newApp.messageExport = childApp.MessageExport()
 	newApp.metrics = childApp.Metrics()
@@ -14970,9 +14915,6 @@ func (a *OpenTracingAppLayer) Compliance() einterfaces.ComplianceInterface {
 func (a *OpenTracingAppLayer) DataRetention() einterfaces.DataRetentionInterface {
 	return a.dataRetention
 }
-func (a *OpenTracingAppLayer) Elasticsearch() einterfaces.ElasticsearchInterface {
-	return a.elasticsearch
-}
 func (a *OpenTracingAppLayer) Ldap() einterfaces.LdapInterface {
 	return a.ldap
 }

app/options.go

@@ -78,8 +78,8 @@ func StartMetrics(s *Server) error {
 	return nil
 }
 
-func StartElasticsearch(s *Server) error {
+func StartSearchEngine(s *Server) error {
-	s.startElasticsearch = true
+	s.startSearchEngine = true
 
 	return nil
 }
@@ -104,7 +104,7 @@ func ServerConnector(s *Server) AppOption {
 		a.cluster = s.Cluster
 		a.compliance = s.Compliance
 		a.dataRetention = s.DataRetention
-		a.elasticsearch = s.Elasticsearch
+		a.searchEngine = s.SearchEngine
 		a.ldap = s.Ldap
 		a.messageExport = s.MessageExport
 		a.metrics = s.Metrics

app/post.go

@@ -297,14 +297,6 @@ func (a *App) CreatePost(post *model.Post, channel *model.Channel, triggerWebhoo
 		})
 	}
 
-	if a.IsESIndexingEnabled() {
-		a.Srv().Go(func() {
-			if err = a.Elasticsearch().IndexPost(rpost, channel.TeamId); err != nil {
-				mlog.Error("Encountered error indexing post", mlog.String("post_id", post.Id), mlog.Err(err))
-			}
-		})
-	}
-
 	if a.Metrics() != nil {
 		a.Metrics().IncrementPostCreate()
 	}
@@ -585,19 +577,6 @@ func (a *App) UpdatePost(post *model.Post, safeUpdate bool) (*model.Post, *model
 		})
 	}
 
-	if a.IsESIndexingEnabled() {
-		a.Srv().Go(func() {
-			channel, chanErr := a.Srv().Store.Channel().GetForPost(rpost.Id)
-			if chanErr != nil {
-				mlog.Error("Couldn't get channel for post for Elasticsearch indexing.", mlog.String("channel_id", rpost.ChannelId), mlog.String("post_id", rpost.Id))
-				return
-			}
-			if err := a.Elasticsearch().IndexPost(rpost, channel.TeamId); err != nil {
-				mlog.Error("Encountered error indexing post", mlog.String("post_id", post.Id), mlog.Err(err))
-			}
-		})
-	}
-
 	rpost = a.PreparePostForClient(rpost, false, true)
 
 	message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_EDITED, "", rpost.ChannelId, "", nil)
@@ -870,14 +849,6 @@ func (a *App) DeletePost(postId, deleteByID string) (*model.Post, *model.AppErro
 		a.DeleteFlaggedPosts(post.Id)
 	})
 
-	if a.IsESIndexingEnabled() {
-		a.Srv().Go(func() {
-			if err := a.Elasticsearch().DeletePost(post); err != nil {
-				mlog.Error("Encountered error deleting post", mlog.String("post_id", post.Id), mlog.Err(err))
-			}
-		})
-	}
-
 	a.invalidateCacheForChannelPosts(post.ChannelId)
 
 	return post, nil
@@ -1006,10 +977,18 @@ func (a *App) SearchPostsInTeam(teamId string, paramsList []*model.SearchParams)
 	})
 }
 
-func (a *App) esSearchPostsInTeamForUser(paramsList []*model.SearchParams, userId, teamId string, isOrSearch, includeDeletedChannels bool, page, perPage int) (*model.PostSearchResults, *model.AppError) {
+func (a *App) SearchPostsInTeamForUser(terms string, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, timeZoneOffset int, page, perPage int) (*model.PostSearchResults, *model.AppError) {
-	finalParamsList := []*model.SearchParams{}
+	var postSearchResults *model.PostSearchResults
+	var err *model.AppError
+	paramsList := model.ParseSearchParams(strings.TrimSpace(terms), timeZoneOffset)
 	includeDeleted := includeDeletedChannels && *a.Config().TeamSettings.ExperimentalViewArchivedChannels
 
+	if !*a.Config().ServiceSettings.EnablePostSearch {
+		return nil, model.NewAppError("SearchPostsInTeamForUser", "store.sql_post.search.disabled", nil, fmt.Sprintf("teamId=%v userId=%v", teamId, userId), http.StatusNotImplemented)
+	}
+
+	finalParamsList := []*model.SearchParams{}
+
 	for _, params := range paramsList {
 		params.OrTerms = isOrSearch
 		// Don't allow users to search for "*"
@@ -1031,90 +1010,11 @@ func (a *App) esSearchPostsInTeamForUser(paramsList []*model.SearchParams, userI
 		return model.MakePostSearchResults(model.NewPostList(), nil), nil
 	}
 
-	// We only allow the user to search in channels they are a member of.
+	postSearchResults, err = a.Srv().Store.Post().SearchPostsInTeamForUser(finalParamsList, userId, teamId, isOrSearch, includeDeleted, page, perPage)
-	userChannels, err := a.GetChannelsForUser(teamId, userId, includeDeleted)
-	if err != nil {
-		mlog.Error("error getting channel for user", mlog.Err(err))
-		return nil, err
-	}
-
-	postIds, matches, err := a.Elasticsearch().SearchPosts(userChannels, finalParamsList, page, perPage)
 	if err != nil {
 		return nil, err
 	}
 
-	// Get the posts
-	postList := model.NewPostList()
-	if len(postIds) > 0 {
-		posts, err := a.Srv().Store.Post().GetPostsByIds(postIds)
-		if err != nil {
-			return nil, err
-		}
-		for _, p := range posts {
-			if p.DeleteAt == 0 {
-				postList.AddPost(p)
-				postList.AddOrder(p.Id)
-			}
-		}
-	}
-
-	return model.MakePostSearchResults(postList, matches), nil
-}
-
-func (a *App) SearchPostsInTeamForUser(terms string, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, timeZoneOffset int, page, perPage int) (*model.PostSearchResults, *model.AppError) {
-	var postSearchResults *model.PostSearchResults
-	var err *model.AppError
-	paramsList := model.ParseSearchParams(strings.TrimSpace(terms), timeZoneOffset)
-
-	if !*a.Config().ServiceSettings.EnablePostSearch {
-		return nil, model.NewAppError("SearchPostsInTeamForUser", "store.sql_post.search.disabled", nil, fmt.Sprintf("teamId=%v userId=%v", teamId, userId), http.StatusNotImplemented)
-	}
-
-	if a.IsESSearchEnabled() {
-		postSearchResults, err = a.esSearchPostsInTeamForUser(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage)
-		if err != nil {
-			mlog.Error("Encountered error on SearchPostsInTeamForUser through Elasticsearch. Falling back to default search.", mlog.Err(err))
-		}
-	}
-
-	if !a.IsESSearchEnabled() || err != nil {
-		// Since we don't support paging for DB search, we just return nothing for later pages
-		if page > 0 {
-			return model.MakePostSearchResults(model.NewPostList(), nil), nil
-		}
-
-		includeDeleted := includeDeletedChannels && *a.Config().TeamSettings.ExperimentalViewArchivedChannels
-		posts, err := a.searchPostsInTeam(teamId, userId, paramsList, func(params *model.SearchParams) {
-			params.IncludeDeletedChannels = includeDeleted
-			params.OrTerms = isOrSearch
-			for idx, channelName := range params.InChannels {
-				if strings.HasPrefix(channelName, "@") {
-					channel, err := a.parseAndFetchChannelIdByNameFromInFilter(channelName, userId, teamId, includeDeletedChannels)
-					if err != nil {
-						mlog.Error("error getting channel_id by name from in filter", mlog.Err(err))
-						continue
-					}
-					params.InChannels[idx] = channel.Name
-				}
-			}
-			for idx, channelName := range params.ExcludedChannels {
-				if strings.HasPrefix(channelName, "@") {
-					channel, err := a.parseAndFetchChannelIdByNameFromInFilter(channelName, userId, teamId, includeDeletedChannels)
-					if err != nil {
-						mlog.Error("error getting channel_id by name from in filter", mlog.Err(err))
-						continue
-					}
-					params.ExcludedChannels[idx] = channel.Name
-				}
-			}
-		})
-		if err != nil {
-			return nil, err
-		}
-
-		postSearchResults = model.MakePostSearchResults(posts, nil)
-	}
-
 	return postSearchResults, nil
 }
 

app/post_test.go

@@ -13,9 +13,9 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/mattermost/mattermost-server/v5/einterfaces/mocks"
 	"github.com/mattermost/mattermost-server/v5/model"
 	"github.com/mattermost/mattermost-server/v5/plugin/plugintest/mock"
+	"github.com/mattermost/mattermost-server/v5/services/searchengine/mocks"
 	"github.com/mattermost/mattermost-server/v5/store/storetest"
 	storemocks "github.com/mattermost/mattermost-server/v5/store/storetest/mocks"
 )
@@ -944,9 +944,16 @@ func TestSearchPostsInTeamForUser(t *testing.T) {
 			posts[2].Id,
 		}
 
-		es := &mocks.ElasticsearchInterface{}
+		es := &mocks.SearchEngineInterface{}
 		es.On("SearchPosts", mock.Anything, mock.Anything, page, perPage).Return(resultsPage, nil, nil)
-		th.App.elasticsearch = es
+		es.On("GetName").Return("mock")
+		es.On("Start").Return(nil).Maybe()
+		es.On("IsActive").Return(true)
+		es.On("IsSearchEnabled").Return(true)
+		th.App.Srv().SearchEngine.ElasticsearchEngine = es
+		defer func() {
+			th.App.Srv().SearchEngine.ElasticsearchEngine = nil
+		}()
 
 		results, err := th.App.SearchPostsInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage)
 
@@ -965,9 +972,16 @@ func TestSearchPostsInTeamForUser(t *testing.T) {
 			posts[0].Id,
 		}
 
-		es := &mocks.ElasticsearchInterface{}
+		es := &mocks.SearchEngineInterface{}
 		es.On("SearchPosts", mock.Anything, mock.Anything, page, perPage).Return(resultsPage, nil, nil)
-		th.App.elasticsearch = es
+		es.On("GetName").Return("mock")
+		es.On("Start").Return(nil).Maybe()
+		es.On("IsActive").Return(true)
+		es.On("IsSearchEnabled").Return(true)
+		th.App.Srv().SearchEngine.ElasticsearchEngine = es
+		defer func() {
+			th.App.Srv().SearchEngine.ElasticsearchEngine = nil
+		}()
 
 		results, err := th.App.SearchPostsInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage)
 
@@ -982,9 +996,16 @@ func TestSearchPostsInTeamForUser(t *testing.T) {
 
 		page := 0
 
-		es := &mocks.ElasticsearchInterface{}
+		es := &mocks.SearchEngineInterface{}
 		es.On("SearchPosts", mock.Anything, mock.Anything, page, perPage).Return(nil, nil, &model.AppError{})
-		th.App.elasticsearch = es
+		es.On("GetName").Return("mock")
+		es.On("Start").Return(nil).Maybe()
+		es.On("IsActive").Return(true)
+		es.On("IsSearchEnabled").Return(true)
+		th.App.Srv().SearchEngine.ElasticsearchEngine = es
+		defer func() {
+			th.App.Srv().SearchEngine.ElasticsearchEngine = nil
+		}()
 
 		results, err := th.App.SearchPostsInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage)
 
@@ -1007,9 +1028,16 @@ func TestSearchPostsInTeamForUser(t *testing.T) {
 
 		page := 1
 
-		es := &mocks.ElasticsearchInterface{}
+		es := &mocks.SearchEngineInterface{}
 		es.On("SearchPosts", mock.Anything, mock.Anything, page, perPage).Return(nil, nil, &model.AppError{})
-		th.App.elasticsearch = es
+		es.On("GetName").Return("mock")
+		es.On("Start").Return(nil).Maybe()
+		es.On("IsActive").Return(true)
+		es.On("IsSearchEnabled").Return(true)
+		th.App.Srv().SearchEngine.ElasticsearchEngine = es
+		defer func() {
+			th.App.Srv().SearchEngine.ElasticsearchEngine = nil
+		}()
 
 		results, err := th.App.SearchPostsInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage)
 

app/searchengine.go

@@ -18,12 +18,12 @@ func (a *App) TestElasticsearch(cfg *model.Config) *model.AppError {
 		}
 	}
 
-	esI := a.Elasticsearch()
+	seI := a.SearchEngine().ElasticsearchEngine
-	if esI == nil {
+	if seI == nil {
 		err := model.NewAppError("TestElasticsearch", "ent.elasticsearch.test_config.license.error", nil, "", http.StatusNotImplemented)
 		return err
 	}
-	if err := esI.TestConfig(cfg); err != nil {
+	if err := seI.TestConfig(cfg); err != nil {
 		return err
 	}
 
@@ -31,13 +31,13 @@ func (a *App) TestElasticsearch(cfg *model.Config) *model.AppError {
 }
 
 func (a *App) PurgeElasticsearchIndexes() *model.AppError {
-	esI := a.Elasticsearch()
+	seI := a.SearchEngine().ElasticsearchEngine
-	if esI == nil {
+	if seI == nil {
 		err := model.NewAppError("PurgeElasticsearchIndexes", "ent.elasticsearch.test_config.license.error", nil, "", http.StatusNotImplemented)
 		return err
 	}
 
-	if err := esI.PurgeIndexes(); err != nil {
+	if err := seI.PurgeIndexes(); err != nil {
 		return err
 	}
 

app/server.go

@@ -35,6 +35,7 @@ import (
 	"github.com/mattermost/mattermost-server/v5/services/cache/lru"
 	"github.com/mattermost/mattermost-server/v5/services/httpservice"
 	"github.com/mattermost/mattermost-server/v5/services/imageproxy"
+	"github.com/mattermost/mattermost-server/v5/services/searchengine"
 	"github.com/mattermost/mattermost-server/v5/services/timezones"
 	"github.com/mattermost/mattermost-server/v5/services/tracing"
 	"github.com/mattermost/mattermost-server/v5/store"
@@ -98,6 +99,8 @@ type Server struct {
 	licenseListenerId       string
 	logListenerId           string
 	clusterLeaderListenerId string
+	searchConfigListenerId  string
+	searchLicenseListenerId string
 	configStore             config.Store
 	asymmetricSigningKey    *ecdsa.PrivateKey
 	postActionCookieSecret  []byte
@@ -122,15 +125,16 @@ type Server struct {
 	Log              *mlog.Logger
 	NotificationsLog *mlog.Logger
 
-	joinCluster        bool
+	joinCluster       bool
-	startMetrics       bool
+	startMetrics      bool
-	startElasticsearch bool
+	startSearchEngine bool
+
+	SearchEngine *searchengine.Broker
 
 	AccountMigration einterfaces.AccountMigrationInterface
 	Cluster          einterfaces.ClusterInterface
 	Compliance       einterfaces.ComplianceInterface
 	DataRetention    einterfaces.DataRetentionInterface
-	Elasticsearch    einterfaces.ElasticsearchInterface
 	Ldap             einterfaces.LdapInterface
 	MessageExport    einterfaces.MessageExportInterface
 	Metrics          einterfaces.MetricsInterface
@@ -206,6 +210,8 @@ func NewServer(options ...Option) (*Server, error) {
 		return nil, errors.Wrapf(err, "unable to load Mattermost translation files")
 	}
 
+	s.SearchEngine = searchengine.NewBroker(s.Config(), s.Jobs)
+
 	// at the moment we only have this implementation
 	// in the future the cache provider will be built based on the loaded config
 	s.CacheProvider = new(lru.CacheProvider)
@@ -304,10 +310,6 @@ func NewServer(options ...Option) (*Server, error) {
 		s.Metrics.StartServer()
 	}
 
-	if s.startElasticsearch && s.Elasticsearch != nil {
-		s.StartElasticsearch()
-	}
-
 	s.AddConfigListener(func(oldConfig *model.Config, newConfig *model.Config) {
 		if *oldConfig.GuestAccountsSettings.Enable && !*newConfig.GuestAccountsSettings.Enable {
 			if appErr := s.FakeApp().DeactivateGuests(); appErr != nil {
@@ -354,6 +356,11 @@ func NewServer(options ...Option) (*Server, error) {
 		}
 	}
 
+	s.SearchEngine.UpdateConfig(s.Config())
+	searchConfigListenerId, searchLicenseListenerId := s.StartSearchEngine()
+	s.searchConfigListenerId = searchConfigListenerId
+	s.searchLicenseListenerId = searchLicenseListenerId
+
 	return s, nil
 }
 
@@ -413,6 +420,7 @@ func (s *Server) Shutdown() error {
 
 	s.RemoveConfigListener(s.configListenerId)
 	s.RemoveConfigListener(s.logListenerId)
+	s.stopSearchEngine()
 
 	s.Audit.Shutdown()
 
@@ -769,33 +777,40 @@ func doSessionCleanup(s *Server) {
 	s.Store.Session().Cleanup(model.GetMillis(), SESSIONS_CLEANUP_BATCH_SIZE)
 }
 
-func (s *Server) StartElasticsearch() {
+func (s *Server) StartSearchEngine() (string, string) {
-	s.Go(func() {
+	if s.SearchEngine.ElasticsearchEngine != nil && s.SearchEngine.ElasticsearchEngine.IsActive() {
-		if err := s.Elasticsearch.Start(); err != nil {
+		s.Go(func() {
-			s.Log.Error(err.Error())
+			if err := s.SearchEngine.ElasticsearchEngine.Start(); err != nil {
-		}
+				s.Log.Error(err.Error())
-	})
+			}
+		})
+	}
 
-	s.AddConfigListener(func(oldConfig *model.Config, newConfig *model.Config) {
+	configListenerId := s.AddConfigListener(func(oldConfig *model.Config, newConfig *model.Config) {
-		if !*oldConfig.ElasticsearchSettings.EnableIndexing && *newConfig.ElasticsearchSettings.EnableIndexing {
+		if s.SearchEngine == nil {
+			return
+		}
+		s.SearchEngine.UpdateConfig(newConfig)
+
+		if s.SearchEngine.ElasticsearchEngine != nil && !*oldConfig.ElasticsearchSettings.EnableIndexing && *newConfig.ElasticsearchSettings.EnableIndexing {
 			s.Go(func() {
-				if err := s.Elasticsearch.Start(); err != nil {
+				if err := s.SearchEngine.ElasticsearchEngine.Start(); err != nil {
 					mlog.Error(err.Error())
 				}
 			})
-		} else if *oldConfig.ElasticsearchSettings.EnableIndexing && !*newConfig.ElasticsearchSettings.EnableIndexing {
+		} else if s.SearchEngine.ElasticsearchEngine != nil && *oldConfig.ElasticsearchSettings.EnableIndexing && !*newConfig.ElasticsearchSettings.EnableIndexing {
 			s.Go(func() {
-				if err := s.Elasticsearch.Stop(); err != nil {
+				if err := s.SearchEngine.ElasticsearchEngine.Stop(); err != nil {
 					mlog.Error(err.Error())
 				}
 			})
-		} else if *oldConfig.ElasticsearchSettings.Password != *newConfig.ElasticsearchSettings.Password || *oldConfig.ElasticsearchSettings.Username != *newConfig.ElasticsearchSettings.Username || *oldConfig.ElasticsearchSettings.ConnectionUrl != *newConfig.ElasticsearchSettings.ConnectionUrl || *oldConfig.ElasticsearchSettings.Sniff != *newConfig.ElasticsearchSettings.Sniff {
+		} else if s.SearchEngine.ElasticsearchEngine != nil && *oldConfig.ElasticsearchSettings.Password != *newConfig.ElasticsearchSettings.Password || *oldConfig.ElasticsearchSettings.Username != *newConfig.ElasticsearchSettings.Username || *oldConfig.ElasticsearchSettings.ConnectionUrl != *newConfig.ElasticsearchSettings.ConnectionUrl || *oldConfig.ElasticsearchSettings.Sniff != *newConfig.ElasticsearchSettings.Sniff {
 			s.Go(func() {
 				if *oldConfig.ElasticsearchSettings.EnableIndexing {
-					if err := s.Elasticsearch.Stop(); err != nil {
+					if err := s.SearchEngine.ElasticsearchEngine.Stop(); err != nil {
 						mlog.Error(err.Error())
 					}
-					if err := s.Elasticsearch.Start(); err != nil {
+					if err := s.SearchEngine.ElasticsearchEngine.Start(); err != nil {
 						mlog.Error(err.Error())
 					}
 				}
@@ -803,21 +818,38 @@ func (s *Server) StartElasticsearch() {
 		}
 	})
 
-	s.AddLicenseListener(func(oldLicense, newLicense *model.License) {
+	licenseListenerId := s.AddLicenseListener(func(oldLicense, newLicense *model.License) {
+		if s.SearchEngine == nil {
+			return
+		}
 		if oldLicense == nil && newLicense != nil {
-			s.Go(func() {
+			if s.SearchEngine.ElasticsearchEngine != nil && s.SearchEngine.ElasticsearchEngine.IsActive() {
-				if err := s.Elasticsearch.Start(); err != nil {
+				s.Go(func() {
-					mlog.Error(err.Error())
+					if err := s.SearchEngine.ElasticsearchEngine.Start(); err != nil {
-				}
+						mlog.Error(err.Error())
-			})
+					}
+				})
+			}
 		} else if oldLicense != nil && newLicense == nil {
-			s.Go(func() {
+			if s.SearchEngine.ElasticsearchEngine != nil {
-				if err := s.Elasticsearch.Stop(); err != nil {
+				s.Go(func() {
-					mlog.Error(err.Error())
+					if err := s.SearchEngine.ElasticsearchEngine.Stop(); err != nil {
-				}
+						mlog.Error(err.Error())
-			})
+					}
+				})
+			}
 		}
 	})
+
+	return configListenerId, licenseListenerId
+}
+
+func (s *Server) stopSearchEngine() {
+	s.RemoveConfigListener(s.searchConfigListenerId)
+	s.RemoveLicenseListener(s.searchLicenseListenerId)
+	if s.SearchEngine != nil && s.SearchEngine.ElasticsearchEngine != nil && s.SearchEngine.ElasticsearchEngine.IsActive() {
+		s.SearchEngine.ElasticsearchEngine.Stop()
+	}
 }
 
 func (s *Server) initDiagnostics(endpoint string) {

app/server_app_adapters.go

@@ -13,6 +13,7 @@ import (
 	"github.com/mattermost/mattermost-server/v5/services/mailservice"
 	"github.com/mattermost/mattermost-server/v5/store"
 	"github.com/mattermost/mattermost-server/v5/store/localcachelayer"
+	"github.com/mattermost/mattermost-server/v5/store/searchlayer"
 	"github.com/mattermost/mattermost-server/v5/store/sqlstore"
 	"github.com/mattermost/mattermost-server/v5/utils"
 	"github.com/pkg/errors"
@@ -61,10 +62,17 @@ func (s *Server) RunOldAppInitialization() error {
 	if s.FakeApp().Srv().newStore == nil {
 		s.FakeApp().Srv().newStore = func() store.Store {
 			return store.NewTimerLayer(
-				localcachelayer.NewLocalCacheLayer(
+				searchlayer.NewSearchLayer(
-					sqlstore.NewSqlSupplier(s.FakeApp().Config().SqlSettings, s.Metrics),
+					localcachelayer.NewLocalCacheLayer(
-					s.Metrics, s.Cluster, s.CacheProvider),
+						sqlstore.NewSqlSupplier(s.FakeApp().Config().SqlSettings, s.Metrics),
-				s.Metrics)
+						s.Metrics,
+						s.Cluster,
+						s.CacheProvider,
+					),
+					s.SearchEngine,
+				),
+				s.Metrics,
+			)
 		}
 	}
 

app/team.go

@@ -489,7 +489,7 @@ func (a *App) AddUserToTeamByToken(userId string, tokenId string) (*model.Team,
 	}
 
 	if token.Type == TOKEN_TYPE_GUEST_INVITATION {
-		channels, err := a.Srv().Store.Channel().GetChannelsByIds(strings.Split(tokenData["channels"], " "))
+		channels, err := a.Srv().Store.Channel().GetChannelsByIds(strings.Split(tokenData["channels"], " "), false)
 		if err != nil {
 			return nil, err
 		}
@@ -944,15 +944,6 @@ func (a *App) RemoveTeamMemberFromTeam(teamMember *model.TeamMember, requestorId
 		})
 	}
 
-	esInterface := a.Elasticsearch()
-	if esInterface != nil && *a.Config().ElasticsearchSettings.EnableIndexing {
-		a.Srv().Go(func() {
-			if err := a.indexUser(user); err != nil {
-				mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
-			}
-		})
-	}
-
 	if _, err := a.Srv().Store.User().UpdateUpdateAt(user.Id); err != nil {
 		return err
 	}
@@ -1135,7 +1126,7 @@ func (a *App) prepareInviteGuestsToChannels(teamId string, guestsInvite *model.G
 	}()
 	cchan := make(chan store.StoreResult, 1)
 	go func() {
-		channels, err := a.Srv().Store.Channel().GetChannelsByIds(guestsInvite.Channels)
+		channels, err := a.Srv().Store.Channel().GetChannelsByIds(guestsInvite.Channels, false)
 		cchan <- store.StoreResult{Data: channels, Err: err}
 		close(cchan)
 	}()

app/user.go

@@ -67,7 +67,7 @@ func (a *App) CreateUserWithToken(user *model.User, token *model.Token) (*model.
 		return nil, err
 	}
 
-	channels, err := a.Srv().Store.Channel().GetChannelsByIds(strings.Split(tokenData["channels"], " "))
+	channels, err := a.Srv().Store.Channel().GetChannelsByIds(strings.Split(tokenData["channels"], " "), false)
 	if err != nil {
 		return nil, err
 	}
@@ -201,40 +201,6 @@ func (a *App) IsFirstUserAccount() bool {
 	return false
 }
 
-// indexUser fetches the required information to index a user from the database and
-// calls the elasticsearch interface method
-func (a *App) indexUser(user *model.User) *model.AppError {
-	userTeams, err := a.Srv().Store.Team().GetTeamsByUserId(user.Id)
-	if err != nil {
-		return err
-	}
-
-	userTeamsIds := []string{}
-	for _, team := range userTeams {
-		userTeamsIds = append(userTeamsIds, team.Id)
-	}
-
-	userChannelMembers, err := a.Srv().Store.Channel().GetAllChannelMembersForUser(user.Id, false, true)
-	if err != nil {
-		return err
-	}
-
-	userChannelsIds := []string{}
-	for channelId := range userChannelMembers {
-		userChannelsIds = append(userChannelsIds, channelId)
-	}
-
-	return a.Elasticsearch().IndexUser(user, userTeamsIds, userChannelsIds)
-}
-
-func (a *App) indexUserFromId(userId string) *model.AppError {
-	user, err := a.GetUser(userId)
-	if err != nil {
-		return err
-	}
-	return a.indexUser(user)
-}
-
 // CreateUser creates a user and sets several fields of the returned User struct to
 // their zero values.
 func (a *App) CreateUser(user *model.User) (*model.User, *model.AppError) {
@@ -294,14 +260,6 @@ func (a *App) createUserOrGuest(user *model.User, guest bool) (*model.User, *mod
 		})
 	}
 
-	if a.IsESIndexingEnabled() {
-		a.Srv().Go(func() {
-			if err := a.indexUser(user); err != nil {
-				mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
-			}
-		})
-	}
-
 	return ruser, nil
 }
 
@@ -1193,14 +1151,6 @@ func (a *App) UpdateUser(user *model.User, sendNotifications bool) (*model.User,
 
 	a.InvalidateCacheForUser(user.Id)
 
-	if a.IsESIndexingEnabled() {
-		a.Srv().Go(func() {
-			if err := a.indexUser(user); err != nil {
-				mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
-			}
-		})
-	}
-
 	return userUpdate.New, nil
 }
 
@@ -1554,14 +1504,6 @@ func (a *App) PermanentDeleteUser(user *model.User) *model.AppError {
 
 	mlog.Warn("Permanently deleted account", mlog.String("user_email", user.Email), mlog.String("user_id", user.Id))
 
-	if a.IsESIndexingEnabled() {
-		a.Srv().Go(func() {
-			if err := a.Elasticsearch().DeleteUser(user); err != nil {
-				mlog.Error("Encountered error deleting user", mlog.String("user_id", user.Id), mlog.Err(err))
-			}
-		})
-	}
-
 	return nil
 }
 
@@ -1743,21 +1685,12 @@ func (a *App) SearchUsersNotInChannel(teamId string, channelId string, term stri
 	return users, nil
 }
 
-func (a *App) esSearchUsersInTeam(teamId, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError) {
+func (a *App) SearchUsersInTeam(teamId, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError) {
-	listOfAllowedChannels, err := a.GetViewUsersRestrictionsForTeam(a.Session().UserId, teamId)
+	var users []*model.User
-	if err != nil {
+	var err *model.AppError
-		return nil, err
+	term = strings.TrimSpace(term)
-	}
-	if listOfAllowedChannels != nil && len(listOfAllowedChannels) == 0 {
-		return []*model.User{}, nil
-	}
 
-	usersIds, err := a.Elasticsearch().SearchUsersInTeam(teamId, listOfAllowedChannels, term, options)
+	users, err = a.Srv().Store.User().Search(teamId, term, options)
-	if err != nil {
-		return nil, err
-	}
-
-	users, err := a.Srv().Store.User().GetProfileByIds(usersIds, nil, false)
 	if err != nil {
 		return nil, err
 	}
@@ -1769,32 +1702,6 @@ func (a *App) esSearchUsersInTeam(teamId, term string, options *model.UserSearch
 	return users, nil
 }
 
-func (a *App) SearchUsersInTeam(teamId, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError) {
-	var users []*model.User
-	var err *model.AppError
-	term = strings.TrimSpace(term)
-
-	if a.IsESAutocompletionEnabled() {
-		users, err = a.esSearchUsersInTeam(teamId, term, options)
-		if err != nil {
-			mlog.Error("Encountered error on SearchUsersInTeam through Elasticsearch. Falling back to default search.", mlog.Err(err))
-		}
-	}
-
-	if !a.IsESAutocompletionEnabled() || err != nil {
-		users, err = a.Srv().Store.User().Search(teamId, term, options)
-		if err != nil {
-			return nil, err
-		}
-
-		for _, user := range users {
-			a.SanitizeProfile(user, options.IsAdmin)
-		}
-	}
-
-	return users, nil
-}
-
 func (a *App) SearchUsersNotInTeam(notInTeamId string, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError) {
 	term = strings.TrimSpace(term)
 	users, err := a.Srv().Store.User().SearchNotInTeam(notInTeamId, term, options)
@@ -1823,141 +1730,31 @@ func (a *App) SearchUsersWithoutTeam(term string, options *model.UserSearchOptio
 	return users, nil
 }
 
-func (a *App) esAutocompleteUsersInChannel(teamId, channelId, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) {
-	listOfAllowedChannels, err := a.getListOfAllowedChannelsForTeam(teamId, options.ViewRestrictions)
-	if err != nil {
-		return nil, err
-	}
-	if len(listOfAllowedChannels) == 0 {
-		return &model.UserAutocompleteInChannel{}, nil
-	}
-	uchanIds := []string{}
-	nuchanIds := []string{}
-	if !strings.Contains(strings.Join(listOfAllowedChannels, "."), channelId) {
-		nuchanIds, err = a.Elasticsearch().SearchUsersInTeam(teamId, listOfAllowedChannels, term, options)
-	} else {
-		uchanIds, nuchanIds, err = a.Elasticsearch().SearchUsersInChannel(teamId, channelId, listOfAllowedChannels, term, options)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	uchan := make(chan store.StoreResult, 1)
-	go func() {
-		users, err := a.Srv().Store.User().GetProfileByIds(uchanIds, nil, false)
-		uchan <- store.StoreResult{Data: users, Err: err}
-		close(uchan)
-	}()
-
-	nuchan := make(chan store.StoreResult, 1)
-	go func() {
-		users, err := a.Srv().Store.User().GetProfileByIds(nuchanIds, nil, false)
-		nuchan <- store.StoreResult{Data: users, Err: err}
-		close(nuchan)
-	}()
-
-	autocomplete := &model.UserAutocompleteInChannel{}
-
-	result := <-uchan
-	if result.Err != nil {
-		return nil, result.Err
-	}
-	users := result.Data.([]*model.User)
-
-	for _, user := range users {
-		a.SanitizeProfile(user, options.IsAdmin)
-	}
-
-	autocomplete.InChannel = users
-
-	result = <-nuchan
-	if result.Err != nil {
-		return nil, result.Err
-	}
-	users = result.Data.([]*model.User)
-
-	for _, user := range users {
-		a.SanitizeProfile(user, options.IsAdmin)
-	}
-
-	autocomplete.OutOfChannel = users
-
-	return autocomplete, nil
-}
-
 func (a *App) AutocompleteUsersInChannel(teamId string, channelId string, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) {
-	var autocomplete *model.UserAutocompleteInChannel
-	var err *model.AppError
 	term = strings.TrimSpace(term)
 
-	if a.IsESAutocompletionEnabled() {
+	autocomplete, err := a.Srv().Store.User().AutocompleteUsersInChannel(teamId, channelId, term, options)
-		autocomplete, err = a.esAutocompleteUsersInChannel(teamId, channelId, term, options)
+	if err != nil {
-		if err != nil {
+		return nil, err
-			mlog.Error("Encountered error on AutocompleteUsersInChannel through Elasticsearch. Falling back to default autocompletion.", mlog.Err(err))
-		}
 	}
 
-	if !a.IsESAutocompletionEnabled() || err != nil {
+	for _, user := range autocomplete.InChannel {
-		autocomplete = &model.UserAutocompleteInChannel{}
+		a.SanitizeProfile(user, options.IsAdmin)
+	}
 
-		uchan := make(chan store.StoreResult, 1)
+	for _, user := range autocomplete.OutOfChannel {
-		go func() {
+		a.SanitizeProfile(user, options.IsAdmin)
-			users, err := a.Srv().Store.User().SearchInChannel(channelId, term, options)
-			uchan <- store.StoreResult{Data: users, Err: err}
-			close(uchan)
-		}()
-
-		nuchan := make(chan store.StoreResult, 1)
-		go func() {
-			users, err := a.Srv().Store.User().SearchNotInChannel(teamId, channelId, term, options)
-			nuchan <- store.StoreResult{Data: users, Err: err}
-			close(nuchan)
-		}()
-
-		result := <-uchan
-		if result.Err != nil {
-			return nil, result.Err
-		}
-
-		users := result.Data.([]*model.User)
-
-		for _, user := range users {
-			a.SanitizeProfile(user, options.IsAdmin)
-		}
-
-		autocomplete.InChannel = users
-
-		result = <-nuchan
-		if result.Err != nil {
-			return nil, result.Err
-		}
-		users = result.Data.([]*model.User)
-
-		for _, user := range users {
-			a.SanitizeProfile(user, options.IsAdmin)
-		}
-
-		autocomplete.OutOfChannel = users
 	}
 
 	return autocomplete, nil
 }
 
-func (a *App) esAutocompleteUsersInTeam(teamId, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInTeam, *model.AppError) {
+func (a *App) AutocompleteUsersInTeam(teamId string, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInTeam, *model.AppError) {
-	listOfAllowedChannels, err := a.getListOfAllowedChannelsForTeam(teamId, options.ViewRestrictions)
+	var err *model.AppError
-	if err != nil {
-		return nil, err
-	}
-	if len(listOfAllowedChannels) == 0 {
-		return &model.UserAutocompleteInTeam{}, nil
-	}
 
-	usersIds, err := a.Elasticsearch().SearchUsersInTeam(teamId, listOfAllowedChannels, term, options)
+	term = strings.TrimSpace(term)
-	if err != nil {
-		return nil, err
-	}
 
-	users, err := a.Srv().Store.User().GetProfileByIds(usersIds, nil, false)
+	users, err := a.Srv().Store.User().Search(teamId, term, options)
 	if err != nil {
 		return nil, err
 	}
@@ -1968,37 +1765,6 @@ func (a *App) esAutocompleteUsersInTeam(teamId, term string, options *model.User
 
 	autocomplete := &model.UserAutocompleteInTeam{}
 	autocomplete.InTeam = users
-
-	return autocomplete, nil
-}
-
-func (a *App) AutocompleteUsersInTeam(teamId string, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInTeam, *model.AppError) {
-	var autocomplete *model.UserAutocompleteInTeam
-	var err *model.AppError
-
-	term = strings.TrimSpace(term)
-
-	if a.IsESAutocompletionEnabled() {
-		autocomplete, err = a.esAutocompleteUsersInTeam(teamId, term, options)
-		if err != nil {
-			mlog.Error("Encountered error on AutocompleteUsersInTeam through Elasticsearch. Falling back to default autocompletion.", mlog.Err(err))
-		}
-	}
-
-	if !a.IsESAutocompletionEnabled() || err != nil {
-		autocomplete = &model.UserAutocompleteInTeam{}
-		users, err := a.Srv().Store.User().Search(teamId, term, options)
-		if err != nil {
-			return nil, err
-		}
-
-		for _, user := range users {
-			a.SanitizeProfile(user, options.IsAdmin)
-		}
-
-		autocomplete.InTeam = users
-	}
-
 	return autocomplete, nil
 }
 
@@ -2044,14 +1810,6 @@ func (a *App) UpdateOAuthUserAttrs(userData io.Reader, user *model.User, provide
 
 		user = users.New
 		a.InvalidateCacheForUser(user.Id)
-
-		if a.IsESIndexingEnabled() {
-			a.Srv().Go(func() {
-				if err := a.indexUser(user); err != nil {
-					mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
-				}
-			})
-		}
 	}
 
 	return nil
@@ -2194,61 +1952,6 @@ func (a *App) GetViewUsersRestrictions(userId string) (*model.ViewUsersRestricti
 	return &model.ViewUsersRestrictions{Teams: teamIdsWithPermission, Channels: channelIds}, nil
 }
 
-/**
- * Returns a list with the channel ids that the user has permissions to view on a
- * team. If the result is an empty list, the user can't view any channel; if it's
- * nil, there are no restrictions for the user in the specified team.
- */
-func (a *App) GetViewUsersRestrictionsForTeam(userId string, teamId string) ([]string, *model.AppError) {
-	if a.HasPermissionTo(userId, model.PERMISSION_VIEW_MEMBERS) {
-		return nil, nil
-	}
-
-	if a.HasPermissionToTeam(userId, teamId, model.PERMISSION_VIEW_MEMBERS) {
-		return nil, nil
-	}
-
-	members, err := a.Srv().Store.Channel().GetMembersForUser(teamId, userId)
-	if err != nil {
-		return nil, err
-	}
-
-	channelIds := []string{}
-	for _, membership := range *members {
-		channelIds = append(channelIds, membership.ChannelId)
-	}
-
-	return channelIds, nil
-}
-
-func (a *App) getListOfAllowedChannelsForTeam(teamId string, viewRestrictions *model.ViewUsersRestrictions) ([]string, *model.AppError) {
-	var listOfAllowedChannels []string
-	if viewRestrictions == nil || strings.Contains(strings.Join(viewRestrictions.Teams, "."), teamId) {
-		channels, err := a.Srv().Store.Channel().GetTeamChannels(teamId)
-		if err != nil {
-			return nil, err
-		}
-		channelIds := []string{}
-		for _, channel := range *channels {
-			channelIds = append(channelIds, channel.Id)
-		}
-
-		return channelIds, nil
-	}
-
-	channels, err := a.Srv().Store.Channel().GetChannelsByIds(viewRestrictions.Channels)
-	if err != nil {
-		return nil, err
-	}
-	for _, c := range channels {
-		if c.TeamId == teamId {
-			listOfAllowedChannels = append(listOfAllowedChannels, c.Id)
-		}
-	}
-
-	return listOfAllowedChannels, nil
-}
-
 // PromoteGuestToUser Convert user's roles and all his mermbership's roles from
 // guest roles to regular user roles.
 func (a *App) PromoteGuestToUser(user *model.User, requestorId string) *model.AppError {

app/user_test.go

@@ -942,90 +942,6 @@ func TestGetViewUsersRestrictions(t *testing.T) {
 	})
 }
 
-func TestGetViewUsersRestrictionsForTeam(t *testing.T) {
-	th := Setup(t).InitBasic()
-	defer th.TearDown()
-
-	team1 := th.CreateTeam()
-	team2 := th.CreateTeam()
-	th.CreateTeam() // Another team
-
-	user1 := th.CreateUser()
-
-	th.LinkUserToTeam(user1, team1)
-	th.LinkUserToTeam(user1, team2)
-
-	th.App.UpdateTeamMemberRoles(team1.Id, user1.Id, "team_user team_admin")
-
-	team1channel1 := th.CreateChannel(team1)
-	team1channel2 := th.CreateChannel(team1)
-	th.CreateChannel(team1) // Another channel
-	team1offtopic, err := th.App.GetChannelByName("off-topic", team1.Id, false)
-	require.Nil(t, err)
-	team1townsquare, err := th.App.GetChannelByName("town-square", team1.Id, false)
-	require.Nil(t, err)
-
-	team2channel1 := th.CreateChannel(team2)
-	th.CreateChannel(team2) // Another channel
-
-	th.App.AddUserToChannel(user1, team1channel1)
-	th.App.AddUserToChannel(user1, team1channel2)
-	th.App.AddUserToChannel(user1, team2channel1)
-
-	addPermission := func(role *model.Role, permission string) *model.AppError {
-		newPermissions := append(role.Permissions, permission)
-		_, err := th.App.PatchRole(role, &model.RolePatch{Permissions: &newPermissions})
-		return err
-	}
-
-	removePermission := func(role *model.Role, permission string) *model.AppError {
-		newPermissions := []string{}
-		for _, oldPermission := range role.Permissions {
-			if permission != oldPermission {
-				newPermissions = append(newPermissions, oldPermission)
-			}
-		}
-		_, err := th.App.PatchRole(role, &model.RolePatch{Permissions: &newPermissions})
-		return err
-	}
-
-	t.Run("VIEW_MEMBERS permission granted at system level", func(t *testing.T) {
-		restrictions, err := th.App.GetViewUsersRestrictionsForTeam(user1.Id, team1.Id)
-		require.Nil(t, err)
-
-		assert.Nil(t, restrictions)
-	})
-
-	t.Run("VIEW_MEMBERS permission granted at team level", func(t *testing.T) {
-		systemUserRole, err := th.App.GetRoleByName(model.SYSTEM_USER_ROLE_ID)
-		require.Nil(t, err)
-		teamUserRole, err := th.App.GetRoleByName(model.TEAM_USER_ROLE_ID)
-		require.Nil(t, err)
-
-		require.Nil(t, removePermission(systemUserRole, model.PERMISSION_VIEW_MEMBERS.Id))
-		defer addPermission(systemUserRole, model.PERMISSION_VIEW_MEMBERS.Id)
-		require.Nil(t, addPermission(teamUserRole, model.PERMISSION_VIEW_MEMBERS.Id))
-		defer removePermission(teamUserRole, model.PERMISSION_VIEW_MEMBERS.Id)
-
-		restrictions, err := th.App.GetViewUsersRestrictionsForTeam(user1.Id, team1.Id)
-		require.Nil(t, err)
-		assert.Nil(t, restrictions)
-	})
-
-	t.Run("VIEW_MEMBERS permission not granted at any level", func(t *testing.T) {
-		systemUserRole, err := th.App.GetRoleByName(model.SYSTEM_USER_ROLE_ID)
-		require.Nil(t, err)
-		require.Nil(t, removePermission(systemUserRole, model.PERMISSION_VIEW_MEMBERS.Id))
-		defer addPermission(systemUserRole, model.PERMISSION_VIEW_MEMBERS.Id)
-
-		restrictions, err := th.App.GetViewUsersRestrictionsForTeam(user1.Id, team1.Id)
-		require.Nil(t, err)
-
-		assert.NotNil(t, restrictions)
-		assert.ElementsMatch(t, []string{team1townsquare.Id, team1offtopic.Id, team1channel1.Id, team1channel2.Id}, restrictions)
-	})
-}
-
 func TestPromoteGuestToUser(t *testing.T) {
 	th := Setup(t).InitBasic()
 	defer th.TearDown()

cmd/mattermost/commands/init.go

@@ -35,7 +35,7 @@ func InitDBCommandContext(configDSN string) (*app.App, error) {
 
 	s, err := app.NewServer(
 		app.Config(configDSN, false),
-		app.StartElasticsearch,
+		app.StartSearchEngine,
 	)
 	if err != nil {
 		return nil, err

cmd/mattermost/commands/server.go

@@ -58,7 +58,7 @@ func runServer(configStore config.Store, disableConfigWatch bool, usedPlatform b
 		app.ConfigStore(configStore),
 		app.RunJobs,
 		app.JoinCluster,
-		app.StartElasticsearch,
+		app.StartSearchEngine,
 		app.StartMetrics,
 	}
 	server, err := app.NewServer(options...)

jobs/interfaces/searchengine.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package interfaces
+
+import (
+	"github.com/mattermost/mattermost-server/v5/model"
+)
+
+type SearchEngineIndexerInterface interface {
+	MakeWorker() model.Worker
+}
+
+type SearchEngineAggregatorInterface interface {
+	MakeWorker() model.Worker
+	MakeScheduler() model.Scheduler
+}

model/user_search.go

@@ -62,4 +62,6 @@ type UserSearchOptions struct {
 	Role string
 	// Restrict to search in a list of teams and channels
 	ViewRestrictions *ViewUsersRestrictions
+	// List of allowed channels
+	ListOfAllowedChannels []string
 }

services/searchengine/interface.go

@@ -1,7 +1,7 @@
 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
 // See LICENSE.txt for license information.
 
-package einterfaces
+package searchengine
 
 import (
 	"time"
@@ -9,10 +9,16 @@ import (
 	"github.com/mattermost/mattermost-server/v5/model"
 )
 
-type ElasticsearchInterface interface {
+type SearchEngineInterface interface {
 	Start() *model.AppError
 	Stop() *model.AppError
 	GetVersion() int
+	UpdateConfig(cfg *model.Config)
+	GetName() string
+	IsActive() bool
+	IsIndexingEnabled() bool
+	IsSearchEnabled() bool
+	IsAutocompletionEnabled() bool
 	IndexPost(post *model.Post, teamId string) *model.AppError
 	SearchPosts(channels *model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, model.PostSearchMatches, *model.AppError)
 	DeletePost(post *model.Post) *model.AppError

services/searchengine/mocks/SearchEngineInterface.go

@@ -1,6 +1,6 @@
 // Code generated by mockery v1.0.0. DO NOT EDIT.
 
-// Regenerate this file using `make einterfaces-mocks`.
+// Regenerate this file using `make searchengine-mocks`.
 
 package mocks
 
@@ -11,13 +11,13 @@ import (
 	mock "github.com/stretchr/testify/mock"
 )
 
-// ElasticsearchInterface is an autogenerated mock type for the ElasticsearchInterface type
+// SearchEngineInterface is an autogenerated mock type for the SearchEngineInterface type
-type ElasticsearchInterface struct {
+type SearchEngineInterface struct {
 	mock.Mock
 }
 
 // DataRetentionDeleteIndexes provides a mock function with given fields: cutoff
-func (_m *ElasticsearchInterface) DataRetentionDeleteIndexes(cutoff time.Time) *model.AppError {
+func (_m *SearchEngineInterface) DataRetentionDeleteIndexes(cutoff time.Time) *model.AppError {
 	ret := _m.Called(cutoff)
 
 	var r0 *model.AppError
@@ -33,7 +33,7 @@ func (_m *ElasticsearchInterface) DataRetentionDeleteIndexes(cutoff time.Time) *
 }
 
 // DeleteChannel provides a mock function with given fields: channel
-func (_m *ElasticsearchInterface) DeleteChannel(channel *model.Channel) *model.AppError {
+func (_m *SearchEngineInterface) DeleteChannel(channel *model.Channel) *model.AppError {
 	ret := _m.Called(channel)
 
 	var r0 *model.AppError
@@ -49,7 +49,7 @@ func (_m *ElasticsearchInterface) DeleteChannel(channel *model.Channel) *model.A
 }
 
 // DeletePost provides a mock function with given fields: post
-func (_m *ElasticsearchInterface) DeletePost(post *model.Post) *model.AppError {
+func (_m *SearchEngineInterface) DeletePost(post *model.Post) *model.AppError {
 	ret := _m.Called(post)
 
 	var r0 *model.AppError
@@ -65,7 +65,7 @@ func (_m *ElasticsearchInterface) DeletePost(post *model.Post) *model.AppError {
 }
 
 // DeleteUser provides a mock function with given fields: user
-func (_m *ElasticsearchInterface) DeleteUser(user *model.User) *model.AppError {
+func (_m *SearchEngineInterface) DeleteUser(user *model.User) *model.AppError {
 	ret := _m.Called(user)
 
 	var r0 *model.AppError
@@ -80,8 +80,22 @@ func (_m *ElasticsearchInterface) DeleteUser(user *model.User) *model.AppError {
 	return r0
 }
 
+// GetName provides a mock function with given fields:
+func (_m *SearchEngineInterface) GetName() string {
+	ret := _m.Called()
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func() string); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	return r0
+}
+
 // GetVersion provides a mock function with given fields:
-func (_m *ElasticsearchInterface) GetVersion() int {
+func (_m *SearchEngineInterface) GetVersion() int {
 	ret := _m.Called()
 
 	var r0 int
@@ -95,7 +109,7 @@ func (_m *ElasticsearchInterface) GetVersion() int {
 }
 
 // IndexChannel provides a mock function with given fields: channel
-func (_m *ElasticsearchInterface) IndexChannel(channel *model.Channel) *model.AppError {
+func (_m *SearchEngineInterface) IndexChannel(channel *model.Channel) *model.AppError {
 	ret := _m.Called(channel)
 
 	var r0 *model.AppError
@@ -111,7 +125,7 @@ func (_m *ElasticsearchInterface) IndexChannel(channel *model.Channel) *model.Ap
 }
 
 // IndexPost provides a mock function with given fields: post, teamId
-func (_m *ElasticsearchInterface) IndexPost(post *model.Post, teamId string) *model.AppError {
+func (_m *SearchEngineInterface) IndexPost(post *model.Post, teamId string) *model.AppError {
 	ret := _m.Called(post, teamId)
 
 	var r0 *model.AppError
@@ -127,7 +141,7 @@ func (_m *ElasticsearchInterface) IndexPost(post *model.Post, teamId string) *mo
 }
 
 // IndexUser provides a mock function with given fields: user, teamsIds, channelsIds
-func (_m *ElasticsearchInterface) IndexUser(user *model.User, teamsIds []string, channelsIds []string) *model.AppError {
+func (_m *SearchEngineInterface) IndexUser(user *model.User, teamsIds []string, channelsIds []string) *model.AppError {
 	ret := _m.Called(user, teamsIds, channelsIds)
 
 	var r0 *model.AppError
@@ -142,8 +156,64 @@ func (_m *ElasticsearchInterface) IndexUser(user *model.User, teamsIds []string,
 	return r0
 }
 
+// IsActive provides a mock function with given fields:
+func (_m *SearchEngineInterface) IsActive() bool {
+	ret := _m.Called()
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func() bool); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}
+
+// IsAutocompletionEnabled provides a mock function with given fields:
+func (_m *SearchEngineInterface) IsAutocompletionEnabled() bool {
+	ret := _m.Called()
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func() bool); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}
+
+// IsIndexingEnabled provides a mock function with given fields:
+func (_m *SearchEngineInterface) IsIndexingEnabled() bool {
+	ret := _m.Called()
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func() bool); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}
+
+// IsSearchEnabled provides a mock function with given fields:
+func (_m *SearchEngineInterface) IsSearchEnabled() bool {
+	ret := _m.Called()
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func() bool); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}
+
 // PurgeIndexes provides a mock function with given fields:
-func (_m *ElasticsearchInterface) PurgeIndexes() *model.AppError {
+func (_m *SearchEngineInterface) PurgeIndexes() *model.AppError {
 	ret := _m.Called()
 
 	var r0 *model.AppError
@@ -159,7 +229,7 @@ func (_m *ElasticsearchInterface) PurgeIndexes() *model.AppError {
 }
 
 // SearchChannels provides a mock function with given fields: teamId, term
-func (_m *ElasticsearchInterface) SearchChannels(teamId string, term string) ([]string, *model.AppError) {
+func (_m *SearchEngineInterface) SearchChannels(teamId string, term string) ([]string, *model.AppError) {
 	ret := _m.Called(teamId, term)
 
 	var r0 []string
@@ -184,7 +254,7 @@ func (_m *ElasticsearchInterface) SearchChannels(teamId string, term string) ([]
 }
 
 // SearchPosts provides a mock function with given fields: channels, searchParams, page, perPage
-func (_m *ElasticsearchInterface) SearchPosts(channels *model.ChannelList, searchParams []*model.SearchParams, page int, perPage int) ([]string, model.PostSearchMatches, *model.AppError) {
+func (_m *SearchEngineInterface) SearchPosts(channels *model.ChannelList, searchParams []*model.SearchParams, page int, perPage int) ([]string, model.PostSearchMatches, *model.AppError) {
 	ret := _m.Called(channels, searchParams, page, perPage)
 
 	var r0 []string
@@ -218,7 +288,7 @@ func (_m *ElasticsearchInterface) SearchPosts(channels *model.ChannelList, searc
 }
 
 // SearchUsersInChannel provides a mock function with given fields: teamId, channelId, restrictedToChannels, term, options
-func (_m *ElasticsearchInterface) SearchUsersInChannel(teamId string, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError) {
+func (_m *SearchEngineInterface) SearchUsersInChannel(teamId string, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError) {
 	ret := _m.Called(teamId, channelId, restrictedToChannels, term, options)
 
 	var r0 []string
@@ -252,7 +322,7 @@ func (_m *ElasticsearchInterface) SearchUsersInChannel(teamId string, channelId
 }
 
 // SearchUsersInTeam provides a mock function with given fields: teamId, restrictedToChannels, term, options
-func (_m *ElasticsearchInterface) SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError) {
+func (_m *SearchEngineInterface) SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError) {
 	ret := _m.Called(teamId, restrictedToChannels, term, options)
 
 	var r0 []string
@@ -277,7 +347,7 @@ func (_m *ElasticsearchInterface) SearchUsersInTeam(teamId string, restrictedToC
 }
 
 // Start provides a mock function with given fields:
-func (_m *ElasticsearchInterface) Start() *model.AppError {
+func (_m *SearchEngineInterface) Start() *model.AppError {
 	ret := _m.Called()
 
 	var r0 *model.AppError
@@ -293,7 +363,7 @@ func (_m *ElasticsearchInterface) Start() *model.AppError {
 }
 
 // Stop provides a mock function with given fields:
-func (_m *ElasticsearchInterface) Stop() *model.AppError {
+func (_m *SearchEngineInterface) Stop() *model.AppError {
 	ret := _m.Called()
 
 	var r0 *model.AppError
@@ -309,7 +379,7 @@ func (_m *ElasticsearchInterface) Stop() *model.AppError {
 }
 
 // TestConfig provides a mock function with given fields: cfg
-func (_m *ElasticsearchInterface) TestConfig(cfg *model.Config) *model.AppError {
+func (_m *SearchEngineInterface) TestConfig(cfg *model.Config) *model.AppError {
 	ret := _m.Called(cfg)
 
 	var r0 *model.AppError
@@ -323,3 +393,8 @@ func (_m *ElasticsearchInterface) TestConfig(cfg *model.Config) *model.AppError
 
 	return r0
 }
+
+// UpdateConfig provides a mock function with given fields: cfg
+func (_m *SearchEngineInterface) UpdateConfig(cfg *model.Config) {
+	_m.Called(cfg)
+}

services/searchengine/searchengine.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package searchengine
+
+import (
+	"github.com/mattermost/mattermost-server/v5/jobs"
+	"github.com/mattermost/mattermost-server/v5/model"
+)
+
+func NewBroker(cfg *model.Config, jobServer *jobs.JobServer) *Broker {
+	return &Broker{
+		cfg:       cfg,
+		jobServer: jobServer,
+	}
+}
+
+func (seb *Broker) RegisterElasticsearchEngine(es SearchEngineInterface) {
+	seb.ElasticsearchEngine = es
+}
+
+type Broker struct {
+	cfg                 *model.Config
+	jobServer           *jobs.JobServer
+	ElasticsearchEngine SearchEngineInterface
+}
+
+func (seb *Broker) UpdateConfig(cfg *model.Config) *model.AppError {
+	seb.cfg = cfg
+	if seb.ElasticsearchEngine != nil {
+		seb.ElasticsearchEngine.UpdateConfig(cfg)
+	}
+
+	return nil
+}
+
+func (seb *Broker) GetActiveEngines() []SearchEngineInterface {
+	engines := []SearchEngineInterface{}
+	if seb.ElasticsearchEngine != nil && seb.ElasticsearchEngine.IsActive() {
+		engines = append(engines, seb.ElasticsearchEngine)
+	}
+	return engines
+}

store/opentracing_layer.go

@@ -936,7 +936,7 @@ func (s *OpenTracingLayerChannelStore) GetChannelsBatchForIndexing(startTime int
 	return resultVar0, resultVar1
 }
 
-func (s *OpenTracingLayerChannelStore) GetChannelsByIds(channelIds []string) ([]*model.Channel, *model.AppError) {
+func (s *OpenTracingLayerChannelStore) GetChannelsByIds(channelIds []string, includeDeleted bool) ([]*model.Channel, *model.AppError) {
 	origCtx := s.Root.Store.Context()
 	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "ChannelStore.GetChannelsByIds")
 	s.Root.Store.SetContext(newCtx)
@@ -945,7 +945,7 @@ func (s *OpenTracingLayerChannelStore) GetChannelsByIds(channelIds []string) ([]
 	}()
 
 	defer span.Finish()
-	resultVar0, resultVar1 := s.ChannelStore.GetChannelsByIds(channelIds)
+	resultVar0, resultVar1 := s.ChannelStore.GetChannelsByIds(channelIds, includeDeleted)
 	if resultVar1 != nil {
 		span.LogFields(spanlog.Error(resultVar1))
 		ext.Error.Set(span, true)
@@ -4873,6 +4873,24 @@ func (s *OpenTracingLayerPostStore) Search(teamId string, userId string, params
 	return resultVar0, resultVar1
 }
 
+func (s *OpenTracingLayerPostStore) SearchPostsInTeamForUser(paramsList []*model.SearchParams, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, page int, perPage int) (*model.PostSearchResults, *model.AppError) {
+	origCtx := s.Root.Store.Context()
+	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "PostStore.SearchPostsInTeamForUser")
+	s.Root.Store.SetContext(newCtx)
+	defer func() {
+		s.Root.Store.SetContext(origCtx)
+	}()
+
+	defer span.Finish()
+	resultVar0, resultVar1 := s.PostStore.SearchPostsInTeamForUser(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage)
+	if resultVar1 != nil {
+		span.LogFields(spanlog.Error(resultVar1))
+		ext.Error.Set(span, true)
+	}
+
+	return resultVar0, resultVar1
+}
+
 func (s *OpenTracingLayerPostStore) Update(newPost *model.Post, oldPost *model.Post) (*model.Post, *model.AppError) {
 	origCtx := s.Root.Store.Context()
 	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "PostStore.Update")
@@ -6905,6 +6923,24 @@ func (s *OpenTracingLayerUserStore) AnalyticsGetSystemAdminCount() (int64, *mode
 	return resultVar0, resultVar1
 }
 
+func (s *OpenTracingLayerUserStore) AutocompleteUsersInChannel(teamId string, channelId string, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) {
+	origCtx := s.Root.Store.Context()
+	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "UserStore.AutocompleteUsersInChannel")
+	s.Root.Store.SetContext(newCtx)
+	defer func() {
+		s.Root.Store.SetContext(origCtx)
+	}()
+
+	defer span.Finish()
+	resultVar0, resultVar1 := s.UserStore.AutocompleteUsersInChannel(teamId, channelId, term, options)
+	if resultVar1 != nil {
+		span.LogFields(spanlog.Error(resultVar1))
+		ext.Error.Set(span, true)
+	}
+
+	return resultVar0, resultVar1
+}
+
 func (s *OpenTracingLayerUserStore) ClearAllCustomRoleAssignments() *model.AppError {
 	origCtx := s.Root.Store.Context()
 	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "UserStore.ClearAllCustomRoleAssignments")

store/searchlayer/channel_layer.go

@@ -0,0 +1,207 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package searchlayer
+
+import (
+	"github.com/mattermost/mattermost-server/v5/mlog"
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/services/searchengine"
+	"github.com/mattermost/mattermost-server/v5/store"
+)
+
+type SearchChannelStore struct {
+	store.ChannelStore
+	rootStore *SearchStore
+}
+
+func (c *SearchChannelStore) deleteChannelIndex(channel *model.Channel) {
+	if channel.Type == model.CHANNEL_OPEN {
+		for _, engine := range c.rootStore.searchEngine.GetActiveEngines() {
+			if engine.IsIndexingEnabled() {
+				go (func(engineCopy searchengine.SearchEngineInterface) {
+					if err := engineCopy.DeleteChannel(channel); err != nil {
+						mlog.Error("Encountered error deleting channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+					}
+					mlog.Debug("Removed channel from index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id))
+				})(engine)
+			}
+		}
+	}
+}
+
+func (c *SearchChannelStore) indexChannel(channel *model.Channel) {
+	if channel.Type == model.CHANNEL_OPEN {
+		for _, engine := range c.rootStore.searchEngine.GetActiveEngines() {
+			if engine.IsIndexingEnabled() {
+				go (func(engineCopy searchengine.SearchEngineInterface) {
+					if err := engineCopy.IndexChannel(channel); err != nil {
+						mlog.Error("Encountered error indexing channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+					}
+					mlog.Debug("Indexed channel in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id))
+				})(engine)
+			}
+		}
+	}
+}
+
+func (c *SearchChannelStore) Save(channel *model.Channel, maxChannels int64) (*model.Channel, *model.AppError) {
+	newChannel, err := c.ChannelStore.Save(channel, maxChannels)
+	if err == nil {
+		c.indexChannel(newChannel)
+	}
+	return newChannel, err
+}
+
+func (c *SearchChannelStore) Update(channel *model.Channel) (*model.Channel, *model.AppError) {
+	updatedChannel, err := c.ChannelStore.Update(channel)
+	if err == nil {
+		c.indexChannel(updatedChannel)
+	}
+	return updatedChannel, err
+}
+
+func (c *SearchChannelStore) UpdateMember(cm *model.ChannelMember) (*model.ChannelMember, *model.AppError) {
+	member, err := c.ChannelStore.UpdateMember(cm)
+	if err == nil {
+		c.rootStore.indexUserFromID(cm.UserId)
+		channel, channelErr := c.ChannelStore.Get(member.ChannelId, true)
+		if channelErr != nil {
+			mlog.Error("Encountered error indexing user in channel", mlog.String("channel_id", member.ChannelId), mlog.Err(err))
+		} else {
+			c.rootStore.indexUserFromID(channel.CreatorId)
+		}
+	}
+	return member, err
+}
+
+func (c *SearchChannelStore) SaveMember(cm *model.ChannelMember) (*model.ChannelMember, *model.AppError) {
+	member, err := c.ChannelStore.SaveMember(cm)
+	if err == nil {
+		c.rootStore.indexUserFromID(cm.UserId)
+		channel, channelErr := c.ChannelStore.Get(member.ChannelId, true)
+		if channelErr != nil {
+			mlog.Error("Encountered error indexing user in channel", mlog.String("channel_id", member.ChannelId), mlog.Err(err))
+		} else {
+			c.rootStore.indexUserFromID(channel.CreatorId)
+		}
+	}
+	return member, err
+}
+
+func (c *SearchChannelStore) RemoveMember(channelId, userIdToRemove string) *model.AppError {
+	err := c.ChannelStore.RemoveMember(channelId, userIdToRemove)
+	if err == nil {
+		c.rootStore.indexUserFromID(userIdToRemove)
+	}
+	return err
+}
+
+func (c *SearchChannelStore) CreateDirectChannel(user *model.User, otherUser *model.User) (*model.Channel, *model.AppError) {
+	channel, err := c.ChannelStore.CreateDirectChannel(user, otherUser)
+	if err == nil {
+		c.rootStore.indexUserFromID(user.Id)
+		c.rootStore.indexUserFromID(otherUser.Id)
+	}
+	return channel, err
+}
+
+func (c *SearchChannelStore) AutocompleteInTeam(teamId string, term string, includeDeleted bool) (*model.ChannelList, *model.AppError) {
+	var channelList *model.ChannelList
+	var err *model.AppError
+
+	allFailed := true
+	for _, engine := range c.rootStore.searchEngine.GetActiveEngines() {
+		if engine.IsAutocompletionEnabled() {
+			channelList, err = c.searchAutocompleteChannels(engine, teamId, term, includeDeleted)
+			if err != nil {
+				mlog.Error("Encountered error on AutocompleteChannels through SearchEngine. Falling back to default autocompletion.", mlog.String("search_engine", engine.GetName()), mlog.Err(err))
+				continue
+			}
+			allFailed = false
+			mlog.Debug("Using the first available search engine", mlog.String("search_engine", engine.GetName()))
+			break
+		}
+	}
+
+	if allFailed {
+		mlog.Debug("Using database search because no other search engine is available")
+		channelList, err = c.ChannelStore.AutocompleteInTeam(teamId, term, includeDeleted)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return channelList, err
+}
+
+func (c *SearchChannelStore) searchAutocompleteChannels(engine searchengine.SearchEngineInterface, teamId, term string, includeDeleted bool) (*model.ChannelList, *model.AppError) {
+	channelIds, err := engine.SearchChannels(teamId, term)
+	if err != nil {
+		return nil, err
+	}
+
+	channelList := model.ChannelList{}
+	if len(channelIds) > 0 {
+		channels, err := c.ChannelStore.GetChannelsByIds(channelIds, includeDeleted)
+		if err != nil {
+			return nil, err
+		}
+		for _, ch := range channels {
+			channelList = append(channelList, ch)
+		}
+	}
+
+	return &channelList, nil
+}
+
+func (c *SearchChannelStore) PermanentDeleteMembersByUser(userId string) *model.AppError {
+	err := c.ChannelStore.PermanentDeleteMembersByUser(userId)
+	if err == nil {
+		c.rootStore.indexUserFromID(userId)
+	}
+	return err
+}
+
+func (c *SearchChannelStore) RemoveAllDeactivatedMembers(channelId string) *model.AppError {
+	profiles, errProfiles := c.rootStore.User().GetAllProfilesInChannel(channelId, true)
+	if errProfiles != nil {
+		mlog.Error("Encountered error indexing users for channel", mlog.String("channel_id", channelId), mlog.Err(errProfiles))
+	}
+
+	err := c.ChannelStore.RemoveAllDeactivatedMembers(channelId)
+	if err == nil && errProfiles == nil {
+		for _, user := range profiles {
+			if user.DeleteAt != 0 {
+				c.rootStore.indexUser(user)
+			}
+		}
+	}
+	return err
+}
+
+func (c *SearchChannelStore) PermanentDeleteMembersByChannel(channelId string) *model.AppError {
+	profiles, errProfiles := c.rootStore.User().GetAllProfilesInChannel(channelId, true)
+	if errProfiles != nil {
+		mlog.Error("Encountered error indexing users for channel", mlog.String("channel_id", channelId), mlog.Err(errProfiles))
+	}
+
+	err := c.ChannelStore.PermanentDeleteMembersByChannel(channelId)
+	if err == nil && errProfiles == nil {
+		for _, user := range profiles {
+			c.rootStore.indexUser(user)
+		}
+	}
+	return err
+}
+
+func (c *SearchChannelStore) PermanentDelete(channelId string) *model.AppError {
+	channel, channelErr := c.ChannelStore.Get(channelId, true)
+	if channelErr != nil {
+		mlog.Error("Encountered error deleting channel", mlog.String("channel_id", channelId), mlog.Err(channelErr))
+	}
+	err := c.ChannelStore.PermanentDelete(channelId)
+	if err == nil && channelErr == nil {
+		c.deleteChannelIndex(channel)
+	}
+	return err
+}

store/searchlayer/layer.go

@@ -0,0 +1,93 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package searchlayer
+
+import (
+	"github.com/mattermost/mattermost-server/v5/mlog"
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/services/searchengine"
+	"github.com/mattermost/mattermost-server/v5/store"
+)
+
+type SearchStore struct {
+	store.Store
+	searchEngine *searchengine.Broker
+	user         *SearchUserStore
+	team         *SearchTeamStore
+	channel      *SearchChannelStore
+	post         *SearchPostStore
+}
+
+func NewSearchLayer(baseStore store.Store, searchEngine *searchengine.Broker) SearchStore {
+	searchStore := SearchStore{
+		Store:        baseStore,
+		searchEngine: searchEngine,
+	}
+	searchStore.channel = &SearchChannelStore{ChannelStore: baseStore.Channel(), rootStore: &searchStore}
+	searchStore.post = &SearchPostStore{PostStore: baseStore.Post(), rootStore: &searchStore}
+	searchStore.team = &SearchTeamStore{TeamStore: baseStore.Team(), rootStore: &searchStore}
+	searchStore.user = &SearchUserStore{UserStore: baseStore.User(), rootStore: &searchStore}
+
+	return searchStore
+}
+
+func (s SearchStore) Channel() store.ChannelStore {
+	return s.channel
+}
+
+func (s SearchStore) Post() store.PostStore {
+	return s.post
+}
+
+func (s SearchStore) Team() store.TeamStore {
+	return s.team
+}
+
+func (s SearchStore) User() store.UserStore {
+	return s.user
+}
+
+func (s SearchStore) indexUserFromID(userId string) {
+	user, err := s.User().Get(userId)
+	if err != nil {
+		return
+	}
+	s.indexUser(user)
+}
+
+func (s SearchStore) indexUser(user *model.User) {
+	for _, engine := range s.searchEngine.GetActiveEngines() {
+		if engine.IsIndexingEnabled() {
+			go (func(engineCopy searchengine.SearchEngineInterface) {
+				userTeams, err := s.Team().GetTeamsByUserId(user.Id)
+				if err != nil {
+					mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+					return
+				}
+
+				userTeamsIds := []string{}
+				for _, team := range userTeams {
+					userTeamsIds = append(userTeamsIds, team.Id)
+				}
+
+				userChannelMembers, err := s.Channel().GetAllChannelMembersForUser(user.Id, false, true)
+				if err != nil {
+					mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+					return
+				}
+
+				userChannelsIds := []string{}
+				for channelId := range userChannelMembers {
+					userChannelsIds = append(userChannelsIds, channelId)
+				}
+
+				if err := engineCopy.IndexUser(user, userTeamsIds, userChannelsIds); err != nil {
+					mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+					return
+				}
+				mlog.Debug("Indexed user in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("user_id", user.Id))
+			})(engine)
+		}
+	}
+}

store/searchlayer/post_layer.go

@@ -0,0 +1,126 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package searchlayer
+
+import (
+	"github.com/mattermost/mattermost-server/v5/mlog"
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/services/searchengine"
+	"github.com/mattermost/mattermost-server/v5/store"
+)
+
+type SearchPostStore struct {
+	store.PostStore
+	rootStore *SearchStore
+}
+
+func (s SearchPostStore) indexPost(post *model.Post) {
+	for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
+		if engine.IsIndexingEnabled() {
+			go (func(engineCopy searchengine.SearchEngineInterface) {
+				channel, chanErr := s.rootStore.Channel().GetForPost(post.Id)
+				if chanErr != nil {
+					mlog.Error("Couldn't get channel for post for SearchEngine indexing.", mlog.String("channel_id", post.ChannelId), mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id))
+					return
+				}
+				if err := engineCopy.IndexPost(post, channel.TeamId); err != nil {
+					mlog.Error("Encountered error indexing post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+				}
+				mlog.Debug("Indexed post in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id))
+			})(engine)
+		}
+	}
+}
+
+func (s SearchPostStore) deletePostIndex(post *model.Post) {
+	for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
+		if engine.IsIndexingEnabled() {
+			go (func(engineCopy searchengine.SearchEngineInterface) {
+				if err := engineCopy.DeletePost(post); err != nil {
+					mlog.Error("Encountered error deleting post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+				}
+				mlog.Debug("Removed post from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id))
+			})(engine)
+		}
+	}
+}
+
+func (s SearchPostStore) Update(newPost, oldPost *model.Post) (*model.Post, *model.AppError) {
+	post, err := s.PostStore.Update(newPost, oldPost)
+
+	if err == nil {
+		s.indexPost(post)
+	}
+	return post, err
+}
+
+func (s SearchPostStore) Save(post *model.Post) (*model.Post, *model.AppError) {
+	npost, err := s.PostStore.Save(post)
+
+	if err == nil {
+		s.indexPost(npost)
+	}
+	return npost, err
+}
+
+func (s SearchPostStore) Delete(postId string, date int64, deletedByID string) *model.AppError {
+	err := s.PostStore.Delete(postId, date, deletedByID)
+
+	if err == nil {
+		postList, err2 := s.PostStore.Get(postId, true)
+		if postList != nil && len(postList.Order) > 0 {
+			if err2 != nil {
+				s.deletePostIndex(postList.Posts[postList.Order[0]])
+			}
+		}
+	}
+	return err
+}
+
+func (s SearchPostStore) searchPostsInTeamForUserByEngine(engine searchengine.SearchEngineInterface, paramsList []*model.SearchParams, userId, teamId string, isOrSearch, includeDeletedChannels bool, page, perPage int) (*model.PostSearchResults, *model.AppError) {
+	// We only allow the user to search in channels they are a member of.
+	userChannels, err := s.rootStore.Channel().GetChannels(teamId, userId, includeDeletedChannels)
+	if err != nil {
+		mlog.Error("error getting channel for user", mlog.Err(err))
+		return nil, err
+	}
+
+	postIds, matches, err := engine.SearchPosts(userChannels, paramsList, page, perPage)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get the posts
+	postList := model.NewPostList()
+	if len(postIds) > 0 {
+		posts, err := s.PostStore.GetPostsByIds(postIds)
+		if err != nil {
+			return nil, err
+		}
+		for _, p := range posts {
+			if p.DeleteAt == 0 {
+				postList.AddPost(p)
+				postList.AddOrder(p.Id)
+			}
+		}
+	}
+
+	return model.MakePostSearchResults(postList, matches), nil
+}
+
+func (s SearchPostStore) SearchPostsInTeamForUser(paramsList []*model.SearchParams, userId, teamId string, isOrSearch, includeDeletedChannels bool, page, perPage int) (*model.PostSearchResults, *model.AppError) {
+	for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
+		if engine.IsSearchEnabled() {
+			results, err := s.searchPostsInTeamForUserByEngine(engine, paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage)
+			if err != nil {
+				mlog.Error("Encountered error on SearchPostsInTeamForUser.", mlog.String("search_engine", engine.GetName()), mlog.Err(err))
+				continue
+			}
+			mlog.Debug("Using the first available search engine", mlog.String("search_engine", engine.GetName()))
+			return results, err
+		}
+	}
+	mlog.Debug("Using database search because no other search engine is available")
+	return s.PostStore.SearchPostsInTeamForUser(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage)
+}

store/searchlayer/team_layer.go

@@ -0,0 +1,46 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package searchlayer
+
+import (
+	model "github.com/mattermost/mattermost-server/v5/model"
+	store "github.com/mattermost/mattermost-server/v5/store"
+)
+
+type SearchTeamStore struct {
+	store.TeamStore
+	rootStore *SearchStore
+}
+
+func (s SearchTeamStore) SaveMember(teamMember *model.TeamMember, maxUsersPerTeam int) (*model.TeamMember, *model.AppError) {
+	member, err := s.TeamStore.SaveMember(teamMember, maxUsersPerTeam)
+	if err == nil {
+		s.rootStore.indexUserFromID(member.UserId)
+	}
+	return member, err
+}
+
+func (s SearchTeamStore) UpdateMember(teamMember *model.TeamMember) (*model.TeamMember, *model.AppError) {
+	member, err := s.TeamStore.UpdateMember(teamMember)
+	if err == nil {
+		s.rootStore.indexUserFromID(member.UserId)
+	}
+	return member, err
+}
+
+func (s SearchTeamStore) RemoveMember(teamId string, userId string) *model.AppError {
+	err := s.TeamStore.RemoveMember(teamId, userId)
+	if err == nil {
+		s.rootStore.indexUserFromID(userId)
+	}
+	return err
+}
+
+func (s SearchTeamStore) RemoveAllMembersByUser(userId string) *model.AppError {
+	err := s.TeamStore.RemoveAllMembersByUser(userId)
+	if err == nil {
+		s.rootStore.indexUserFromID(userId)
+	}
+	return err
+}

store/searchlayer/user_layer.go

@@ -0,0 +1,192 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package searchlayer
+
+import (
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v5/mlog"
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/services/searchengine"
+	"github.com/mattermost/mattermost-server/v5/store"
+)
+
+type SearchUserStore struct {
+	store.UserStore
+	rootStore *SearchStore
+}
+
+func (s *SearchUserStore) deleteUserIndex(user *model.User) {
+	for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
+		if engine.IsIndexingEnabled() {
+			go (func(engineCopy searchengine.SearchEngineInterface) {
+				if err := engineCopy.DeleteUser(user); err != nil {
+					mlog.Error("Encountered error deleting user", mlog.String("user_id", user.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+					return
+				}
+				mlog.Debug("Removed user from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("user_id", user.Id))
+			})(engine)
+		}
+	}
+}
+
+func (s *SearchUserStore) Search(teamId, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError) {
+	for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
+		if engine.IsSearchEnabled() {
+			listOfAllowedChannels, err := s.getListOfAllowedChannelsForTeam(teamId, options.ViewRestrictions)
+			if err != nil {
+				mlog.Error("Encountered error on Search.", mlog.String("search_engine", engine.GetName()), mlog.Err(err))
+				continue
+			}
+			if len(listOfAllowedChannels) == 0 {
+				return []*model.User{}, nil
+			}
+
+			usersIds, err := engine.SearchUsersInTeam(teamId, listOfAllowedChannels, term, options)
+			if err != nil {
+				continue
+			}
+
+			users, err := s.UserStore.GetProfileByIds(usersIds, nil, false)
+			if err != nil {
+				continue
+			}
+
+			mlog.Debug("Using the first available search engine", mlog.String("search_engine", engine.GetName()))
+			return users, nil
+		}
+	}
+	mlog.Debug("Using database search because no other search engine is available")
+	return s.UserStore.Search(teamId, term, options)
+}
+
+func (s *SearchUserStore) Update(user *model.User, trustedUpdateData bool) (*model.UserUpdate, *model.AppError) {
+	userUpdate, err := s.UserStore.Update(user, trustedUpdateData)
+
+	if err == nil {
+		s.rootStore.indexUser(userUpdate.New)
+	}
+	return userUpdate, err
+}
+
+func (s *SearchUserStore) Save(user *model.User) (*model.User, *model.AppError) {
+	nuser, err := s.UserStore.Save(user)
+
+	if err == nil {
+		s.rootStore.indexUser(nuser)
+	}
+	return nuser, err
+}
+
+func (s *SearchUserStore) PermanentDelete(userId string) *model.AppError {
+	user, userErr := s.UserStore.Get(userId)
+	if userErr != nil {
+		mlog.Error("Encountered error deleting user", mlog.String("user_id", userId), mlog.Err(userErr))
+	}
+	err := s.UserStore.PermanentDelete(userId)
+	if err == nil && userErr == nil {
+		s.deleteUserIndex(user)
+	}
+	return err
+}
+
+func (s *SearchUserStore) autocompleteUsersInChannelByEngine(engine searchengine.SearchEngineInterface, teamId, channelId, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) {
+	var err *model.AppError
+	uchanIds := []string{}
+	nuchanIds := []string{}
+	if options.ListOfAllowedChannels != nil && !strings.Contains(strings.Join(options.ListOfAllowedChannels, "."), channelId) {
+		nuchanIds, err = engine.SearchUsersInTeam(teamId, options.ListOfAllowedChannels, term, options)
+	} else {
+		uchanIds, nuchanIds, err = engine.SearchUsersInChannel(teamId, channelId, options.ListOfAllowedChannels, term, options)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	uchan := make(chan store.StoreResult, 1)
+	go func() {
+		users, err := s.UserStore.GetProfileByIds(uchanIds, nil, false)
+		uchan <- store.StoreResult{Data: users, Err: err}
+		close(uchan)
+	}()
+
+	nuchan := make(chan store.StoreResult, 1)
+	go func() {
+		users, err := s.UserStore.GetProfileByIds(nuchanIds, nil, false)
+		nuchan <- store.StoreResult{Data: users, Err: err}
+		close(nuchan)
+	}()
+
+	autocomplete := &model.UserAutocompleteInChannel{}
+
+	result := <-uchan
+	if result.Err != nil {
+		return nil, result.Err
+	}
+	inUsers := result.Data.([]*model.User)
+	autocomplete.InChannel = inUsers
+
+	result = <-nuchan
+	if result.Err != nil {
+		return nil, result.Err
+	}
+	outUsers := result.Data.([]*model.User)
+	autocomplete.OutOfChannel = outUsers
+
+	return autocomplete, nil
+}
+
+func (s *SearchUserStore) getListOfAllowedChannelsForTeam(teamId string, viewRestrictions *model.ViewUsersRestrictions) ([]string, *model.AppError) {
+	var listOfAllowedChannels []string
+	if viewRestrictions == nil || strings.Contains(strings.Join(viewRestrictions.Teams, "."), teamId) {
+		channels, err := s.rootStore.Channel().GetTeamChannels(teamId)
+		if err != nil {
+			return nil, err
+		}
+		channelIds := []string{}
+		for _, channel := range *channels {
+			channelIds = append(channelIds, channel.Id)
+		}
+
+		return channelIds, nil
+	}
+
+	channels, err := s.rootStore.Channel().GetChannelsByIds(viewRestrictions.Channels, false)
+	if err != nil {
+		return nil, err
+	}
+	for _, c := range channels {
+		if c.TeamId == teamId {
+			listOfAllowedChannels = append(listOfAllowedChannels, c.Id)
+		}
+	}
+
+	return listOfAllowedChannels, nil
+}
+
+func (s *SearchUserStore) AutocompleteUsersInChannel(teamId, channelId, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) {
+	for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
+		if engine.IsAutocompletionEnabled() {
+			listOfAllowedChannels, err := s.getListOfAllowedChannelsForTeam(teamId, options.ViewRestrictions)
+			if err != nil {
+				mlog.Error("Encountered error on AutocompleteUsersInChannel.", mlog.String("search_engine", engine.GetName()), mlog.Err(err))
+				continue
+			}
+			if len(listOfAllowedChannels) == 0 {
+				return &model.UserAutocompleteInChannel{}, nil
+			}
+			options.ListOfAllowedChannels = listOfAllowedChannels
+			autocomplete, err := s.autocompleteUsersInChannelByEngine(engine, teamId, channelId, term, options)
+			if err != nil {
+				mlog.Error("Encountered error on AutocompleteUsersInChannel.", mlog.String("search_engine", engine.GetName()), mlog.Err(err))
+				continue
+			}
+			mlog.Debug("Using the first available search engine", mlog.String("search_engine", engine.GetName()))
+			return autocomplete, err
+		}
+	}
+
+	mlog.Debug("Using database search because no other search engine is available")
+	return s.UserStore.AutocompleteUsersInChannel(teamId, channelId, term, options)
+}

store/sqlstore/channel_store.go

@@ -1865,9 +1865,12 @@ func (s SqlChannelStore) GetAll(teamId string) ([]*model.Channel, *model.AppErro
 	return data, nil
 }
 
-func (s SqlChannelStore) GetChannelsByIds(channelIds []string) ([]*model.Channel, *model.AppError) {
+func (s SqlChannelStore) GetChannelsByIds(channelIds []string, includeDeleted bool) ([]*model.Channel, *model.AppError) {
 	keys, params := MapStringsToQueryParams(channelIds, "Channel")
 	query := `SELECT * FROM Channels WHERE Id IN ` + keys + ` ORDER BY Name`
+	if !includeDeleted {
+		query = `SELECT * FROM Channels WHERE DeleteAt=0 AND Id IN ` + keys + ` ORDER BY Name`
+	}
 
 	var channels []*model.Channel
 	_, err := s.GetReplica().Select(&channels, query, params)

store/sqlstore/post_store.go

@@ -999,7 +999,7 @@ func (s *SqlPostStore) buildCreateDateFilterClause(params *model.SearchParams, q
 	return searchQuery, queryParams
 }
 
-func (s *SqlPostStore) buildSearchChannelFilterClause(channels []string, paramPrefix string, exclusion bool, queryParams map[string]interface{}) (string, map[string]interface{}) {
+func (s *SqlPostStore) buildSearchChannelFilterClause(channels []string, paramPrefix string, exclusion bool, queryParams map[string]interface{}, byName bool) (string, map[string]interface{}) {
 	if len(channels) == 0 {
 		return "", queryParams
 	}
@@ -1011,13 +1011,20 @@ func (s *SqlPostStore) buildSearchChannelFilterClause(channels []string, paramPr
 		queryParams[paramName] = channel
 	}
 	clause := strings.Join(clauseSlice, ", ")
-	if exclusion {
+	if byName {
-		return "AND Name NOT IN (" + clause + ")", queryParams
+		if exclusion {
+			return "AND Name NOT IN (" + clause + ")", queryParams
+		}
+		return "AND Name IN (" + clause + ")", queryParams
 	}
-	return "AND Name IN (" + clause + ")", queryParams
+
+	if exclusion {
+		return "AND Id NOT IN (" + clause + ")", queryParams
+	}
+	return "AND Id IN (" + clause + ")", queryParams
 }
 
-func (s *SqlPostStore) buildSearchUserFilterClause(users []string, paramPrefix string, exclusion bool, queryParams map[string]interface{}) (string, map[string]interface{}) {
+func (s *SqlPostStore) buildSearchUserFilterClause(users []string, paramPrefix string, exclusion bool, queryParams map[string]interface{}, byUsername bool) (string, map[string]interface{}) {
 	if len(users) == 0 {
 		return "", queryParams
 	}
@@ -1028,13 +1035,19 @@ func (s *SqlPostStore) buildSearchUserFilterClause(users []string, paramPrefix s
 		queryParams[paramName] = user
 	}
 	clause := strings.Join(clauseSlice, ", ")
-	if exclusion {
+	if byUsername {
-		return "AND Username NOT IN (" + clause + ")", queryParams
+		if exclusion {
+			return "AND Username NOT IN (" + clause + ")", queryParams
+		}
+		return "AND Username IN (" + clause + ")", queryParams
 	}
-	return "AND Username IN (" + clause + ")", queryParams
+	if exclusion {
+		return "AND Id NOT IN (" + clause + ")", queryParams
+	}
+	return "AND Id IN (" + clause + ")", queryParams
 }
 
-func (s *SqlPostStore) buildSearchPostFilterClause(fromUsers []string, excludedUsers []string, queryParams map[string]interface{}) (string, map[string]interface{}) {
+func (s *SqlPostStore) buildSearchPostFilterClause(fromUsers []string, excludedUsers []string, queryParams map[string]interface{}, userByUsername bool) (string, map[string]interface{}) {
 	if len(fromUsers) == 0 && len(excludedUsers) == 0 {
 		return "", queryParams
 	}
@@ -1052,16 +1065,20 @@ func (s *SqlPostStore) buildSearchPostFilterClause(fromUsers []string, excludedU
 				FROM_USER_FILTER
 				EXCLUDED_USER_FILTER)`
 
-	fromUserClause, queryParams := s.buildSearchUserFilterClause(fromUsers, "FromUser", false, queryParams)
+	fromUserClause, queryParams := s.buildSearchUserFilterClause(fromUsers, "FromUser", false, queryParams, userByUsername)
 	filterQuery = strings.Replace(filterQuery, "FROM_USER_FILTER", fromUserClause, 1)
 
-	excludedUserClause, queryParams := s.buildSearchUserFilterClause(excludedUsers, "ExcludedUser", true, queryParams)
+	excludedUserClause, queryParams := s.buildSearchUserFilterClause(excludedUsers, "ExcludedUser", true, queryParams, userByUsername)
 	filterQuery = strings.Replace(filterQuery, "EXCLUDED_USER_FILTER", excludedUserClause, 1)
 
 	return filterQuery, queryParams
 }
 
 func (s *SqlPostStore) Search(teamId string, userId string, params *model.SearchParams) (*model.PostList, *model.AppError) {
+	return s.search(teamId, userId, params, true, true)
+}
+
+func (s *SqlPostStore) search(teamId string, userId string, params *model.SearchParams, channelsByName bool, userByUsername bool) (*model.PostList, *model.AppError) {
 	queryParams := map[string]interface{}{
 		"TeamId": teamId,
 		"UserId": userId,
@@ -1114,13 +1131,13 @@ func (s *SqlPostStore) Search(teamId string, userId string, params *model.Search
 				ORDER BY CreateAt DESC
 			LIMIT 100`
 
-	inChannelClause, queryParams := s.buildSearchChannelFilterClause(params.InChannels, "InChannel", false, queryParams)
+	inChannelClause, queryParams := s.buildSearchChannelFilterClause(params.InChannels, "InChannel", false, queryParams, channelsByName)
 	searchQuery = strings.Replace(searchQuery, "IN_CHANNEL_FILTER", inChannelClause, 1)
 
-	excludedChannelClause, queryParams := s.buildSearchChannelFilterClause(params.ExcludedChannels, "ExcludedChannel", true, queryParams)
+	excludedChannelClause, queryParams := s.buildSearchChannelFilterClause(params.ExcludedChannels, "ExcludedChannel", true, queryParams, channelsByName)
 	searchQuery = strings.Replace(searchQuery, "EXCLUDED_CHANNEL_FILTER", excludedChannelClause, 1)
 
-	postFilterClause, queryParams := s.buildSearchPostFilterClause(params.FromUsers, params.ExcludedUsers, queryParams)
+	postFilterClause, queryParams := s.buildSearchPostFilterClause(params.FromUsers, params.ExcludedUsers, queryParams, userByUsername)
 	searchQuery = strings.Replace(searchQuery, "POST_FILTER", postFilterClause, 1)
 
 	createDateFilterClause, queryParams := s.buildCreateDateFilterClause(params, queryParams)
@@ -1666,3 +1683,49 @@ func (s *SqlPostStore) GetDirectPostParentsForExportAfter(limit int, afterId str
 	}
 	return posts, nil
 }
+
+func (s *SqlPostStore) SearchPostsInTeamForUser(paramsList []*model.SearchParams, userId, teamId string, isOrSearch, includeDeletedChannels bool, page, perPage int) (*model.PostSearchResults, *model.AppError) {
+	// Since we don't support paging for DB search, we just return nothing for later pages
+	if page > 0 {
+		return model.MakePostSearchResults(model.NewPostList(), nil), nil
+	}
+
+	var wg sync.WaitGroup
+
+	pchan := make(chan store.StoreResult, len(paramsList))
+
+	for _, params := range paramsList {
+		// Don't allow users to search for everything.
+		if params.Terms == "*" {
+			continue
+		}
+
+		params.IncludeDeletedChannels = includeDeletedChannels
+		params.OrTerms = isOrSearch
+
+		wg.Add(1)
+
+		go func(params *model.SearchParams) {
+			defer wg.Done()
+			postList, err := s.search(teamId, userId, params, false, false)
+			pchan <- store.StoreResult{Data: postList, Err: err}
+		}(params)
+	}
+
+	wg.Wait()
+	close(pchan)
+
+	posts := model.NewPostList()
+
+	for result := range pchan {
+		if result.Err != nil {
+			return nil, result.Err
+		}
+		data := result.Data.(*model.PostList)
+		posts.Extend(data)
+	}
+
+	posts.SortByCreateAt()
+
+	return model.MakePostSearchResults(posts, nil), nil
+}

store/sqlstore/user_store.go

@@ -1716,3 +1716,35 @@ func (us SqlUserStore) DemoteUserToGuest(userId string) *model.AppError {
 	}
 	return nil
 }
+
+func (us SqlUserStore) AutocompleteUsersInChannel(teamId, channelId, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) {
+	autocomplete := &model.UserAutocompleteInChannel{}
+	uchan := make(chan store.StoreResult, 1)
+	go func() {
+		users, err := us.SearchInChannel(channelId, term, options)
+		uchan <- store.StoreResult{Data: users, Err: err}
+		close(uchan)
+	}()
+
+	nuchan := make(chan store.StoreResult, 1)
+	go func() {
+		users, err := us.SearchNotInChannel(teamId, channelId, term, options)
+		nuchan <- store.StoreResult{Data: users, Err: err}
+		close(nuchan)
+	}()
+
+	result := <-uchan
+	if result.Err != nil {
+		return nil, result.Err
+	}
+	users := result.Data.([]*model.User)
+	autocomplete.InChannel = users
+
+	result = <-nuchan
+	if result.Err != nil {
+		return nil, result.Err
+	}
+	users = result.Data.([]*model.User)
+	autocomplete.OutOfChannel = users
+	return autocomplete, nil
+}

store/store.go

@@ -148,7 +148,7 @@ type ChannelStore interface {
 	GetChannelCounts(teamId string, userId string) (*model.ChannelCounts, *model.AppError)
 	GetTeamChannels(teamId string) (*model.ChannelList, *model.AppError)
 	GetAll(teamId string) ([]*model.Channel, *model.AppError)
-	GetChannelsByIds(channelIds []string) ([]*model.Channel, *model.AppError)
+	GetChannelsByIds(channelIds []string, includeDeleted bool) ([]*model.Channel, *model.AppError)
 	GetForPost(postId string) (*model.Channel, *model.AppError)
 	SaveMember(member *model.ChannelMember) (*model.ChannelMember, *model.AppError)
 	UpdateMember(member *model.ChannelMember) (*model.ChannelMember, *model.AppError)
@@ -256,6 +256,7 @@ type PostStore interface {
 	GetParentsForExportAfter(limit int, afterId string) ([]*model.PostForExport, *model.AppError)
 	GetRepliesForExport(parentId string) ([]*model.ReplyForExport, *model.AppError)
 	GetDirectPostParentsForExportAfter(limit int, afterId string) ([]*model.DirectPostForExport, *model.AppError)
+	SearchPostsInTeamForUser(paramsList []*model.SearchParams, userId, teamId string, isOrSearch, includeDeletedChannels bool, page, perPage int) (*model.PostSearchResults, *model.AppError)
 }
 
 type UserStore interface {
@@ -320,6 +321,7 @@ type UserStore interface {
 	PromoteGuestToUser(userID string) *model.AppError
 	DemoteUserToGuest(userID string) *model.AppError
 	DeactivateGuests() ([]string, *model.AppError)
+	AutocompleteUsersInChannel(teamId, channelId, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError)
 }
 
 type BotStore interface {

store/storetest/channel_store.go

@@ -447,6 +447,18 @@ func testChannelStoreGetChannelsByIds(t *testing.T, ss store.Store) {
 	o2.Name = "bb" + model.NewId() + "b"
 	o2.Type = model.CHANNEL_DIRECT
 
+	o3 := model.Channel{}
+	o3.TeamId = model.NewId()
+	o3.DisplayName = "Deleted channel"
+	o3.Name = "cc" + model.NewId() + "b"
+	o3.Type = model.CHANNEL_OPEN
+	_, err = ss.Channel().Save(&o3, -1)
+	require.Nil(t, err)
+	err = ss.Channel().Delete(o3.Id, 123)
+	require.Nil(t, err)
+	o3.DeleteAt = 123
+	o3.UpdateAt = 123
+
 	m1 := model.ChannelMember{}
 	m1.ChannelId = o2.Id
 	m1.UserId = u1.Id
@@ -460,17 +472,30 @@ func testChannelStoreGetChannelsByIds(t *testing.T, ss store.Store) {
 	_, err = ss.Channel().SaveDirectChannel(&o2, &m1, &m2)
 	require.Nil(t, err)
 
-	r1, err := ss.Channel().GetChannelsByIds([]string{o1.Id, o2.Id})
+	t.Run("Get 2 existing channels", func(t *testing.T) {
-	require.Nil(t, err, err)
+		r1, err := ss.Channel().GetChannelsByIds([]string{o1.Id, o2.Id}, false)
-	require.Len(t, r1, 2, "invalid returned channels, exepected 2 and got "+strconv.Itoa(len(r1)))
+		require.Nil(t, err, err)
-	require.Equal(t, o1.ToJson(), r1[0].ToJson())
+		require.Len(t, r1, 2, "invalid returned channels, exepected 2 and got "+strconv.Itoa(len(r1)))
-	require.Equal(t, o2.ToJson(), r1[1].ToJson())
+		require.Equal(t, o1.ToJson(), r1[0].ToJson())
+		require.Equal(t, o2.ToJson(), r1[1].ToJson())
+	})
 
-	nonexistentId := "abcd1234"
+	t.Run("Get 1 existing and 1 not existing channel", func(t *testing.T) {
-	r2, err := ss.Channel().GetChannelsByIds([]string{o1.Id, nonexistentId})
+		nonexistentId := "abcd1234"
-	require.Nil(t, err, err)
+		r2, err := ss.Channel().GetChannelsByIds([]string{o1.Id, nonexistentId}, false)
-	require.Len(t, r2, 1, "invalid returned channels, expected 1 and got "+strconv.Itoa(len(r2)))
+		require.Nil(t, err, err)
-	require.Equal(t, o1.ToJson(), r2[0].ToJson(), "invalid returned channel")
+		require.Len(t, r2, 1, "invalid returned channels, expected 1 and got "+strconv.Itoa(len(r2)))
+		require.Equal(t, o1.ToJson(), r2[0].ToJson(), "invalid returned channel")
+	})
+
+	t.Run("Get 2 existing and 1 deleted channel", func(t *testing.T) {
+		r1, err := ss.Channel().GetChannelsByIds([]string{o1.Id, o2.Id, o3.Id}, true)
+		require.Nil(t, err, err)
+		require.Len(t, r1, 3, "invalid returned channels, exepected 3 and got "+strconv.Itoa(len(r1)))
+		require.Equal(t, o1.ToJson(), r1[0].ToJson())
+		require.Equal(t, o2.ToJson(), r1[1].ToJson())
+		require.Equal(t, o3.ToJson(), r1[2].ToJson())
+	})
 }
 
 func testChannelStoreGetForPost(t *testing.T, ss store.Store) {

store/storetest/mocks/ChannelStore.go

@@ -619,13 +619,13 @@ func (_m *ChannelStore) GetChannelsBatchForIndexing(startTime int64, endTime int
 	return r0, r1
 }
 
-// GetChannelsByIds provides a mock function with given fields: channelIds
+// GetChannelsByIds provides a mock function with given fields: channelIds, includeDeleted
-func (_m *ChannelStore) GetChannelsByIds(channelIds []string) ([]*model.Channel, *model.AppError) {
+func (_m *ChannelStore) GetChannelsByIds(channelIds []string, includeDeleted bool) ([]*model.Channel, *model.AppError) {
-	ret := _m.Called(channelIds)
+	ret := _m.Called(channelIds, includeDeleted)
 
 	var r0 []*model.Channel
-	if rf, ok := ret.Get(0).(func([]string) []*model.Channel); ok {
+	if rf, ok := ret.Get(0).(func([]string, bool) []*model.Channel); ok {
-		r0 = rf(channelIds)
+		r0 = rf(channelIds, includeDeleted)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).([]*model.Channel)
@@ -633,8 +633,8 @@ func (_m *ChannelStore) GetChannelsByIds(channelIds []string) ([]*model.Channel,
 	}
 
 	var r1 *model.AppError
-	if rf, ok := ret.Get(1).(func([]string) *model.AppError); ok {
+	if rf, ok := ret.Get(1).(func([]string, bool) *model.AppError); ok {
-		r1 = rf(channelIds)
+		r1 = rf(channelIds, includeDeleted)
 	} else {
 		if ret.Get(1) != nil {
 			r1 = ret.Get(1).(*model.AppError)

store/storetest/mocks/PostStore.go

@@ -792,6 +792,31 @@ func (_m *PostStore) Search(teamId string, userId string, params *model.SearchPa
 	return r0, r1
 }
 
+// SearchPostsInTeamForUser provides a mock function with given fields: paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage
+func (_m *PostStore) SearchPostsInTeamForUser(paramsList []*model.SearchParams, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, page int, perPage int) (*model.PostSearchResults, *model.AppError) {
+	ret := _m.Called(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage)
+
+	var r0 *model.PostSearchResults
+	if rf, ok := ret.Get(0).(func([]*model.SearchParams, string, string, bool, bool, int, int) *model.PostSearchResults); ok {
+		r0 = rf(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.PostSearchResults)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func([]*model.SearchParams, string, string, bool, bool, int, int) *model.AppError); ok {
+		r1 = rf(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
 // Update provides a mock function with given fields: newPost, oldPost
 func (_m *PostStore) Update(newPost *model.Post, oldPost *model.Post) (*model.Post, *model.AppError) {
 	ret := _m.Called(newPost, oldPost)

store/storetest/mocks/UserStore.go

@@ -84,6 +84,31 @@ func (_m *UserStore) AnalyticsGetSystemAdminCount() (int64, *model.AppError) {
 	return r0, r1
 }
 
+// AutocompleteUsersInChannel provides a mock function with given fields: teamId, channelId, term, options
+func (_m *UserStore) AutocompleteUsersInChannel(teamId string, channelId string, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) {
+	ret := _m.Called(teamId, channelId, term, options)
+
+	var r0 *model.UserAutocompleteInChannel
+	if rf, ok := ret.Get(0).(func(string, string, string, *model.UserSearchOptions) *model.UserAutocompleteInChannel); ok {
+		r0 = rf(teamId, channelId, term, options)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.UserAutocompleteInChannel)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(string, string, string, *model.UserSearchOptions) *model.AppError); ok {
+		r1 = rf(teamId, channelId, term, options)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
 // ClearAllCustomRoleAssignments provides a mock function with given fields:
 func (_m *UserStore) ClearAllCustomRoleAssignments() *model.AppError {
 	ret := _m.Called()

store/timer_layer.go

@@ -872,10 +872,10 @@ func (s *TimerLayerChannelStore) GetChannelsBatchForIndexing(startTime int64, en
 	return resultVar0, resultVar1
 }
 
-func (s *TimerLayerChannelStore) GetChannelsByIds(channelIds []string) ([]*model.Channel, *model.AppError) {
+func (s *TimerLayerChannelStore) GetChannelsByIds(channelIds []string, includeDeleted bool) ([]*model.Channel, *model.AppError) {
 	start := timemodule.Now()
 
-	resultVar0, resultVar1 := s.ChannelStore.GetChannelsByIds(channelIds)
+	resultVar0, resultVar1 := s.ChannelStore.GetChannelsByIds(channelIds, includeDeleted)
 
 	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
 	if s.Root.Metrics != nil {
@@ -4444,6 +4444,22 @@ func (s *TimerLayerPostStore) Search(teamId string, userId string, params *model
 	return resultVar0, resultVar1
 }
 
+func (s *TimerLayerPostStore) SearchPostsInTeamForUser(paramsList []*model.SearchParams, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, page int, perPage int) (*model.PostSearchResults, *model.AppError) {
+	start := timemodule.Now()
+
+	resultVar0, resultVar1 := s.PostStore.SearchPostsInTeamForUser(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage)
+
+	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if resultVar1 == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("PostStore.SearchPostsInTeamForUser", success, elapsed)
+	}
+	return resultVar0, resultVar1
+}
+
 func (s *TimerLayerPostStore) Update(newPost *model.Post, oldPost *model.Post) (*model.Post, *model.AppError) {
 	start := timemodule.Now()
 
@@ -6264,6 +6280,22 @@ func (s *TimerLayerUserStore) AnalyticsGetSystemAdminCount() (int64, *model.AppE
 	return resultVar0, resultVar1
 }
 
+func (s *TimerLayerUserStore) AutocompleteUsersInChannel(teamId string, channelId string, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) {
+	start := timemodule.Now()
+
+	resultVar0, resultVar1 := s.UserStore.AutocompleteUsersInChannel(teamId, channelId, term, options)
+
+	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if resultVar1 == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("UserStore.AutocompleteUsersInChannel", success, elapsed)
+	}
+	return resultVar0, resultVar1
+}
+
 func (s *TimerLayerUserStore) ClearAllCustomRoleAssignments() *model.AppError {
 	start := timemodule.Now()
 

testlib/helper.go

@@ -12,7 +12,9 @@ import (
 
 	"github.com/mattermost/mattermost-server/v5/mlog"
 	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/services/searchengine"
 	"github.com/mattermost/mattermost-server/v5/store"
+	"github.com/mattermost/mattermost-server/v5/store/searchlayer"
 	"github.com/mattermost/mattermost-server/v5/store/sqlstore"
 	"github.com/mattermost/mattermost-server/v5/store/storetest"
 	"github.com/mattermost/mattermost-server/v5/utils"
@@ -21,6 +23,7 @@ import (
 type MainHelper struct {
 	Settings         *model.SqlSettings
 	Store            store.Store
+	SearchEngine     *searchengine.Broker
 	SQLSupplier      *sqlstore.SqlSupplier
 	ClusterInterface *FakeClusterInterface
 
@@ -100,11 +103,15 @@ func (h *MainHelper) setupStore() {
 
 	h.Settings = storetest.MakeSqlSettings(driverName)
 
+	config := &model.Config{}
+	config.SetDefaults()
+
+	h.SearchEngine = searchengine.NewBroker(config, nil)
 	h.ClusterInterface = &FakeClusterInterface{}
 	h.SQLSupplier = sqlstore.NewSqlSupplier(*h.Settings, nil)
-	h.Store = &TestStore{
+	h.Store = searchlayer.NewSearchLayer(&TestStore{
 		h.SQLSupplier,
-	}
+	}, h.SearchEngine)
 }
 
 func (h *MainHelper) setupResources() {