mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

Adding the new search engine abstraction (#13304)

Просмотр исходного кода 
* WIP

* Adding bleve to go modules

* WIP

* Adding missing files from searchengine implementation

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* User and channel indexing and searches implemented

* Make bleve tests run with in-memory indexes

* Implement post index and deletion tests

* Initial commits for the search layer

* Removing unnecesary indexing

* WIP

* WIP

* More fixes for tests

* Adding the search layer

* Finishing the migration of searchers to the layer

* Removing unnecesary code

* Allowing multiple engines active at the same time

* WIP

* Add simple post search

* Print information when using bleve

* Adding some debugging to understand better how the searches are working

* Making more dynamic config of search engines

* Add post search basics

* Adding the Purge API endpoint

* Fixing bleve config updates

* Adding missed file

* Regenerating search engine mocks

* Adding missed v5 to modules imports

* fixing i18n

* Fixing some test around search engine

* Removing all bleve traces

* Cleaning up the vendors directory and go.mod/go.sum files

* Regenerating timer layer

* Adding properly the license

* Fixing govet shadow error

* Fixing some tests

* Fixing TestSearchPostsFromUser

* Fixing another test

* Fixing more tests

* Fixing more tests

* Removing SearchEngine redundant text from searchengine module code

* Fixing some reindexing problems in members updates

* Fixing tests

* Addressing PR comments

* Reverting go.mod and go.sum

* Addressing PR comments

* Fixing tests compilation

* Fixing govet

* Adding search engine stop method

* Being more explicit on where we use includeDeleted

* Adding GetSqlSupplier test helper method

* Mocking elasticsearch start function

* Fixing tests

Co-authored-by: Miguel de la Cruz <miguel@mcrx.me>
Co-authored-by: mattermod <mattermod@users.noreply.github.com>
Этот коммит содержится в:
Jesús Espino
2020-03-13 15:33:18 +01:00
коммит произвёл GitHub
родитель e2883bfe5f
Коммит c66e182b08
43 изменённых файлов: 1331 добавлений и 908 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

7
app/app.go
Просмотреть файл

@@ -16,6 +16,7 @@ import (
"github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/services/httpservice"
"github.com/mattermost/mattermost-server/v5/services/imageproxy"
"github.com/mattermost/mattermost-server/v5/services/searchengine"
"github.com/mattermost/mattermost-server/v5/services/timezones"
"github.com/mattermost/mattermost-server/v5/utils"
)
@@ -38,7 +39,7 @@ type App struct {
cluster einterfaces.ClusterInterface
compliance einterfaces.ComplianceInterface
dataRetention einterfaces.DataRetentionInterface
elasticsearch einterfaces.ElasticsearchInterface
searchEngine *searchengine.Broker
ldap einterfaces.LdapInterface
messageExport einterfaces.MessageExportInterface
metrics einterfaces.MetricsInterface
@@ -202,8 +203,8 @@ func (a *App) Compliance() einterfaces.ComplianceInterface {
func (a *App) DataRetention() einterfaces.DataRetentionInterface {
return a.dataRetention
}
func (a *App) Elasticsearch() einterfaces.ElasticsearchInterface {
return a.elasticsearch
func (a *App) SearchEngine() *searchengine.Broker {
return a.searchEngine
}
func (a *App) Ldap() einterfaces.LdapInterface {
return a.ldap

10
app/app_iface.go
Просмотреть файл

@@ -29,16 +29,13 @@ import (
"github.com/mattermost/mattermost-server/v5/services/filesstore"
"github.com/mattermost/mattermost-server/v5/services/httpservice"
"github.com/mattermost/mattermost-server/v5/services/imageproxy"
"github.com/mattermost/mattermost-server/v5/services/searchengine"
"github.com/mattermost/mattermost-server/v5/services/timezones"
"github.com/mattermost/mattermost-server/v5/store"
)
// AppIface is extracted from App struct and contains all it's exported methods. It's provided to allow partial interface passing and app layers creation.
type AppIface interface {
// GetViewUsersRestrictionsForTeam returns a list with the channel ids that the user has permissions to view on a
// team. If the result is an empty list, the user can't view any channel; if it's
// nil, there are no restrictions for the user in the specified team.
GetViewUsersRestrictionsForTeam(userId string, teamId string) ([]string, *model.AppError)
// @openTracingParams teamId
// previous ListCommands now ListAutocompleteCommands
ListAutocompleteCommands(teamId string, T goi18n.TranslateFunc) ([]*model.Command, *model.AppError)
@@ -413,7 +410,6 @@ type AppIface interface {
DoUploadFile(now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError)
DoUploadFileExpectModification(now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte) (*model.FileInfo, []byte, *model.AppError)
DownloadFromURL(downloadURL string) ([]byte, error)
Elasticsearch() einterfaces.ElasticsearchInterface
EnableUserAccessToken(token *model.UserAccessToken) *model.AppError
EnsureDiagnosticId()
EnvironmentConfig() map[string]interface{}
@@ -677,9 +673,6 @@ type AppIface interface {
InviteNewUsersToTeam(emailList []string, teamId, senderId string) *model.AppError
InviteNewUsersToTeamGracefully(emailList []string, teamId, senderId string) ([]*model.EmailInviteWithError, *model.AppError)
IpAddress() string
IsESAutocompletionEnabled() bool
IsESIndexingEnabled() bool
IsESSearchEnabled() bool
IsFirstUserAccount() bool
IsLeader() bool
IsPasswordValid(password string) *model.AppError
@@ -795,6 +788,7 @@ type AppIface interface {
SearchChannelsForUser(userId, teamId, term string) (*model.ChannelList, *model.AppError)
SearchChannelsUserNotIn(teamId string, userId string, term string) (*model.ChannelList, *model.AppError)
SearchEmoji(name string, prefixOnly bool, limit int) ([]*model.Emoji, *model.AppError)
SearchEngine() *searchengine.Broker
SearchGroupChannels(userId, term string) (*model.ChannelList, *model.AppError)
SearchPostsInTeam(teamId string, paramsList []*model.SearchParams) (*model.PostList, *model.AppError)
SearchPostsInTeamForUser(terms string, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, timeZoneOffset int, page, perPage int) (*model.PostSearchResults, *model.AppError)

148
app/channel.go
Просмотреть файл

@@ -112,14 +112,6 @@ func (a *App) JoinDefaultChannels(teamId string, user *model.User, shouldBeAdmin
}
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
if err = a.indexUser(user); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
}
})
}
return err
}
@@ -189,14 +181,6 @@ func (a *App) CreateChannelWithUser(channel *model.Channel, userId string) (*mod
message.Add("team_id", channel.TeamId)
a.Publish(message)
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
if err := a.indexUser(user); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
}
})
}
return rchannel, nil
}
@@ -267,23 +251,6 @@ func (a *App) CreateChannel(channel *model.Channel, addMember bool) (*model.Chan
})
}
if a.IsESIndexingEnabled() {
if sc.Type == model.CHANNEL_OPEN {
a.Srv().Go(func() {
if err := a.Elasticsearch().IndexChannel(sc); err != nil {
mlog.Error("Encountered error indexing channel", mlog.String("channel_id", sc.Id), mlog.Err(err))
}
})
}
if addMember {
a.Srv().Go(func() {
if err := a.indexUserFromId(channel.CreatorId); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", channel.CreatorId), mlog.Err(err))
}
})
}
}
return sc, nil
}
@@ -314,16 +281,6 @@ func (a *App) GetOrCreateDirectChannel(userId, otherUserId string) (*model.Chann
})
}
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
for _, id := range []string{userId, otherUserId} {
if indexUserErr := a.indexUserFromId(id); indexUserErr != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", id), mlog.Err(indexUserErr))
}
}
})
}
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_DIRECT_ADDED, "", channel.Id, "", nil)
message.Add("teammate_id", otherUserId)
a.Publish(message)
@@ -431,16 +388,6 @@ func (a *App) CreateGroupChannel(userIds []string, creatorId string) (*model.Cha
message.Add("teammate_ids", model.ArrayToJson(userIds))
a.Publish(message)
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
for _, id := range userIds {
if err := a.indexUserFromId(id); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", id), mlog.Err(err))
}
}
})
}
return channel, nil
}
@@ -536,14 +483,6 @@ func (a *App) UpdateChannel(channel *model.Channel) (*model.Channel, *model.AppE
messageWs.Add("channel", channel.ToJson())
a.Publish(messageWs)
if a.IsESIndexingEnabled() && channel.Type == model.CHANNEL_OPEN {
a.Srv().Go(func() {
if err := a.Elasticsearch().IndexChannel(channel); err != nil {
mlog.Error("Encountered error indexing channel", mlog.String("channel_id", channel.Id), mlog.Err(err))
}
})
}
return channel, nil
}
@@ -1243,14 +1182,6 @@ func (a *App) AddChannelMember(userId string, channel *model.Channel, userReques
})
}
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
if err := a.indexUser(user); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
}
})
}
if userRequestorId == "" || userId == userRequestorId {
a.postJoinChannelMessage(user, channel)
} else {
@@ -1639,14 +1570,6 @@ func (a *App) JoinChannel(channel *model.Channel, userId string) *model.AppError
})
}
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
if err := a.indexUser(user); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
}
})
}
if err := a.postJoinChannelMessage(user, channel); err != nil {
return err
}
@@ -1927,14 +1850,6 @@ func (a *App) removeUserFromChannel(userIdToRemove string, removerUserId string,
})
}
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
if err := a.indexUserFromId(userIdToRemove); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", userIdToRemove), mlog.Err(err))
}
})
}
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_USER_REMOVED, "", channel.Id, "", nil)
message.Add("user_id", userIdToRemove)
message.Add("remover_id", removerUserId)
@@ -2056,50 +1971,11 @@ func (a *App) MarkChannelAsUnreadFromPost(postID string, userID string) (*model.
return channelUnread, nil
}
func (a *App) esAutocompleteChannels(teamId, term string, includeDeleted bool) (*model.ChannelList, *model.AppError) {
channelIds, err := a.Elasticsearch().SearchChannels(teamId, term)
if err != nil {
return nil, err
}
channelList := model.ChannelList{}
if len(channelIds) > 0 {
channels, err := a.Srv().Store.Channel().GetChannelsByIds(channelIds)
if err != nil {
return nil, err
}
for _, c := range channels {
if c.DeleteAt > 0 && !includeDeleted {
continue
}
channelList = append(channelList, c)
}
}
return &channelList, nil
}
func (a *App) AutocompleteChannels(teamId string, term string) (*model.ChannelList, *model.AppError) {
includeDeleted := *a.Config().TeamSettings.ExperimentalViewArchivedChannels
var channelList *model.ChannelList
var err *model.AppError
term = strings.TrimSpace(term)
if a.IsESAutocompletionEnabled() {
channelList, err = a.esAutocompleteChannels(teamId, term, includeDeleted)
if err != nil {
mlog.Error("Encountered error on AutocompleteChannels through Elasticsearch. Falling back to default autocompletion.", mlog.Err(err))
}
}
if !a.IsESAutocompletionEnabled() || err != nil {
channelList, err = a.Srv().Store.Channel().AutocompleteInTeam(teamId, term, includeDeleted)
if err != nil {
return nil, err
}
}
return channelList, nil
return a.Srv().Store.Channel().AutocompleteInTeam(teamId, term, includeDeleted)
}
func (a *App) AutocompleteChannelsForSearch(teamId string, userId string, term string) (*model.ChannelList, *model.AppError) {
@@ -2246,11 +2122,6 @@ func (a *App) ViewChannel(view *model.ChannelView, userId string, currentSession
}
func (a *App) PermanentDeleteChannel(channel *model.Channel) *model.AppError {
profiles, err := a.Srv().Store.User().GetAllProfilesInChannel(channel.Id, false)
if err != nil {
return err
}
if err := a.Srv().Store.Post().PermanentDeleteByChannel(channel.Id); err != nil {
return err
}
@@ -2271,23 +2142,6 @@ func (a *App) PermanentDeleteChannel(channel *model.Channel) *model.AppError {
return err
}
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
for _, user := range profiles {
if err := a.indexUser(user); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
}
}
})
if channel.Type == model.CHANNEL_OPEN {
a.Srv().Go(func() {
if err := a.Elasticsearch().DeleteChannel(channel); err != nil {
mlog.Error("Encountered error deleting channel", mlog.String("channel_id", channel.Id), mlog.Err(err))
}
})
}
}
return nil
}

16
app/config.go
Просмотреть файл

@@ -406,22 +406,6 @@ func (a *App) SaveConfig(newCfg *model.Config, sendConfigChangeClusterMessage bo
return nil
}
func (a *App) IsESIndexingEnabled() bool {
return a.Elasticsearch() != nil && *a.Config().ElasticsearchSettings.EnableIndexing
}
func (a *App) IsESSearchEnabled() bool {
esInterface := a.Elasticsearch()
license := a.License()
return esInterface != nil && *a.Config().ElasticsearchSettings.EnableSearching && license != nil && *license.Features.Elasticsearch
}
func (a *App) IsESAutocompletionEnabled() bool {
esInterface := a.Elasticsearch()
license := a.License()
return esInterface != nil && *a.Config().ElasticsearchSettings.EnableAutocomplete && license != nil && *license.Features.Elasticsearch
}
func (a *App) HandleMessageExportConfig(cfg *model.Config, appCfg *model.Config) {
// If the Message Export feature has been toggled in the System Console, rewrite the ExportFromTimestamp field to an
// appropriate value. The rewriting occurs here to ensure it doesn't affect values written to the config file

4
app/config_test.go
Просмотреть файл

@@ -12,8 +12,6 @@ import (
"github.com/stretchr/testify/mock"
"github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/store/localcachelayer"
"github.com/mattermost/mattermost-server/v5/store/sqlstore"
"github.com/mattermost/mattermost-server/v5/store/storetest/mocks"
"github.com/mattermost/mattermost-server/v5/utils"
)
@@ -126,7 +124,7 @@ func TestEnsureInstallationDate(t *testing.T) {
for _, tc := range tt {
t.Run(tc.Name, func(t *testing.T) {
sqlStore := th.App.Srv().Store.User().(localcachelayer.LocalCacheUserStore).UserStore.(*sqlstore.SqlUserStore)
sqlStore := th.GetSqlSupplier()
sqlStore.GetMaster().Exec("DELETE FROM Users")
for _, createAt := range tc.UsersCreationDates {

6
app/diagnostics.go
Просмотреть файл

@@ -914,8 +914,10 @@ func (a *App) trackPermissions() {
func (a *App) trackElasticsearch() {
data := map[string]interface{}{}
if a.Elasticsearch() != nil && a.Elasticsearch().GetVersion() != 0 {
data["elasticsearch_server_version"] = a.Elasticsearch().GetVersion()
for _, engine := range a.SearchEngine().GetActiveEngines() {
if engine.GetVersion() != 0 && engine.GetName() == "elasticsearch" {
data["elasticsearch_server_version"] = engine.GetVersion()
}
}
a.SendDiagnostic(TRACK_ELASTICSEARCH, data)

12
app/enterprise.go
Просмотреть файл

@@ -9,6 +9,7 @@ import (
tjobs "github.com/mattermost/mattermost-server/v5/jobs/interfaces"
"github.com/mattermost/mattermost-server/v5/mlog"
"github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/services/searchengine"
)
var accountMigrationInterface func(*Server) einterfaces.AccountMigrationInterface
@@ -35,9 +36,9 @@ func RegisterDataRetentionInterface(f func(*App) einterfaces.DataRetentionInterf
dataRetentionInterface = f
}
var elasticsearchInterface func(*App) einterfaces.ElasticsearchInterface
var elasticsearchInterface func(*App) searchengine.SearchEngineInterface
func RegisterElasticsearchInterface(f func(*App) einterfaces.ElasticsearchInterface) {
func RegisterElasticsearchInterface(f func(*App) searchengine.SearchEngineInterface) {
elasticsearchInterface = f
}
@@ -129,9 +130,6 @@ func (s *Server) initEnterprise() {
if complianceInterface != nil {
s.Compliance = complianceInterface(s.FakeApp())
}
if elasticsearchInterface != nil {
s.Elasticsearch = elasticsearchInterface(s.FakeApp())
}
if ldapInterface != nil {
s.Ldap = ldapInterface(s.FakeApp())
}
@@ -161,4 +159,8 @@ func (s *Server) initEnterprise() {
if clusterInterface != nil {
s.Cluster = clusterInterface(s)
}
if elasticsearchInterface != nil {
s.SearchEngine.RegisterElasticsearchEngine(elasticsearchInterface(s.FakeApp()))
}
}

10
app/helper_test.go
Просмотреть файл

@@ -19,6 +19,7 @@ import (
"github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/store"
"github.com/mattermost/mattermost-server/v5/store/localcachelayer"
"github.com/mattermost/mattermost-server/v5/store/sqlstore"
"github.com/mattermost/mattermost-server/v5/store/storetest/mocks"
"github.com/mattermost/mattermost-server/v5/testlib"
"github.com/mattermost/mattermost-server/v5/utils"
@@ -94,6 +95,11 @@ func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer boo
}
th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = prevListenAddress })
th.App.Srv().SearchEngine = mainHelper.SearchEngine
th.App.Srv().Store.MarkSystemRanUnitTests()
th.App.UpdateConfig(func(cfg *model.Config) { *cfg.TeamSettings.EnableOpenServer = true })
// Disable strict password requirements for test
@@ -569,6 +575,10 @@ func (me *TestHelper) TearDown() {
}
}
func (me *TestHelper) GetSqlSupplier() *sqlstore.SqlSupplier {
return mainHelper.GetSQLSupplier()
}
func (me *TestHelper) ResetRoleMigration() {
sqlSupplier := mainHelper.GetSQLSupplier()
if _, err := sqlSupplier.GetMaster().Exec("DELETE from Roles"); err != nil {

9
app/layer_generators/opentracing_layer.go.tmpl
Просмотреть файл

@@ -31,7 +31,7 @@ type {{.Name}} struct {
cluster einterfaces.ClusterInterface
compliance einterfaces.ComplianceInterface
dataRetention einterfaces.DataRetentionInterface
elasticsearch einterfaces.ElasticsearchInterface
searchEngine *searchengine.Broker
ldap einterfaces.LdapInterface
messageExport einterfaces.MessageExportInterface
metrics einterfaces.MetricsInterface
@@ -99,7 +99,7 @@ func NewOpenTracingAppLayer(childApp AppIface, ctx context.Context) *{{.Name}} {
newApp.cluster = childApp.Cluster()
newApp.compliance = childApp.Compliance()
newApp.dataRetention = childApp.DataRetention()
newApp.elasticsearch = childApp.Elasticsearch()
newApp.searchEngine = childApp.SearchEngine()
newApp.ldap = childApp.Ldap()
newApp.messageExport = childApp.MessageExport()
newApp.metrics = childApp.Metrics()
@@ -156,9 +156,6 @@ func (a *{{.Name}}) Compliance() einterfaces.ComplianceInterface {
func (a *{{.Name}}) DataRetention() einterfaces.DataRetentionInterface {
return a.dataRetention
}
func (a *{{.Name}}) Elasticsearch() einterfaces.ElasticsearchInterface {
return a.elasticsearch
}
func (a *{{.Name}}) Ldap() einterfaces.LdapInterface {
return a.ldap
}
@@ -215,4 +212,4 @@ func (a *{{.Name}}) SetServer(srv *Server) {
}
func (a *{{.Name}}) GetT() goi18n.TranslateFunc {
return a.t
}
}

98
app/opentracing_layer.go
Просмотреть файл

@@ -29,6 +29,7 @@ import (
"github.com/mattermost/mattermost-server/v5/services/filesstore"
"github.com/mattermost/mattermost-server/v5/services/httpservice"
"github.com/mattermost/mattermost-server/v5/services/imageproxy"
"github.com/mattermost/mattermost-server/v5/services/searchengine"
"github.com/mattermost/mattermost-server/v5/services/timezones"
"github.com/mattermost/mattermost-server/v5/services/tracing"
"github.com/mattermost/mattermost-server/v5/store"
@@ -56,7 +57,7 @@ type OpenTracingAppLayer struct {
cluster einterfaces.ClusterInterface
compliance einterfaces.ComplianceInterface
dataRetention einterfaces.DataRetentionInterface
elasticsearch einterfaces.ElasticsearchInterface
searchEngine *searchengine.Broker
ldap einterfaces.LdapInterface
messageExport einterfaces.MessageExportInterface
metrics einterfaces.MetricsInterface
@@ -8588,28 +8589,6 @@ func (a *OpenTracingAppLayer) GetViewUsersRestrictions(userId string) (*model.Vi
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) GetViewUsersRestrictionsForTeam(userId string, teamId string) ([]string, *model.AppError) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetViewUsersRestrictionsForTeam")
a.ctx = newCtx
a.app.Srv().Store.SetContext(newCtx)
defer func() {
a.app.Srv().Store.SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0, resultVar1 := a.app.GetViewUsersRestrictionsForTeam(userId, teamId)
if resultVar1 != nil {
span.LogFields(spanlog.Error(resultVar1))
ext.Error.Set(span, true)
}
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) HTMLTemplates() *template.Template {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.HTMLTemplates")
@@ -9229,57 +9208,6 @@ func (a *OpenTracingAppLayer) InviteNewUsersToTeamGracefully(emailList []string,
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) IsESAutocompletionEnabled() bool {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.IsESAutocompletionEnabled")
a.ctx = newCtx
a.app.Srv().Store.SetContext(newCtx)
defer func() {
a.app.Srv().Store.SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0 := a.app.IsESAutocompletionEnabled()
return resultVar0
}
func (a *OpenTracingAppLayer) IsESIndexingEnabled() bool {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.IsESIndexingEnabled")
a.ctx = newCtx
a.app.Srv().Store.SetContext(newCtx)
defer func() {
a.app.Srv().Store.SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0 := a.app.IsESIndexingEnabled()
return resultVar0
}
func (a *OpenTracingAppLayer) IsESSearchEnabled() bool {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.IsESSearchEnabled")
a.ctx = newCtx
a.app.Srv().Store.SetContext(newCtx)
defer func() {
a.app.Srv().Store.SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0 := a.app.IsESSearchEnabled()
return resultVar0
}
func (a *OpenTracingAppLayer) IsFirstUserAccount() bool {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.IsFirstUserAccount")
@@ -11790,6 +11718,23 @@ func (a *OpenTracingAppLayer) SearchEmoji(name string, prefixOnly bool, limit in
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) SearchEngine() *searchengine.Broker {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SearchEngine")
a.ctx = newCtx
a.app.Srv().Store.SetContext(newCtx)
defer func() {
a.app.Srv().Store.SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0 := a.app.SearchEngine()
return resultVar0
}
func (a *OpenTracingAppLayer) SearchGroupChannels(userId string, term string) (*model.ChannelList, *model.AppError) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SearchGroupChannels")
@@ -14914,7 +14859,7 @@ func NewOpenTracingAppLayer(childApp AppIface, ctx context.Context) *OpenTracing
newApp.cluster = childApp.Cluster()
newApp.compliance = childApp.Compliance()
newApp.dataRetention = childApp.DataRetention()
newApp.elasticsearch = childApp.Elasticsearch()
newApp.searchEngine = childApp.SearchEngine()
newApp.ldap = childApp.Ldap()
newApp.messageExport = childApp.MessageExport()
newApp.metrics = childApp.Metrics()
@@ -14970,9 +14915,6 @@ func (a *OpenTracingAppLayer) Compliance() einterfaces.ComplianceInterface {
func (a *OpenTracingAppLayer) DataRetention() einterfaces.DataRetentionInterface {
return a.dataRetention
}
func (a *OpenTracingAppLayer) Elasticsearch() einterfaces.ElasticsearchInterface {
return a.elasticsearch
}
func (a *OpenTracingAppLayer) Ldap() einterfaces.LdapInterface {
return a.ldap
}

6
app/options.go
Просмотреть файл

@@ -78,8 +78,8 @@ func StartMetrics(s *Server) error {
return nil
}
func StartElasticsearch(s *Server) error {
s.startElasticsearch = true
func StartSearchEngine(s *Server) error {
s.startSearchEngine = true
return nil
}
@@ -104,7 +104,7 @@ func ServerConnector(s *Server) AppOption {
a.cluster = s.Cluster
a.compliance = s.Compliance
a.dataRetention = s.DataRetention
a.elasticsearch = s.Elasticsearch
a.searchEngine = s.SearchEngine
a.ldap = s.Ldap
a.messageExport = s.MessageExport
a.metrics = s.Metrics

122
app/post.go
Просмотреть файл

@@ -297,14 +297,6 @@ func (a *App) CreatePost(post *model.Post, channel *model.Channel, triggerWebhoo
})
}
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
if err = a.Elasticsearch().IndexPost(rpost, channel.TeamId); err != nil {
mlog.Error("Encountered error indexing post", mlog.String("post_id", post.Id), mlog.Err(err))
}
})
}
if a.Metrics() != nil {
a.Metrics().IncrementPostCreate()
}
@@ -585,19 +577,6 @@ func (a *App) UpdatePost(post *model.Post, safeUpdate bool) (*model.Post, *model
})
}
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
channel, chanErr := a.Srv().Store.Channel().GetForPost(rpost.Id)
if chanErr != nil {
mlog.Error("Couldn't get channel for post for Elasticsearch indexing.", mlog.String("channel_id", rpost.ChannelId), mlog.String("post_id", rpost.Id))
return
}
if err := a.Elasticsearch().IndexPost(rpost, channel.TeamId); err != nil {
mlog.Error("Encountered error indexing post", mlog.String("post_id", post.Id), mlog.Err(err))
}
})
}
rpost = a.PreparePostForClient(rpost, false, true)
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_EDITED, "", rpost.ChannelId, "", nil)
@@ -870,14 +849,6 @@ func (a *App) DeletePost(postId, deleteByID string) (*model.Post, *model.AppErro
a.DeleteFlaggedPosts(post.Id)
})
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
if err := a.Elasticsearch().DeletePost(post); err != nil {
mlog.Error("Encountered error deleting post", mlog.String("post_id", post.Id), mlog.Err(err))
}
})
}
a.invalidateCacheForChannelPosts(post.ChannelId)
return post, nil
@@ -1006,10 +977,18 @@ func (a *App) SearchPostsInTeam(teamId string, paramsList []*model.SearchParams)
})
}
func (a *App) esSearchPostsInTeamForUser(paramsList []*model.SearchParams, userId, teamId string, isOrSearch, includeDeletedChannels bool, page, perPage int) (*model.PostSearchResults, *model.AppError) {
finalParamsList := []*model.SearchParams{}
func (a *App) SearchPostsInTeamForUser(terms string, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, timeZoneOffset int, page, perPage int) (*model.PostSearchResults, *model.AppError) {
var postSearchResults *model.PostSearchResults
var err *model.AppError
paramsList := model.ParseSearchParams(strings.TrimSpace(terms), timeZoneOffset)
includeDeleted := includeDeletedChannels && *a.Config().TeamSettings.ExperimentalViewArchivedChannels
if !*a.Config().ServiceSettings.EnablePostSearch {
return nil, model.NewAppError("SearchPostsInTeamForUser", "store.sql_post.search.disabled", nil, fmt.Sprintf("teamId=%v userId=%v", teamId, userId), http.StatusNotImplemented)
}
finalParamsList := []*model.SearchParams{}
for _, params := range paramsList {
params.OrTerms = isOrSearch
// Don't allow users to search for "*"
@@ -1031,90 +1010,11 @@ func (a *App) esSearchPostsInTeamForUser(paramsList []*model.SearchParams, userI
return model.MakePostSearchResults(model.NewPostList(), nil), nil
}
// We only allow the user to search in channels they are a member of.
userChannels, err := a.GetChannelsForUser(teamId, userId, includeDeleted)
if err != nil {
mlog.Error("error getting channel for user", mlog.Err(err))
return nil, err
}
postIds, matches, err := a.Elasticsearch().SearchPosts(userChannels, finalParamsList, page, perPage)
postSearchResults, err = a.Srv().Store.Post().SearchPostsInTeamForUser(finalParamsList, userId, teamId, isOrSearch, includeDeleted, page, perPage)
if err != nil {
return nil, err
}
// Get the posts
postList := model.NewPostList()
if len(postIds) > 0 {
posts, err := a.Srv().Store.Post().GetPostsByIds(postIds)
if err != nil {
return nil, err
}
for _, p := range posts {
if p.DeleteAt == 0 {
postList.AddPost(p)
postList.AddOrder(p.Id)
}
}
}
return model.MakePostSearchResults(postList, matches), nil
}
func (a *App) SearchPostsInTeamForUser(terms string, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, timeZoneOffset int, page, perPage int) (*model.PostSearchResults, *model.AppError) {
var postSearchResults *model.PostSearchResults
var err *model.AppError
paramsList := model.ParseSearchParams(strings.TrimSpace(terms), timeZoneOffset)
if !*a.Config().ServiceSettings.EnablePostSearch {
return nil, model.NewAppError("SearchPostsInTeamForUser", "store.sql_post.search.disabled", nil, fmt.Sprintf("teamId=%v userId=%v", teamId, userId), http.StatusNotImplemented)
}
if a.IsESSearchEnabled() {
postSearchResults, err = a.esSearchPostsInTeamForUser(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage)
if err != nil {
mlog.Error("Encountered error on SearchPostsInTeamForUser through Elasticsearch. Falling back to default search.", mlog.Err(err))
}
}
if !a.IsESSearchEnabled() || err != nil {
// Since we don't support paging for DB search, we just return nothing for later pages
if page > 0 {
return model.MakePostSearchResults(model.NewPostList(), nil), nil
}
includeDeleted := includeDeletedChannels && *a.Config().TeamSettings.ExperimentalViewArchivedChannels
posts, err := a.searchPostsInTeam(teamId, userId, paramsList, func(params *model.SearchParams) {
params.IncludeDeletedChannels = includeDeleted
params.OrTerms = isOrSearch
for idx, channelName := range params.InChannels {
if strings.HasPrefix(channelName, "@") {
channel, err := a.parseAndFetchChannelIdByNameFromInFilter(channelName, userId, teamId, includeDeletedChannels)
if err != nil {
mlog.Error("error getting channel_id by name from in filter", mlog.Err(err))
continue
}
params.InChannels[idx] = channel.Name
}
}
for idx, channelName := range params.ExcludedChannels {
if strings.HasPrefix(channelName, "@") {
channel, err := a.parseAndFetchChannelIdByNameFromInFilter(channelName, userId, teamId, includeDeletedChannels)
if err != nil {
mlog.Error("error getting channel_id by name from in filter", mlog.Err(err))
continue
}
params.ExcludedChannels[idx] = channel.Name
}
}
})
if err != nil {
return nil, err
}
postSearchResults = model.MakePostSearchResults(posts, nil)
}
return postSearchResults, nil
}

46
app/post_test.go
Просмотреть файл

@@ -13,9 +13,9 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost-server/v5/einterfaces/mocks"
"github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/plugin/plugintest/mock"
"github.com/mattermost/mattermost-server/v5/services/searchengine/mocks"
"github.com/mattermost/mattermost-server/v5/store/storetest"
storemocks "github.com/mattermost/mattermost-server/v5/store/storetest/mocks"
)
@@ -944,9 +944,16 @@ func TestSearchPostsInTeamForUser(t *testing.T) {
posts[2].Id,
}
es := &mocks.ElasticsearchInterface{}
es := &mocks.SearchEngineInterface{}
es.On("SearchPosts", mock.Anything, mock.Anything, page, perPage).Return(resultsPage, nil, nil)
th.App.elasticsearch = es
es.On("GetName").Return("mock")
es.On("Start").Return(nil).Maybe()
es.On("IsActive").Return(true)
es.On("IsSearchEnabled").Return(true)
th.App.Srv().SearchEngine.ElasticsearchEngine = es
defer func() {
th.App.Srv().SearchEngine.ElasticsearchEngine = nil
}()
results, err := th.App.SearchPostsInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage)
@@ -965,9 +972,16 @@ func TestSearchPostsInTeamForUser(t *testing.T) {
posts[0].Id,
}
es := &mocks.ElasticsearchInterface{}
es := &mocks.SearchEngineInterface{}
es.On("SearchPosts", mock.Anything, mock.Anything, page, perPage).Return(resultsPage, nil, nil)
th.App.elasticsearch = es
es.On("GetName").Return("mock")
es.On("Start").Return(nil).Maybe()
es.On("IsActive").Return(true)
es.On("IsSearchEnabled").Return(true)
th.App.Srv().SearchEngine.ElasticsearchEngine = es
defer func() {
th.App.Srv().SearchEngine.ElasticsearchEngine = nil
}()
results, err := th.App.SearchPostsInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage)
@@ -982,9 +996,16 @@ func TestSearchPostsInTeamForUser(t *testing.T) {
page := 0
es := &mocks.ElasticsearchInterface{}
es := &mocks.SearchEngineInterface{}
es.On("SearchPosts", mock.Anything, mock.Anything, page, perPage).Return(nil, nil, &model.AppError{})
th.App.elasticsearch = es
es.On("GetName").Return("mock")
es.On("Start").Return(nil).Maybe()
es.On("IsActive").Return(true)
es.On("IsSearchEnabled").Return(true)
th.App.Srv().SearchEngine.ElasticsearchEngine = es
defer func() {
th.App.Srv().SearchEngine.ElasticsearchEngine = nil
}()
results, err := th.App.SearchPostsInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage)
@@ -1007,9 +1028,16 @@ func TestSearchPostsInTeamForUser(t *testing.T) {
page := 1
es := &mocks.ElasticsearchInterface{}
es := &mocks.SearchEngineInterface{}
es.On("SearchPosts", mock.Anything, mock.Anything, page, perPage).Return(nil, nil, &model.AppError{})
th.App.elasticsearch = es
es.On("GetName").Return("mock")
es.On("Start").Return(nil).Maybe()
es.On("IsActive").Return(true)
es.On("IsSearchEnabled").Return(true)
th.App.Srv().SearchEngine.ElasticsearchEngine = es
defer func() {
th.App.Srv().SearchEngine.ElasticsearchEngine = nil
}()
results, err := th.App.SearchPostsInTeamForUser(searchTerm, th.BasicUser.Id, th.BasicTeam.Id, false, false, 0, page, perPage)

12
app/elasticsearch.go → app/searchengine.go
Просмотреть файл

@@ -18,12 +18,12 @@ func (a *App) TestElasticsearch(cfg *model.Config) *model.AppError {
}
}
esI := a.Elasticsearch()
if esI == nil {
seI := a.SearchEngine().ElasticsearchEngine
if seI == nil {
err := model.NewAppError("TestElasticsearch", "ent.elasticsearch.test_config.license.error", nil, "", http.StatusNotImplemented)
return err
}
if err := esI.TestConfig(cfg); err != nil {
if err := seI.TestConfig(cfg); err != nil {
return err
}
@@ -31,13 +31,13 @@ func (a *App) TestElasticsearch(cfg *model.Config) *model.AppError {
}
func (a *App) PurgeElasticsearchIndexes() *model.AppError {
esI := a.Elasticsearch()
if esI == nil {
seI := a.SearchEngine().ElasticsearchEngine
if seI == nil {
err := model.NewAppError("PurgeElasticsearchIndexes", "ent.elasticsearch.test_config.license.error", nil, "", http.StatusNotImplemented)
return err
}
if err := esI.PurgeIndexes(); err != nil {
if err := seI.PurgeIndexes(); err != nil {
return err
}

98
app/server.go
Просмотреть файл

@@ -35,6 +35,7 @@ import (
"github.com/mattermost/mattermost-server/v5/services/cache/lru"
"github.com/mattermost/mattermost-server/v5/services/httpservice"
"github.com/mattermost/mattermost-server/v5/services/imageproxy"
"github.com/mattermost/mattermost-server/v5/services/searchengine"
"github.com/mattermost/mattermost-server/v5/services/timezones"
"github.com/mattermost/mattermost-server/v5/services/tracing"
"github.com/mattermost/mattermost-server/v5/store"
@@ -98,6 +99,8 @@ type Server struct {
licenseListenerId string
logListenerId string
clusterLeaderListenerId string
searchConfigListenerId string
searchLicenseListenerId string
configStore config.Store
asymmetricSigningKey *ecdsa.PrivateKey
postActionCookieSecret []byte
@@ -122,15 +125,16 @@ type Server struct {
Log *mlog.Logger
NotificationsLog *mlog.Logger
joinCluster bool
startMetrics bool
startElasticsearch bool
joinCluster bool
startMetrics bool
startSearchEngine bool
SearchEngine *searchengine.Broker
AccountMigration einterfaces.AccountMigrationInterface
Cluster einterfaces.ClusterInterface
Compliance einterfaces.ComplianceInterface
DataRetention einterfaces.DataRetentionInterface
Elasticsearch einterfaces.ElasticsearchInterface
Ldap einterfaces.LdapInterface
MessageExport einterfaces.MessageExportInterface
Metrics einterfaces.MetricsInterface
@@ -206,6 +210,8 @@ func NewServer(options ...Option) (*Server, error) {
return nil, errors.Wrapf(err, "unable to load Mattermost translation files")
}
s.SearchEngine = searchengine.NewBroker(s.Config(), s.Jobs)
// at the moment we only have this implementation
// in the future the cache provider will be built based on the loaded config
s.CacheProvider = new(lru.CacheProvider)
@@ -304,10 +310,6 @@ func NewServer(options ...Option) (*Server, error) {
s.Metrics.StartServer()
}
if s.startElasticsearch && s.Elasticsearch != nil {
s.StartElasticsearch()
}
s.AddConfigListener(func(oldConfig *model.Config, newConfig *model.Config) {
if *oldConfig.GuestAccountsSettings.Enable && !*newConfig.GuestAccountsSettings.Enable {
if appErr := s.FakeApp().DeactivateGuests(); appErr != nil {
@@ -354,6 +356,11 @@ func NewServer(options ...Option) (*Server, error) {
}
}
s.SearchEngine.UpdateConfig(s.Config())
searchConfigListenerId, searchLicenseListenerId := s.StartSearchEngine()
s.searchConfigListenerId = searchConfigListenerId
s.searchLicenseListenerId = searchLicenseListenerId
return s, nil
}
@@ -413,6 +420,7 @@ func (s *Server) Shutdown() error {
s.RemoveConfigListener(s.configListenerId)
s.RemoveConfigListener(s.logListenerId)
s.stopSearchEngine()
s.Audit.Shutdown()
@@ -769,33 +777,40 @@ func doSessionCleanup(s *Server) {
s.Store.Session().Cleanup(model.GetMillis(), SESSIONS_CLEANUP_BATCH_SIZE)
}
func (s *Server) StartElasticsearch() {
s.Go(func() {
if err := s.Elasticsearch.Start(); err != nil {
s.Log.Error(err.Error())
}
})
func (s *Server) StartSearchEngine() (string, string) {
if s.SearchEngine.ElasticsearchEngine != nil && s.SearchEngine.ElasticsearchEngine.IsActive() {
s.Go(func() {
if err := s.SearchEngine.ElasticsearchEngine.Start(); err != nil {
s.Log.Error(err.Error())
}
})
}
s.AddConfigListener(func(oldConfig *model.Config, newConfig *model.Config) {
if !*oldConfig.ElasticsearchSettings.EnableIndexing && *newConfig.ElasticsearchSettings.EnableIndexing {
configListenerId := s.AddConfigListener(func(oldConfig *model.Config, newConfig *model.Config) {
if s.SearchEngine == nil {
return
}
s.SearchEngine.UpdateConfig(newConfig)
if s.SearchEngine.ElasticsearchEngine != nil && !*oldConfig.ElasticsearchSettings.EnableIndexing && *newConfig.ElasticsearchSettings.EnableIndexing {
s.Go(func() {
if err := s.Elasticsearch.Start(); err != nil {
if err := s.SearchEngine.ElasticsearchEngine.Start(); err != nil {
mlog.Error(err.Error())
}
})
} else if *oldConfig.ElasticsearchSettings.EnableIndexing && !*newConfig.ElasticsearchSettings.EnableIndexing {
} else if s.SearchEngine.ElasticsearchEngine != nil && *oldConfig.ElasticsearchSettings.EnableIndexing && !*newConfig.ElasticsearchSettings.EnableIndexing {
s.Go(func() {
if err := s.Elasticsearch.Stop(); err != nil {
if err := s.SearchEngine.ElasticsearchEngine.Stop(); err != nil {
mlog.Error(err.Error())
}
})
} else if *oldConfig.ElasticsearchSettings.Password != *newConfig.ElasticsearchSettings.Password || *oldConfig.ElasticsearchSettings.Username != *newConfig.ElasticsearchSettings.Username || *oldConfig.ElasticsearchSettings.ConnectionUrl != *newConfig.ElasticsearchSettings.ConnectionUrl || *oldConfig.ElasticsearchSettings.Sniff != *newConfig.ElasticsearchSettings.Sniff {
} else if s.SearchEngine.ElasticsearchEngine != nil && *oldConfig.ElasticsearchSettings.Password != *newConfig.ElasticsearchSettings.Password || *oldConfig.ElasticsearchSettings.Username != *newConfig.ElasticsearchSettings.Username || *oldConfig.ElasticsearchSettings.ConnectionUrl != *newConfig.ElasticsearchSettings.ConnectionUrl || *oldConfig.ElasticsearchSettings.Sniff != *newConfig.ElasticsearchSettings.Sniff {
s.Go(func() {
if *oldConfig.ElasticsearchSettings.EnableIndexing {
if err := s.Elasticsearch.Stop(); err != nil {
if err := s.SearchEngine.ElasticsearchEngine.Stop(); err != nil {
mlog.Error(err.Error())
}
if err := s.Elasticsearch.Start(); err != nil {
if err := s.SearchEngine.ElasticsearchEngine.Start(); err != nil {
mlog.Error(err.Error())
}
}
@@ -803,21 +818,38 @@ func (s *Server) StartElasticsearch() {
}
})
s.AddLicenseListener(func(oldLicense, newLicense *model.License) {
licenseListenerId := s.AddLicenseListener(func(oldLicense, newLicense *model.License) {
if s.SearchEngine == nil {
return
}
if oldLicense == nil && newLicense != nil {
s.Go(func() {
if err := s.Elasticsearch.Start(); err != nil {
mlog.Error(err.Error())
}
})
if s.SearchEngine.ElasticsearchEngine != nil && s.SearchEngine.ElasticsearchEngine.IsActive() {
s.Go(func() {
if err := s.SearchEngine.ElasticsearchEngine.Start(); err != nil {
mlog.Error(err.Error())
}
})
}
} else if oldLicense != nil && newLicense == nil {
s.Go(func() {
if err := s.Elasticsearch.Stop(); err != nil {
mlog.Error(err.Error())
}
})
if s.SearchEngine.ElasticsearchEngine != nil {
s.Go(func() {
if err := s.SearchEngine.ElasticsearchEngine.Stop(); err != nil {
mlog.Error(err.Error())
}
})
}
}
})
return configListenerId, licenseListenerId
}
func (s *Server) stopSearchEngine() {
s.RemoveConfigListener(s.searchConfigListenerId)
s.RemoveLicenseListener(s.searchLicenseListenerId)
if s.SearchEngine != nil && s.SearchEngine.ElasticsearchEngine != nil && s.SearchEngine.ElasticsearchEngine.IsActive() {
s.SearchEngine.ElasticsearchEngine.Stop()
}
}
func (s *Server) initDiagnostics(endpoint string) {

16
app/server_app_adapters.go
Просмотреть файл

@@ -13,6 +13,7 @@ import (
"github.com/mattermost/mattermost-server/v5/services/mailservice"
"github.com/mattermost/mattermost-server/v5/store"
"github.com/mattermost/mattermost-server/v5/store/localcachelayer"
"github.com/mattermost/mattermost-server/v5/store/searchlayer"
"github.com/mattermost/mattermost-server/v5/store/sqlstore"
"github.com/mattermost/mattermost-server/v5/utils"
"github.com/pkg/errors"
@@ -61,10 +62,17 @@ func (s *Server) RunOldAppInitialization() error {
if s.FakeApp().Srv().newStore == nil {
s.FakeApp().Srv().newStore = func() store.Store {
return store.NewTimerLayer(
localcachelayer.NewLocalCacheLayer(
sqlstore.NewSqlSupplier(s.FakeApp().Config().SqlSettings, s.Metrics),
s.Metrics, s.Cluster, s.CacheProvider),
s.Metrics)
searchlayer.NewSearchLayer(
localcachelayer.NewLocalCacheLayer(
sqlstore.NewSqlSupplier(s.FakeApp().Config().SqlSettings, s.Metrics),
s.Metrics,
s.Cluster,
s.CacheProvider,
),
s.SearchEngine,
),
s.Metrics,
)
}
}

13
app/team.go
Просмотреть файл

@@ -489,7 +489,7 @@ func (a *App) AddUserToTeamByToken(userId string, tokenId string) (*model.Team,
}
if token.Type == TOKEN_TYPE_GUEST_INVITATION {
channels, err := a.Srv().Store.Channel().GetChannelsByIds(strings.Split(tokenData["channels"], " "))
channels, err := a.Srv().Store.Channel().GetChannelsByIds(strings.Split(tokenData["channels"], " "), false)
if err != nil {
return nil, err
}
@@ -944,15 +944,6 @@ func (a *App) RemoveTeamMemberFromTeam(teamMember *model.TeamMember, requestorId
})
}
esInterface := a.Elasticsearch()
if esInterface != nil && *a.Config().ElasticsearchSettings.EnableIndexing {
a.Srv().Go(func() {
if err := a.indexUser(user); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
}
})
}
if _, err := a.Srv().Store.User().UpdateUpdateAt(user.Id); err != nil {
return err
}
@@ -1135,7 +1126,7 @@ func (a *App) prepareInviteGuestsToChannels(teamId string, guestsInvite *model.G
}()
cchan := make(chan store.StoreResult, 1)
go func() {
channels, err := a.Srv().Store.Channel().GetChannelsByIds(guestsInvite.Channels)
channels, err := a.Srv().Store.Channel().GetChannelsByIds(guestsInvite.Channels, false)
cchan <- store.StoreResult{Data: channels, Err: err}
close(cchan)
}()

333
app/user.go
Просмотреть файл

@@ -67,7 +67,7 @@ func (a *App) CreateUserWithToken(user *model.User, token *model.Token) (*model.
return nil, err
}
channels, err := a.Srv().Store.Channel().GetChannelsByIds(strings.Split(tokenData["channels"], " "))
channels, err := a.Srv().Store.Channel().GetChannelsByIds(strings.Split(tokenData["channels"], " "), false)
if err != nil {
return nil, err
}
@@ -201,40 +201,6 @@ func (a *App) IsFirstUserAccount() bool {
return false
}
// indexUser fetches the required information to index a user from the database and
// calls the elasticsearch interface method
func (a *App) indexUser(user *model.User) *model.AppError {
userTeams, err := a.Srv().Store.Team().GetTeamsByUserId(user.Id)
if err != nil {
return err
}
userTeamsIds := []string{}
for _, team := range userTeams {
userTeamsIds = append(userTeamsIds, team.Id)
}
userChannelMembers, err := a.Srv().Store.Channel().GetAllChannelMembersForUser(user.Id, false, true)
if err != nil {
return err
}
userChannelsIds := []string{}
for channelId := range userChannelMembers {
userChannelsIds = append(userChannelsIds, channelId)
}
return a.Elasticsearch().IndexUser(user, userTeamsIds, userChannelsIds)
}
func (a *App) indexUserFromId(userId string) *model.AppError {
user, err := a.GetUser(userId)
if err != nil {
return err
}
return a.indexUser(user)
}
// CreateUser creates a user and sets several fields of the returned User struct to
// their zero values.
func (a *App) CreateUser(user *model.User) (*model.User, *model.AppError) {
@@ -294,14 +260,6 @@ func (a *App) createUserOrGuest(user *model.User, guest bool) (*model.User, *mod
})
}
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
if err := a.indexUser(user); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
}
})
}
return ruser, nil
}
@@ -1193,14 +1151,6 @@ func (a *App) UpdateUser(user *model.User, sendNotifications bool) (*model.User,
a.InvalidateCacheForUser(user.Id)
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
if err := a.indexUser(user); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
}
})
}
return userUpdate.New, nil
}
@@ -1554,14 +1504,6 @@ func (a *App) PermanentDeleteUser(user *model.User) *model.AppError {
mlog.Warn("Permanently deleted account", mlog.String("user_email", user.Email), mlog.String("user_id", user.Id))
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
if err := a.Elasticsearch().DeleteUser(user); err != nil {
mlog.Error("Encountered error deleting user", mlog.String("user_id", user.Id), mlog.Err(err))
}
})
}
return nil
}
@@ -1743,21 +1685,12 @@ func (a *App) SearchUsersNotInChannel(teamId string, channelId string, term stri
return users, nil
}
func (a *App) esSearchUsersInTeam(teamId, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError) {
listOfAllowedChannels, err := a.GetViewUsersRestrictionsForTeam(a.Session().UserId, teamId)
if err != nil {
return nil, err
}
if listOfAllowedChannels != nil && len(listOfAllowedChannels) == 0 {
return []*model.User{}, nil
}
func (a *App) SearchUsersInTeam(teamId, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError) {
var users []*model.User
var err *model.AppError
term = strings.TrimSpace(term)
usersIds, err := a.Elasticsearch().SearchUsersInTeam(teamId, listOfAllowedChannels, term, options)
if err != nil {
return nil, err
}
users, err := a.Srv().Store.User().GetProfileByIds(usersIds, nil, false)
users, err = a.Srv().Store.User().Search(teamId, term, options)
if err != nil {
return nil, err
}
@@ -1769,32 +1702,6 @@ func (a *App) esSearchUsersInTeam(teamId, term string, options *model.UserSearch
return users, nil
}
func (a *App) SearchUsersInTeam(teamId, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError) {
var users []*model.User
var err *model.AppError
term = strings.TrimSpace(term)
if a.IsESAutocompletionEnabled() {
users, err = a.esSearchUsersInTeam(teamId, term, options)
if err != nil {
mlog.Error("Encountered error on SearchUsersInTeam through Elasticsearch. Falling back to default search.", mlog.Err(err))
}
}
if !a.IsESAutocompletionEnabled() || err != nil {
users, err = a.Srv().Store.User().Search(teamId, term, options)
if err != nil {
return nil, err
}
for _, user := range users {
a.SanitizeProfile(user, options.IsAdmin)
}
}
return users, nil
}
func (a *App) SearchUsersNotInTeam(notInTeamId string, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError) {
term = strings.TrimSpace(term)
users, err := a.Srv().Store.User().SearchNotInTeam(notInTeamId, term, options)
@@ -1823,141 +1730,31 @@ func (a *App) SearchUsersWithoutTeam(term string, options *model.UserSearchOptio
return users, nil
}
func (a *App) esAutocompleteUsersInChannel(teamId, channelId, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) {
listOfAllowedChannels, err := a.getListOfAllowedChannelsForTeam(teamId, options.ViewRestrictions)
if err != nil {
return nil, err
}
if len(listOfAllowedChannels) == 0 {
return &model.UserAutocompleteInChannel{}, nil
}
uchanIds := []string{}
nuchanIds := []string{}
if !strings.Contains(strings.Join(listOfAllowedChannels, "."), channelId) {
nuchanIds, err = a.Elasticsearch().SearchUsersInTeam(teamId, listOfAllowedChannels, term, options)
} else {
uchanIds, nuchanIds, err = a.Elasticsearch().SearchUsersInChannel(teamId, channelId, listOfAllowedChannels, term, options)
}
if err != nil {
return nil, err
}
uchan := make(chan store.StoreResult, 1)
go func() {
users, err := a.Srv().Store.User().GetProfileByIds(uchanIds, nil, false)
uchan <- store.StoreResult{Data: users, Err: err}
close(uchan)
}()
nuchan := make(chan store.StoreResult, 1)
go func() {
users, err := a.Srv().Store.User().GetProfileByIds(nuchanIds, nil, false)
nuchan <- store.StoreResult{Data: users, Err: err}
close(nuchan)
}()
autocomplete := &model.UserAutocompleteInChannel{}
result := <-uchan
if result.Err != nil {
return nil, result.Err
}
users := result.Data.([]*model.User)
for _, user := range users {
a.SanitizeProfile(user, options.IsAdmin)
}
autocomplete.InChannel = users
result = <-nuchan
if result.Err != nil {
return nil, result.Err
}
users = result.Data.([]*model.User)
for _, user := range users {
a.SanitizeProfile(user, options.IsAdmin)
}
autocomplete.OutOfChannel = users
return autocomplete, nil
}
func (a *App) AutocompleteUsersInChannel(teamId string, channelId string, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) {
var autocomplete *model.UserAutocompleteInChannel
var err *model.AppError
term = strings.TrimSpace(term)
if a.IsESAutocompletionEnabled() {
autocomplete, err = a.esAutocompleteUsersInChannel(teamId, channelId, term, options)
if err != nil {
mlog.Error("Encountered error on AutocompleteUsersInChannel through Elasticsearch. Falling back to default autocompletion.", mlog.Err(err))
}
autocomplete, err := a.Srv().Store.User().AutocompleteUsersInChannel(teamId, channelId, term, options)
if err != nil {
return nil, err
}
if !a.IsESAutocompletionEnabled() || err != nil {
autocomplete = &model.UserAutocompleteInChannel{}
for _, user := range autocomplete.InChannel {
a.SanitizeProfile(user, options.IsAdmin)
}
uchan := make(chan store.StoreResult, 1)
go func() {
users, err := a.Srv().Store.User().SearchInChannel(channelId, term, options)
uchan <- store.StoreResult{Data: users, Err: err}
close(uchan)
}()
nuchan := make(chan store.StoreResult, 1)
go func() {
users, err := a.Srv().Store.User().SearchNotInChannel(teamId, channelId, term, options)
nuchan <- store.StoreResult{Data: users, Err: err}
close(nuchan)
}()
result := <-uchan
if result.Err != nil {
return nil, result.Err
}
users := result.Data.([]*model.User)
for _, user := range users {
a.SanitizeProfile(user, options.IsAdmin)
}
autocomplete.InChannel = users
result = <-nuchan
if result.Err != nil {
return nil, result.Err
}
users = result.Data.([]*model.User)
for _, user := range users {
a.SanitizeProfile(user, options.IsAdmin)
}
autocomplete.OutOfChannel = users
for _, user := range autocomplete.OutOfChannel {
a.SanitizeProfile(user, options.IsAdmin)
}
return autocomplete, nil
}
func (a *App) esAutocompleteUsersInTeam(teamId, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInTeam, *model.AppError) {
listOfAllowedChannels, err := a.getListOfAllowedChannelsForTeam(teamId, options.ViewRestrictions)
if err != nil {
return nil, err
}
if len(listOfAllowedChannels) == 0 {
return &model.UserAutocompleteInTeam{}, nil
}
func (a *App) AutocompleteUsersInTeam(teamId string, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInTeam, *model.AppError) {
var err *model.AppError
usersIds, err := a.Elasticsearch().SearchUsersInTeam(teamId, listOfAllowedChannels, term, options)
if err != nil {
return nil, err
}
term = strings.TrimSpace(term)
users, err := a.Srv().Store.User().GetProfileByIds(usersIds, nil, false)
users, err := a.Srv().Store.User().Search(teamId, term, options)
if err != nil {
return nil, err
}
@@ -1968,37 +1765,6 @@ func (a *App) esAutocompleteUsersInTeam(teamId, term string, options *model.User
autocomplete := &model.UserAutocompleteInTeam{}
autocomplete.InTeam = users
return autocomplete, nil
}
func (a *App) AutocompleteUsersInTeam(teamId string, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInTeam, *model.AppError) {
var autocomplete *model.UserAutocompleteInTeam
var err *model.AppError
term = strings.TrimSpace(term)
if a.IsESAutocompletionEnabled() {
autocomplete, err = a.esAutocompleteUsersInTeam(teamId, term, options)
if err != nil {
mlog.Error("Encountered error on AutocompleteUsersInTeam through Elasticsearch. Falling back to default autocompletion.", mlog.Err(err))
}
}
if !a.IsESAutocompletionEnabled() || err != nil {
autocomplete = &model.UserAutocompleteInTeam{}
users, err := a.Srv().Store.User().Search(teamId, term, options)
if err != nil {
return nil, err
}
for _, user := range users {
a.SanitizeProfile(user, options.IsAdmin)
}
autocomplete.InTeam = users
}
return autocomplete, nil
}
@@ -2044,14 +1810,6 @@ func (a *App) UpdateOAuthUserAttrs(userData io.Reader, user *model.User, provide
user = users.New
a.InvalidateCacheForUser(user.Id)
if a.IsESIndexingEnabled() {
a.Srv().Go(func() {
if err := a.indexUser(user); err != nil {
mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.Err(err))
}
})
}
}
return nil
@@ -2194,61 +1952,6 @@ func (a *App) GetViewUsersRestrictions(userId string) (*model.ViewUsersRestricti
return &model.ViewUsersRestrictions{Teams: teamIdsWithPermission, Channels: channelIds}, nil
}
/**
* Returns a list with the channel ids that the user has permissions to view on a
* team. If the result is an empty list, the user can't view any channel; if it's
* nil, there are no restrictions for the user in the specified team.
*/
func (a *App) GetViewUsersRestrictionsForTeam(userId string, teamId string) ([]string, *model.AppError) {
if a.HasPermissionTo(userId, model.PERMISSION_VIEW_MEMBERS) {
return nil, nil
}
if a.HasPermissionToTeam(userId, teamId, model.PERMISSION_VIEW_MEMBERS) {
return nil, nil
}
members, err := a.Srv().Store.Channel().GetMembersForUser(teamId, userId)
if err != nil {
return nil, err
}
channelIds := []string{}
for _, membership := range *members {
channelIds = append(channelIds, membership.ChannelId)
}
return channelIds, nil
}
func (a *App) getListOfAllowedChannelsForTeam(teamId string, viewRestrictions *model.ViewUsersRestrictions) ([]string, *model.AppError) {
var listOfAllowedChannels []string
if viewRestrictions == nil || strings.Contains(strings.Join(viewRestrictions.Teams, "."), teamId) {
channels, err := a.Srv().Store.Channel().GetTeamChannels(teamId)
if err != nil {
return nil, err
}
channelIds := []string{}
for _, channel := range *channels {
channelIds = append(channelIds, channel.Id)
}
return channelIds, nil
}
channels, err := a.Srv().Store.Channel().GetChannelsByIds(viewRestrictions.Channels)
if err != nil {
return nil, err
}
for _, c := range channels {
if c.TeamId == teamId {
listOfAllowedChannels = append(listOfAllowedChannels, c.Id)
}
}
return listOfAllowedChannels, nil
}
// PromoteGuestToUser Convert user's roles and all his mermbership's roles from
// guest roles to regular user roles.
func (a *App) PromoteGuestToUser(user *model.User, requestorId string) *model.AppError {

84
app/user_test.go
Просмотреть файл

@@ -942,90 +942,6 @@ func TestGetViewUsersRestrictions(t *testing.T) {
})
}
func TestGetViewUsersRestrictionsForTeam(t *testing.T) {
th := Setup(t).InitBasic()
defer th.TearDown()
team1 := th.CreateTeam()
team2 := th.CreateTeam()
th.CreateTeam() // Another team
user1 := th.CreateUser()
th.LinkUserToTeam(user1, team1)
th.LinkUserToTeam(user1, team2)
th.App.UpdateTeamMemberRoles(team1.Id, user1.Id, "team_user team_admin")
team1channel1 := th.CreateChannel(team1)
team1channel2 := th.CreateChannel(team1)
th.CreateChannel(team1) // Another channel
team1offtopic, err := th.App.GetChannelByName("off-topic", team1.Id, false)
require.Nil(t, err)
team1townsquare, err := th.App.GetChannelByName("town-square", team1.Id, false)
require.Nil(t, err)
team2channel1 := th.CreateChannel(team2)
th.CreateChannel(team2) // Another channel
th.App.AddUserToChannel(user1, team1channel1)
th.App.AddUserToChannel(user1, team1channel2)
th.App.AddUserToChannel(user1, team2channel1)
addPermission := func(role *model.Role, permission string) *model.AppError {
newPermissions := append(role.Permissions, permission)
_, err := th.App.PatchRole(role, &model.RolePatch{Permissions: &newPermissions})
return err
}
removePermission := func(role *model.Role, permission string) *model.AppError {
newPermissions := []string{}
for _, oldPermission := range role.Permissions {
if permission != oldPermission {
newPermissions = append(newPermissions, oldPermission)
}
}
_, err := th.App.PatchRole(role, &model.RolePatch{Permissions: &newPermissions})
return err
}
t.Run("VIEW_MEMBERS permission granted at system level", func(t *testing.T) {
restrictions, err := th.App.GetViewUsersRestrictionsForTeam(user1.Id, team1.Id)
require.Nil(t, err)
assert.Nil(t, restrictions)
})
t.Run("VIEW_MEMBERS permission granted at team level", func(t *testing.T) {
systemUserRole, err := th.App.GetRoleByName(model.SYSTEM_USER_ROLE_ID)
require.Nil(t, err)
teamUserRole, err := th.App.GetRoleByName(model.TEAM_USER_ROLE_ID)
require.Nil(t, err)
require.Nil(t, removePermission(systemUserRole, model.PERMISSION_VIEW_MEMBERS.Id))
defer addPermission(systemUserRole, model.PERMISSION_VIEW_MEMBERS.Id)
require.Nil(t, addPermission(teamUserRole, model.PERMISSION_VIEW_MEMBERS.Id))
defer removePermission(teamUserRole, model.PERMISSION_VIEW_MEMBERS.Id)
restrictions, err := th.App.GetViewUsersRestrictionsForTeam(user1.Id, team1.Id)
require.Nil(t, err)
assert.Nil(t, restrictions)
})
t.Run("VIEW_MEMBERS permission not granted at any level", func(t *testing.T) {
systemUserRole, err := th.App.GetRoleByName(model.SYSTEM_USER_ROLE_ID)
require.Nil(t, err)
require.Nil(t, removePermission(systemUserRole, model.PERMISSION_VIEW_MEMBERS.Id))
defer addPermission(systemUserRole, model.PERMISSION_VIEW_MEMBERS.Id)
restrictions, err := th.App.GetViewUsersRestrictionsForTeam(user1.Id, team1.Id)
require.Nil(t, err)
assert.NotNil(t, restrictions)
assert.ElementsMatch(t, []string{team1townsquare.Id, team1offtopic.Id, team1channel1.Id, team1channel2.Id}, restrictions)
})
}
func TestPromoteGuestToUser(t *testing.T) {
th := Setup(t).InitBasic()
defer th.TearDown()