Add search engine support for files (#16190)

* Add search engine support for files * Fixing i18n * Fix golangci-lint * Fix consistency problem in the Search receiver functio of the SqlFileStore * Fixing some tests * Fixing test * Apply suggestions from code review Co-authored-by: Mario de Frutos Dieguez <mario@defrutos.org> * Addressing PR review comments * Removing some empty lines * Address PR review comments * Fixing problem after merge master * Fixing spelling problem * Add missed translations * Fixing certain global variable usages after merge master * Fixing some constants usage * Fixing goimports order Co-authored-by: Mario de Frutos Dieguez <mario@defrutos.org>
2021-01-11 15:14:16 +01:00
--- a/store/searchlayer/channel_layer.go
+++ b/store/searchlayer/channel_layer.go
@@ -24,6 +24,7 @@ func (c *SearchChannelStore) deleteChannelIndex(channel *model.Channel) {
 				runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
 					if err := engineCopy.DeleteChannel(channel); err != nil {
 						mlog.Warn("Encountered error deleting channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+						return
 					}
 					mlog.Debug("Removed channel from index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id))
 				})
@@ -39,6 +40,7 @@ func (c *SearchChannelStore) indexChannel(channel *model.Channel) {
 				runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
 					if err := engineCopy.IndexChannel(channel); err != nil {
 						mlog.Warn("Encountered error indexing channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+						return
 					}
 					mlog.Debug("Indexed channel in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id))
 				})
--- a/store/searchlayer/file_info_layer.go
+++ b/store/searchlayer/file_info_layer.go
@@ -0,0 +1,197 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package searchlayer
+
+import (
+	"github.com/mattermost/mattermost-server/v5/mlog"
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/services/searchengine"
+	"github.com/mattermost/mattermost-server/v5/store"
+)
+
+type SearchFileInfoStore struct {
+	store.FileInfoStore
+	rootStore *SearchStore
+}
+
+func (s SearchFileInfoStore) indexFile(file *model.FileInfo) {
+	for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
+		if engine.IsIndexingEnabled() {
+			runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
+				if file.PostId == "" {
+					return
+				}
+				post, postErr := s.rootStore.Post().GetSingle(file.PostId)
+				if postErr != nil {
+					mlog.Error("Couldn't get post for file for SearchEngine indexing.", mlog.String("post_id", file.PostId), mlog.String("search_engine", engineCopy.GetName()), mlog.String("file_info_id", file.Id), mlog.Err(postErr))
+					return
+				}
+
+				if err := engineCopy.IndexFile(file, post.ChannelId); err != nil {
+					mlog.Error("Encountered error indexing file", mlog.String("file_info_id", file.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+					return
+				}
+				mlog.Debug("Indexed file in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("file_info_id", file.Id))
+			})
+		}
+	}
+}
+
+func (s SearchFileInfoStore) deleteFileIndex(fileID string) {
+	for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
+		if engine.IsIndexingEnabled() {
+			runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
+				if err := engineCopy.DeleteFile(fileID); err != nil {
+					mlog.Error("Encountered error deleting file", mlog.String("file_info_id", fileID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+					return
+				}
+				mlog.Debug("Removed file from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("file_info_id", fileID))
+			})
+		}
+	}
+}
+
+func (s SearchFileInfoStore) deleteFileIndexForUser(userID string) {
+	for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
+		if engine.IsIndexingEnabled() {
+			runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
+				if err := engineCopy.DeleteUserFiles(userID); err != nil {
+					mlog.Error("Encountered error deleting files for user", mlog.String("user_id", userID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+					return
+				}
+				mlog.Debug("Removed user's files from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("user_id", userID))
+			})
+		}
+	}
+}
+
+func (s SearchFileInfoStore) deleteFileIndexForPost(postID string) {
+	for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
+		if engine.IsIndexingEnabled() {
+			runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
+				if err := engineCopy.DeletePostFiles(postID); err != nil {
+					mlog.Error("Encountered error deleting files for post", mlog.String("post_id", postID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+					return
+				}
+				mlog.Debug("Removed post's files from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", postID))
+			})
+		}
+	}
+}
+
+func (s SearchFileInfoStore) deleteFileIndexBatch(endTime, limit int64) {
+	for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
+		if engine.IsIndexingEnabled() {
+			runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
+				if err := engineCopy.DeleteFilesBatch(endTime, limit); err != nil {
+					mlog.Error("Encountered error deleting a batch of files", mlog.Int64("limit", limit), mlog.Int64("end_time", endTime), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+					return
+				}
+				mlog.Debug("Removed batch of files from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.Int64("end_time", endTime), mlog.Int64("limit", limit))
+			})
+		}
+	}
+}
+
+func (s SearchFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, error) {
+	nfile, err := s.FileInfoStore.Save(info)
+	if err == nil {
+		s.indexFile(nfile)
+	}
+	return nfile, err
+}
+
+func (s SearchFileInfoStore) SetContent(fileID, content string) error {
+	err := s.FileInfoStore.SetContent(fileID, content)
+	if err == nil {
+		nfile, err2 := s.FileInfoStore.Get(fileID)
+		if err2 == nil {
+			nfile.Content = content
+			s.indexFile(nfile)
+		}
+	}
+	return err
+}
+
+func (s SearchFileInfoStore) AttachToPost(fileId, postId, creatorId string) error {
+	err := s.FileInfoStore.AttachToPost(fileId, postId, creatorId)
+	if err == nil {
+		nFileInfo, err2 := s.FileInfoStore.Get(fileId)
+		if err2 == nil {
+			s.indexFile(nFileInfo)
+		}
+	}
+	return err
+}
+
+func (s SearchFileInfoStore) DeleteForPost(postId string) (string, error) {
+	result, err := s.FileInfoStore.DeleteForPost(postId)
+	if err == nil {
+		s.deleteFileIndexForPost(postId)
+	}
+	return result, err
+}
+
+func (s SearchFileInfoStore) PermanentDelete(fileId string) error {
+	err := s.FileInfoStore.PermanentDelete(fileId)
+	if err == nil {
+		s.deleteFileIndex(fileId)
+	}
+	return err
+}
+
+func (s SearchFileInfoStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, error) {
+	result, err := s.FileInfoStore.PermanentDeleteBatch(endTime, limit)
+	if err == nil {
+		s.deleteFileIndexBatch(endTime, limit)
+	}
+	return result, err
+}
+
+func (s SearchFileInfoStore) PermanentDeleteByUser(userId string) (int64, error) {
+	result, err := s.FileInfoStore.PermanentDeleteByUser(userId)
+	if err == nil {
+		s.deleteFileIndexForUser(userId)
+	}
+	return result, err
+}
+
+func (s SearchFileInfoStore) Search(paramsList []*model.SearchParams, userId, teamId string, page, perPage int) (*model.FileInfoList, error) {
+	for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
+		if engine.IsSearchEnabled() {
+			userChannels, nErr := s.rootStore.Channel().GetChannels(teamId, userId, paramsList[0].IncludeDeletedChannels, 0)
+			if nErr != nil {
+				return nil, nErr
+			}
+			fileIds, appErr := engine.SearchFiles(userChannels, paramsList, page, perPage)
+			if appErr != nil {
+				mlog.Error("Encountered error on Search.", mlog.String("search_engine", engine.GetName()), mlog.Err(appErr))
+				continue
+			}
+			mlog.Debug("Using the first available search engine", mlog.String("search_engine", engine.GetName()))
+
+			// Get the files
+			filesList := model.NewFileInfoList()
+			if len(fileIds) > 0 {
+				files, nErr := s.FileInfoStore.GetByIds(fileIds)
+				if nErr != nil {
+					return nil, nErr
+				}
+				for _, f := range files {
+					filesList.AddFileInfo(f)
+					filesList.AddOrder(f.Id)
+				}
+			}
+			return filesList, nil
+		}
+	}
+
+	if *s.rootStore.getConfig().SqlSettings.DisableDatabaseSearch {
+		mlog.Debug("Returning empty results for file Search as the database search is disabled")
+		return model.NewFileInfoList(), nil
+	}
+
+	mlog.Debug("Using database search because no other search engine is available")
+	return s.FileInfoStore.Search(paramsList, userId, teamId, page, perPage)
+}
--- a/store/searchlayer/layer.go
+++ b/store/searchlayer/layer.go
@@ -19,6 +19,7 @@ type SearchStore struct {
 	team         *SearchTeamStore
 	channel      *SearchChannelStore
 	post         *SearchPostStore
+	fileInfo     *SearchFileInfoStore
 	configValue  atomic.Value
 }

@@ -32,6 +33,7 @@ func NewSearchLayer(baseStore store.Store, searchEngine *searchengine.Broker, cf
 	searchStore.post = &SearchPostStore{PostStore: baseStore.Post(), rootStore: searchStore}
 	searchStore.team = &SearchTeamStore{TeamStore: baseStore.Team(), rootStore: searchStore}
 	searchStore.user = &SearchUserStore{UserStore: baseStore.User(), rootStore: searchStore}
+	searchStore.fileInfo = &SearchFileInfoStore{FileInfoStore: baseStore.FileInfo(), rootStore: searchStore}

 	return searchStore
 }
@@ -52,6 +54,10 @@ func (s *SearchStore) Post() store.PostStore {
 	return s.post
 }

+func (s *SearchStore) FileInfo() store.FileInfoStore {
+	return s.fileInfo
+}
+
 func (s *SearchStore) Team() store.TeamStore {
 	return s.team
 }
--- a/store/searchlayer/post_layer.go
+++ b/store/searchlayer/post_layer.go
@@ -28,6 +28,7 @@ func (s SearchPostStore) indexPost(post *model.Post) {
 				}
 				if err := engineCopy.IndexPost(post, channel.TeamId); err != nil {
 					mlog.Warn("Encountered error indexing post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+					return
 				}
 				mlog.Debug("Indexed post in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id))
 			})
@@ -41,6 +42,7 @@ func (s SearchPostStore) deletePostIndex(post *model.Post) {
 			runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
 				if err := engineCopy.DeletePost(post); err != nil {
 					mlog.Warn("Encountered error deleting post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+					return
 				}
 				mlog.Debug("Removed post from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id))
 			})
@@ -54,6 +56,7 @@ func (s SearchPostStore) deleteChannelPostsIndex(channelID string) {
 			runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
 				if err := engineCopy.DeleteChannelPosts(channelID); err != nil {
 					mlog.Warn("Encountered error deleting channel posts", mlog.String("channel_id", channelID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+					return
 				}
 				mlog.Debug("Removed all channel posts from the index in search engine", mlog.String("channel_id", channelID), mlog.String("search_engine", engineCopy.GetName()))
 			})
@@ -67,6 +70,7 @@ func (s SearchPostStore) deleteUserPostsIndex(userID string) {
 			runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
 				if err := engineCopy.DeleteUserPosts(userID); err != nil {
 					mlog.Warn("Encountered error deleting user posts", mlog.String("user_id", userID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+					return
 				}
 				mlog.Debug("Removed all user posts from the index in search engine", mlog.String("user_id", userID), mlog.String("search_engine", engineCopy.GetName()))
 			})