[MM-21551] Add search tests structure to test the search engines (#14031)

* WIP

* Adding bleve to go modules

* WIP

* Adding missing files from searchengine implementation

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* User and channel indexing and searches implemented

* Make bleve tests run with in-memory indexes

* Implement post index and deletion tests

* Initial commits for the search layer

* Removing unnecesary indexing

* WIP

* WIP

* More fixes for tests

* Adding the search layer

* Finishing the migration of searchers to the layer

* Removing unnecesary code

* Allowing multiple engines active at the same time

* WIP

* Add simple post search

* Print information when using bleve

* Adding some debugging to understand better how the searches are working

* Making more dynamic config of search engines

* Add post search basics

* Adding the Purge API endpoint

* Fixing bleve config updates

* Adding missed file

* Regenerating search engine mocks

* Adding missed v5 to modules imports

* fixing i18n

* Fixing some test around search engine

* Removing all bleve traces

* Cleaning up the vendors directory and go.mod/go.sum files

* Regenerating timer layer

* Adding properly the license

* Fixing govet shadow error

* Fixing some tests

* Fixing TestSearchPostsFromUser

* Fixing another test

* Fixing more tests

* Fixing more tests

* Removing SearchEngine redundant text from searchengine module code

* Fixing some reindexing problems in members updates

* Fixing tests

* Addressing PR comments

* Reverting go.mod and go.sum

* Addressing PR comments

* Fixing tests compilation

* Fixing govet

* Adding search engine stop method

* Being more explicit on where we use includeDeleted

* Adding GetSqlSupplier test helper method

* Mocking elasticsearch start function

* Fixing tests

* Search tests

* Fix tests

* Fix mod

* Fixing searchEngine for test helpers with store mocks

* Remove loglines

* Fix i18n strings

* Migrate search posts tests

* Fix linter

* Do not run search tests if -short flag is enabled

* Migrate back store tests that didn't belong to the searchlayer

* Fix scopelint issues

Co-authored-by: Jesús Espino <jespinog@gmail.com>

store/searchlayer/channel_layer.go

@@ -19,12 +19,12 @@ func (c *SearchChannelStore) deleteChannelIndex(channel *model.Channel) {
 	if channel.Type == model.CHANNEL_OPEN {
 		for _, engine := range c.rootStore.searchEngine.GetActiveEngines() {
 			if engine.IsIndexingEnabled() {
-				go (func(engineCopy searchengine.SearchEngineInterface) {
+				runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
 					if err := engineCopy.DeleteChannel(channel); err != nil {
 						mlog.Error("Encountered error deleting channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
 					}
 					mlog.Debug("Removed channel from index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id))
-				})(engine)
+				})
 			}
 		}
 	}
@@ -34,12 +34,12 @@ func (c *SearchChannelStore) indexChannel(channel *model.Channel) {
 	if channel.Type == model.CHANNEL_OPEN {
 		for _, engine := range c.rootStore.searchEngine.GetActiveEngines() {
 			if engine.IsIndexingEnabled() {
-				go (func(engineCopy searchengine.SearchEngineInterface) {
+				runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
 					if err := engineCopy.IndexChannel(channel); err != nil {
 						mlog.Error("Encountered error indexing channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
 					}
 					mlog.Debug("Indexed channel in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id))
-				})(engine)
+				})
 			}
 		}
 	}

store/searchlayer/layer.go

@@ -59,7 +59,7 @@ func (s SearchStore) indexUserFromID(userId string) {
 func (s SearchStore) indexUser(user *model.User) {
 	for _, engine := range s.searchEngine.GetActiveEngines() {
 		if engine.IsIndexingEnabled() {
-			go (func(engineCopy searchengine.SearchEngineInterface) {
+			runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
 				userTeams, err := s.Team().GetTeamsByUserId(user.Id)
 				if err != nil {
 					mlog.Error("Encountered error indexing user", mlog.String("user_id", user.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
@@ -87,7 +87,21 @@ func (s SearchStore) indexUser(user *model.User) {
 					return
 				}
 				mlog.Debug("Indexed user in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("user_id", user.Id))
-			})(engine)
+			})
 		}
 	}
 }
+
+// Runs an indexing function synchronously or asynchronously depending on the engine
+func runIndexFn(engine searchengine.SearchEngineInterface, indexFn func(searchengine.SearchEngineInterface)) {
+	if engine.IsIndexingSync() {
+		indexFn(engine)
+		if err := engine.RefreshIndexes(); err != nil {
+			mlog.Error("Encountered error refresh the indexes", mlog.Err(err))
+		}
+	} else {
+		go (func(engineCopy searchengine.SearchEngineInterface) {
+			indexFn(engineCopy)
+		})(engine)
+	}
+}

store/searchlayer/post_layer.go

@@ -18,7 +18,7 @@ type SearchPostStore struct {
 func (s SearchPostStore) indexPost(post *model.Post) {
 	for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
 		if engine.IsIndexingEnabled() {
-			go (func(engineCopy searchengine.SearchEngineInterface) {
+			runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
 				channel, chanErr := s.rootStore.Channel().Get(post.ChannelId, true)
 				if chanErr != nil {
 					mlog.Error("Couldn't get channel for post for SearchEngine indexing.", mlog.String("channel_id", post.ChannelId), mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id), mlog.Err(chanErr))
@@ -28,7 +28,7 @@ func (s SearchPostStore) indexPost(post *model.Post) {
 					mlog.Error("Encountered error indexing post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
 				}
 				mlog.Debug("Indexed post in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id))
-			})(engine)
+			})
 		}
 	}
 }
@@ -36,12 +36,12 @@ func (s SearchPostStore) indexPost(post *model.Post) {
 func (s SearchPostStore) deletePostIndex(post *model.Post) {
 	for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
 		if engine.IsIndexingEnabled() {
-			go (func(engineCopy searchengine.SearchEngineInterface) {
+			runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
 				if err := engineCopy.DeletePost(post); err != nil {
 					mlog.Error("Encountered error deleting post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
 				}
 				mlog.Debug("Removed post from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id))
-			})(engine)
+			})
 		}
 	}
 }

store/searchlayer/user_layer.go

@@ -4,6 +4,7 @@
 package searchlayer
 
 import (
+	"net/http"
 	"strings"
 
 	"github.com/mattermost/mattermost-server/v5/mlog"
@@ -20,13 +21,13 @@ type SearchUserStore struct {
 func (s *SearchUserStore) deleteUserIndex(user *model.User) {
 	for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
 		if engine.IsIndexingEnabled() {
-			go (func(engineCopy searchengine.SearchEngineInterface) {
+			runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
 				if err := engineCopy.DeleteUser(user); err != nil {
 					mlog.Error("Encountered error deleting user", mlog.String("user_id", user.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
 					return
 				}
 				mlog.Debug("Removed user from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("user_id", user.Id))
-			})(engine)
+			})
 		}
 	}
 }
@@ -45,11 +46,13 @@ func (s *SearchUserStore) Search(teamId, term string, options *model.UserSearchO
 
 			usersIds, err := engine.SearchUsersInTeam(teamId, listOfAllowedChannels, term, options)
 			if err != nil {
+				mlog.Error("Encountered error on Search", mlog.String("search_engine", engine.GetName()), mlog.Err(err))
 				continue
 			}
 
 			users, err := s.UserStore.GetProfileByIds(usersIds, nil, false)
 			if err != nil {
+				mlog.Error("Encountered error on Search", mlog.String("search_engine", engine.GetName()), mlog.Err(err))
 				continue
 			}
 
@@ -58,6 +61,7 @@ func (s *SearchUserStore) Search(teamId, term string, options *model.UserSearchO
 		}
 	}
 	mlog.Debug("Using database search because no other search engine is available")
+
 	return s.UserStore.Search(teamId, term, options)
 }
 
@@ -138,6 +142,10 @@ func (s *SearchUserStore) autocompleteUsersInChannelByEngine(engine searchengine
 }
 
 func (s *SearchUserStore) getListOfAllowedChannelsForTeam(teamId string, viewRestrictions *model.ViewUsersRestrictions) ([]string, *model.AppError) {
+	if len(teamId) == 0 {
+		return nil, model.NewAppError("SearchUserStore", "store.search_user_store.empty_team_id", nil, "", http.StatusInternalServerError)
+	}
+
 	var listOfAllowedChannels []string
 	if viewRestrictions == nil || strings.Contains(strings.Join(viewRestrictions.Teams, "."), teamId) {
 		channels, err := s.rootStore.Channel().GetTeamChannels(teamId)
@@ -152,7 +160,12 @@ func (s *SearchUserStore) getListOfAllowedChannelsForTeam(teamId string, viewRes
 		return channelIds, nil
 	}
 
+	if len(viewRestrictions.Channels) == 0 {
+		return []string{}, nil
+	}
+
 	channels, err := s.rootStore.Channel().GetChannelsByIds(viewRestrictions.Channels, false)
+
 	if err != nil {
 		return nil, err
 	}