Adding the new search engine abstraction (#13304)

* WIP

* Adding bleve to go modules

* WIP

* Adding missing files from searchengine implementation

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* User and channel indexing and searches implemented

* Make bleve tests run with in-memory indexes

* Implement post index and deletion tests

* Initial commits for the search layer

* Removing unnecesary indexing

* WIP

* WIP

* More fixes for tests

* Adding the search layer

* Finishing the migration of searchers to the layer

* Removing unnecesary code

* Allowing multiple engines active at the same time

* WIP

* Add simple post search

* Print information when using bleve

* Adding some debugging to understand better how the searches are working

* Making more dynamic config of search engines

* Add post search basics

* Adding the Purge API endpoint

* Fixing bleve config updates

* Adding missed file

* Regenerating search engine mocks

* Adding missed v5 to modules imports

* fixing i18n

* Fixing some test around search engine

* Removing all bleve traces

* Cleaning up the vendors directory and go.mod/go.sum files

* Regenerating timer layer

* Adding properly the license

* Fixing govet shadow error

* Fixing some tests

* Fixing TestSearchPostsFromUser

* Fixing another test

* Fixing more tests

* Fixing more tests

* Removing SearchEngine redundant text from searchengine module code

* Fixing some reindexing problems in members updates

* Fixing tests

* Addressing PR comments

* Reverting go.mod and go.sum

* Addressing PR comments

* Fixing tests compilation

* Fixing govet

* Adding search engine stop method

* Being more explicit on where we use includeDeleted

* Adding GetSqlSupplier test helper method

* Mocking elasticsearch start function

* Fixing tests

Co-authored-by: Miguel de la Cruz <miguel@mcrx.me>
Co-authored-by: mattermod <mattermod@users.noreply.github.com>

store/storetest/channel_store.go

@@ -447,6 +447,18 @@ func testChannelStoreGetChannelsByIds(t *testing.T, ss store.Store) {
 	o2.Name = "bb" + model.NewId() + "b"
 	o2.Type = model.CHANNEL_DIRECT
 
+	o3 := model.Channel{}
+	o3.TeamId = model.NewId()
+	o3.DisplayName = "Deleted channel"
+	o3.Name = "cc" + model.NewId() + "b"
+	o3.Type = model.CHANNEL_OPEN
+	_, err = ss.Channel().Save(&o3, -1)
+	require.Nil(t, err)
+	err = ss.Channel().Delete(o3.Id, 123)
+	require.Nil(t, err)
+	o3.DeleteAt = 123
+	o3.UpdateAt = 123
+
 	m1 := model.ChannelMember{}
 	m1.ChannelId = o2.Id
 	m1.UserId = u1.Id
@@ -460,17 +472,30 @@ func testChannelStoreGetChannelsByIds(t *testing.T, ss store.Store) {
 	_, err = ss.Channel().SaveDirectChannel(&o2, &m1, &m2)
 	require.Nil(t, err)
 
-	r1, err := ss.Channel().GetChannelsByIds([]string{o1.Id, o2.Id})
-	require.Nil(t, err, err)
-	require.Len(t, r1, 2, "invalid returned channels, exepected 2 and got "+strconv.Itoa(len(r1)))
-	require.Equal(t, o1.ToJson(), r1[0].ToJson())
-	require.Equal(t, o2.ToJson(), r1[1].ToJson())
+	t.Run("Get 2 existing channels", func(t *testing.T) {
+		r1, err := ss.Channel().GetChannelsByIds([]string{o1.Id, o2.Id}, false)
+		require.Nil(t, err, err)
+		require.Len(t, r1, 2, "invalid returned channels, exepected 2 and got "+strconv.Itoa(len(r1)))
+		require.Equal(t, o1.ToJson(), r1[0].ToJson())
+		require.Equal(t, o2.ToJson(), r1[1].ToJson())
+	})
 
-	nonexistentId := "abcd1234"
-	r2, err := ss.Channel().GetChannelsByIds([]string{o1.Id, nonexistentId})
-	require.Nil(t, err, err)
-	require.Len(t, r2, 1, "invalid returned channels, expected 1 and got "+strconv.Itoa(len(r2)))
-	require.Equal(t, o1.ToJson(), r2[0].ToJson(), "invalid returned channel")
+	t.Run("Get 1 existing and 1 not existing channel", func(t *testing.T) {
+		nonexistentId := "abcd1234"
+		r2, err := ss.Channel().GetChannelsByIds([]string{o1.Id, nonexistentId}, false)
+		require.Nil(t, err, err)
+		require.Len(t, r2, 1, "invalid returned channels, expected 1 and got "+strconv.Itoa(len(r2)))
+		require.Equal(t, o1.ToJson(), r2[0].ToJson(), "invalid returned channel")
+	})
+
+	t.Run("Get 2 existing and 1 deleted channel", func(t *testing.T) {
+		r1, err := ss.Channel().GetChannelsByIds([]string{o1.Id, o2.Id, o3.Id}, true)
+		require.Nil(t, err, err)
+		require.Len(t, r1, 3, "invalid returned channels, exepected 3 and got "+strconv.Itoa(len(r1)))
+		require.Equal(t, o1.ToJson(), r1[0].ToJson())
+		require.Equal(t, o2.ToJson(), r1[1].ToJson())
+		require.Equal(t, o3.ToJson(), r1[2].ToJson())
+	})
 }
 
 func testChannelStoreGetForPost(t *testing.T, ss store.Store) {

store/storetest/mocks/ChannelStore.go

@@ -619,13 +619,13 @@ func (_m *ChannelStore) GetChannelsBatchForIndexing(startTime int64, endTime int
 	return r0, r1
 }
 
-// GetChannelsByIds provides a mock function with given fields: channelIds
-func (_m *ChannelStore) GetChannelsByIds(channelIds []string) ([]*model.Channel, *model.AppError) {
-	ret := _m.Called(channelIds)
+// GetChannelsByIds provides a mock function with given fields: channelIds, includeDeleted
+func (_m *ChannelStore) GetChannelsByIds(channelIds []string, includeDeleted bool) ([]*model.Channel, *model.AppError) {
+	ret := _m.Called(channelIds, includeDeleted)
 
 	var r0 []*model.Channel
-	if rf, ok := ret.Get(0).(func([]string) []*model.Channel); ok {
-		r0 = rf(channelIds)
+	if rf, ok := ret.Get(0).(func([]string, bool) []*model.Channel); ok {
+		r0 = rf(channelIds, includeDeleted)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).([]*model.Channel)
@@ -633,8 +633,8 @@ func (_m *ChannelStore) GetChannelsByIds(channelIds []string) ([]*model.Channel,
 	}
 
 	var r1 *model.AppError
-	if rf, ok := ret.Get(1).(func([]string) *model.AppError); ok {
-		r1 = rf(channelIds)
+	if rf, ok := ret.Get(1).(func([]string, bool) *model.AppError); ok {
+		r1 = rf(channelIds, includeDeleted)
 	} else {
 		if ret.Get(1) != nil {
 			r1 = ret.Get(1).(*model.AppError)

store/storetest/mocks/PostStore.go

@@ -792,6 +792,31 @@ func (_m *PostStore) Search(teamId string, userId string, params *model.SearchPa
 	return r0, r1
 }
 
+// SearchPostsInTeamForUser provides a mock function with given fields: paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage
+func (_m *PostStore) SearchPostsInTeamForUser(paramsList []*model.SearchParams, userId string, teamId string, isOrSearch bool, includeDeletedChannels bool, page int, perPage int) (*model.PostSearchResults, *model.AppError) {
+	ret := _m.Called(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage)
+
+	var r0 *model.PostSearchResults
+	if rf, ok := ret.Get(0).(func([]*model.SearchParams, string, string, bool, bool, int, int) *model.PostSearchResults); ok {
+		r0 = rf(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.PostSearchResults)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func([]*model.SearchParams, string, string, bool, bool, int, int) *model.AppError); ok {
+		r1 = rf(paramsList, userId, teamId, isOrSearch, includeDeletedChannels, page, perPage)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
 // Update provides a mock function with given fields: newPost, oldPost
 func (_m *PostStore) Update(newPost *model.Post, oldPost *model.Post) (*model.Post, *model.AppError) {
 	ret := _m.Called(newPost, oldPost)

store/storetest/mocks/UserStore.go

@@ -84,6 +84,31 @@ func (_m *UserStore) AnalyticsGetSystemAdminCount() (int64, *model.AppError) {
 	return r0, r1
 }
 
+// AutocompleteUsersInChannel provides a mock function with given fields: teamId, channelId, term, options
+func (_m *UserStore) AutocompleteUsersInChannel(teamId string, channelId string, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) {
+	ret := _m.Called(teamId, channelId, term, options)
+
+	var r0 *model.UserAutocompleteInChannel
+	if rf, ok := ret.Get(0).(func(string, string, string, *model.UserSearchOptions) *model.UserAutocompleteInChannel); ok {
+		r0 = rf(teamId, channelId, term, options)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.UserAutocompleteInChannel)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(string, string, string, *model.UserSearchOptions) *model.AppError); ok {
+		r1 = rf(teamId, channelId, term, options)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
 // ClearAllCustomRoleAssignments provides a mock function with given fields:
 func (_m *UserStore) ClearAllCustomRoleAssignments() *model.AppError {
 	ret := _m.Called()