[MM-18946] [MM-26721] [MM-6842] Cross team search+private channel autocomplete (#18468)

* Show private channels in autocomplete This is supported in all Engines: MySQL, Postgres, Bleve, Elasticsearch. https://mattermost.atlassian.net/browse/MM-18496 ```release-note Private channels will now appear in channel autocomplete. If you are using Bleve or ElasticSearch, you will have to reindex the channels again to populate them with the new attributes. ``` A large chunk of this work has been based on the earlier effort at https://github.com/mattermost/mattermost-server/pull/17804. Full credit goes to https://github.com/arvinDarmawan. * Add comment ```release-note NONE ``` * Adding more tests ```release-note NONE ``` * fix more tests ```release-note NONE ``` * tmp ```release-note NONE ``` * more fixes ```release-note NONE ``` * add tests ```release-note NONE ``` * Add review comments from previous PR ```release-note NONE ``` * Add API to return all channels from all team ```release-note NONE ``` * Added support for bleve and ES ```release-note NONE ``` * Streaming response for GetAllChannels ```release-note NONE ``` * Fix tests ```release-note NONE ``` * Trigger CI ```release-note NONE ``` * fix tests ```release-note NONE ``` * Addressing review comments ```release-note NONE ``` * Fix lint ```release-note NONE ``` * Removing flaky test ```release-note NONE ``` * Address comments ```release-note NONE ``` * Trigger CI ```release-note NONE ``` * Added /users/<userid>/channel_members endpoint ```release-note NONE ``` * Minor edit ```release-note NONE ``` * Improve embedding ```release-note NONE ``` * Fix lint error ```release-note NONE ``` Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
2021-10-26 11:30:59 +05:30
--- a/services/searchengine/bleveengine/bleve.go
+++ b/services/searchengine/bleveengine/bleve.go
@@ -59,8 +59,11 @@ func init() {
 func getChannelIndexMapping() *mapping.IndexMappingImpl {
 	channelMapping := bleve.NewDocumentMapping()
 	channelMapping.AddFieldMappingsAt("Id", keywordMapping)
+	channelMapping.AddFieldMappingsAt("Type", keywordMapping)
 	channelMapping.AddFieldMappingsAt("TeamId", keywordMapping)
 	channelMapping.AddFieldMappingsAt("NameSuggest", keywordMapping)
+	channelMapping.AddFieldMappingsAt("UserIDs", keywordMapping)
+	channelMapping.AddFieldMappingsAt("TeamMemberIDs", keywordMapping)

 	indexMapping := bleve.NewIndexMapping()
 	indexMapping.AddDocumentMapping("_default", channelMapping)
--- a/services/searchengine/bleveengine/common.go
+++ b/services/searchengine/bleveengine/common.go
@@ -11,9 +11,12 @@ import (
 )

 type BLVChannel struct {
-	Id          string
-	TeamId      []string
-	NameSuggest []string
+	Id            string
+	Type          model.ChannelType
+	UserIDs       []string
+	TeamId        []string
+	TeamMemberIDs []string
+	NameSuggest   []string
 }

 type BLVUser struct {
@@ -46,14 +49,17 @@ type BLVFile struct {
 	Extension string
 }

-func BLVChannelFromChannel(channel *model.Channel) *BLVChannel {
+func BLVChannelFromChannel(channel *model.Channel, userIDs, teamMemberIDs []string) *BLVChannel {
 	displayNameInputs := searchengine.GetSuggestionInputsSplitBy(channel.DisplayName, " ")
 	nameInputs := searchengine.GetSuggestionInputsSplitByMultiple(channel.Name, []string{"-", "_"})

 	return &BLVChannel{
-		Id:          channel.Id,
-		TeamId:      []string{channel.TeamId},
-		NameSuggest: append(displayNameInputs, nameInputs...),
+		Id:            channel.Id,
+		Type:          channel.Type,
+		TeamId:        []string{channel.TeamId},
+		NameSuggest:   append(displayNameInputs, nameInputs...),
+		UserIDs:       userIDs,
+		TeamMemberIDs: teamMemberIDs,
 	}
 }

--- a/services/searchengine/bleveengine/indexer/indexing_job.go
+++ b/services/searchengine/bleveengine/indexer/indexing_job.go
@@ -510,7 +510,22 @@ func (worker *BleveIndexerWorker) BulkIndexChannels(channels []*model.Channel, p

 	for _, channel := range channels {
 		if channel.DeleteAt == 0 {
-			searchChannel := bleveengine.BLVChannelFromChannel(channel)
+			var userIDs []string
+			var err error
+			if channel.Type == model.ChannelTypePrivate {
+				userIDs, err = worker.jobServer.Store.Channel().GetAllChannelMembersById(channel.Id)
+				if err != nil {
+					return 0, model.NewAppError("BleveIndexerWorker.BulkIndexChannels", "bleveengine.indexer.do_job.bulk_index_channels.batch_error", nil, err.Error(), http.StatusInternalServerError)
+				}
+			}
+
+			// Get teamMember ids from channelid
+			teamMemberIDs, err := worker.jobServer.Store.Channel().GetTeamMembersForChannel(channel.Id)
+			if err != nil {
+				return 0, model.NewAppError("BleveIndexerWorker.BulkIndexChannels", "bleveengine.indexer.do_job.bulk_index_channels.batch_error", nil, err.Error(), http.StatusInternalServerError)
+			}
+
+			searchChannel := bleveengine.BLVChannelFromChannel(channel, userIDs, teamMemberIDs)
 			batch.Index(searchChannel.Id, searchChannel)
 		} else {
 			batch.Delete(channel.Id)
--- a/services/searchengine/bleveengine/search.go
+++ b/services/searchengine/bleveengine/search.go
@@ -303,21 +303,59 @@ func (b *BleveEngine) DeletePost(post *model.Post) *model.AppError {
 	return nil
 }

-func (b *BleveEngine) IndexChannel(channel *model.Channel) *model.AppError {
+func (b *BleveEngine) IndexChannel(channel *model.Channel, userIDs, teamMemberIDs []string) *model.AppError {
 	b.Mutex.RLock()
 	defer b.Mutex.RUnlock()

-	blvChannel := BLVChannelFromChannel(channel)
+	blvChannel := BLVChannelFromChannel(channel, userIDs, teamMemberIDs)
 	if err := b.ChannelIndex.Index(blvChannel.Id, blvChannel); err != nil {
 		return model.NewAppError("Bleveengine.IndexChannel", "bleveengine.index_channel.error", nil, err.Error(), http.StatusInternalServerError)
 	}
 	return nil
 }

-func (b *BleveEngine) SearchChannels(teamId, term string) ([]string, *model.AppError) {
-	teamIdQ := bleve.NewTermQuery(teamId)
-	teamIdQ.SetField("TeamId")
-	queries := []query.Query{teamIdQ}
+func (b *BleveEngine) SearchChannels(teamId, userID, term string) ([]string, *model.AppError) {
+	// This query essentially boils down to (if teamID is passed):
+	// match teamID == <>
+	// AND
+	// match term == <>
+	// AND
+	// match (channelType != 'P' || (<> in userIDs && channelType == 'P'))
+
+	// (or if teamID is not passed)
+	// <> in teamMemberIds
+	// AND
+	// match term == <>
+	// AND
+	// match (channelType != 'P' || (<> in userIDs && channelType == 'P'))
+
+	queries := []query.Query{}
+	if teamId != "" {
+		teamIdQ := bleve.NewTermQuery(teamId)
+		teamIdQ.SetField("TeamId")
+		queries = append(queries, teamIdQ)
+	} else {
+		teamMemberQ := bleve.NewTermQuery(userID)
+		teamMemberQ.SetField("TeamMemberIDs")
+		queries = append(queries, teamMemberQ)
+	}
+
+	boolNotPrivate := bleve.NewBooleanQuery()
+	privateQ := bleve.NewTermQuery(string(model.ChannelTypePrivate))
+	privateQ.SetField("Type")
+	boolNotPrivate.AddMustNot(privateQ)
+
+	userQ := bleve.NewBooleanQuery()
+	userIDQ := bleve.NewTermQuery(userID)
+	userIDQ.SetField("UserIDs")
+	userQ.AddMust(userIDQ)
+	userQ.AddMust(privateQ)
+
+	channelTypeQ := bleve.NewDisjunctionQuery()
+	channelTypeQ.AddQuery(boolNotPrivate)
+	channelTypeQ.AddQuery(userQ) // userID && 'p'
+
+	queries = append(queries, channelTypeQ)

 	if term != "" {
 		nameSuggestQ := bleve.NewPrefixQuery(strings.ToLower(term))
--- a/services/searchengine/interface.go
+++ b/services/searchengine/interface.go
@@ -27,8 +27,10 @@ type SearchEngineInterface interface {
 	DeletePost(post *model.Post) *model.AppError
 	DeleteChannelPosts(channelID string) *model.AppError
 	DeleteUserPosts(userID string) *model.AppError
-	IndexChannel(channel *model.Channel) *model.AppError
-	SearchChannels(teamId, term string) ([]string, *model.AppError)
+	// IndexChannel indexes a given channel. The userIDs are only populated
+	// for private channels.
+	IndexChannel(channel *model.Channel, userIDs, teamMemberIDs []string) *model.AppError
+	SearchChannels(teamId, userID, term string) ([]string, *model.AppError)
 	DeleteChannel(channel *model.Channel) *model.AppError
 	IndexUser(user *model.User, teamsIds, channelsIds []string) *model.AppError
 	SearchUsersInChannel(teamId, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError)
--- a/services/searchengine/mocks/SearchEngineInterface.go
+++ b/services/searchengine/mocks/SearchEngineInterface.go
@@ -234,13 +234,13 @@ func (_m *SearchEngineInterface) GetVersion() int {
 	return r0
 }

-// IndexChannel provides a mock function with given fields: channel
-func (_m *SearchEngineInterface) IndexChannel(channel *model.Channel) *model.AppError {
-	ret := _m.Called(channel)
+// IndexChannel provides a mock function with given fields: channel, userIDs, teamMemberIDs
+func (_m *SearchEngineInterface) IndexChannel(channel *model.Channel, userIDs []string, teamMemberIDs []string) *model.AppError {
+	ret := _m.Called(channel, userIDs, teamMemberIDs)

 	var r0 *model.AppError
-	if rf, ok := ret.Get(0).(func(*model.Channel) *model.AppError); ok {
-		r0 = rf(channel)
+	if rf, ok := ret.Get(0).(func(*model.Channel, []string, []string) *model.AppError); ok {
+		r0 = rf(channel, userIDs, teamMemberIDs)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).(*model.AppError)
@@ -400,13 +400,13 @@ func (_m *SearchEngineInterface) RefreshIndexes() *model.AppError {
 	return r0
 }

-// SearchChannels provides a mock function with given fields: teamId, term
-func (_m *SearchEngineInterface) SearchChannels(teamId string, term string) ([]string, *model.AppError) {
-	ret := _m.Called(teamId, term)
+// SearchChannels provides a mock function with given fields: teamId, userID, term
+func (_m *SearchEngineInterface) SearchChannels(teamId string, userID string, term string) ([]string, *model.AppError) {
+	ret := _m.Called(teamId, userID, term)

 	var r0 []string
-	if rf, ok := ret.Get(0).(func(string, string) []string); ok {
-		r0 = rf(teamId, term)
+	if rf, ok := ret.Get(0).(func(string, string, string) []string); ok {
+		r0 = rf(teamId, userID, term)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).([]string)
@@ -414,8 +414,8 @@ func (_m *SearchEngineInterface) SearchChannels(teamId string, term string) ([]s
 	}

 	var r1 *model.AppError
-	if rf, ok := ret.Get(1).(func(string, string) *model.AppError); ok {
-		r1 = rf(teamId, term)
+	if rf, ok := ret.Get(1).(func(string, string, string) *model.AppError); ok {
+		r1 = rf(teamId, userID, term)
 	} else {
 		if ret.Get(1) != nil {
 			r1 = ret.Get(1).(*model.AppError)