[MM-18946] [MM-26721] [MM-6842] Cross team search+private channel autocomplete (#18468)

* Show private channels in autocomplete This is supported in all Engines: MySQL, Postgres, Bleve, Elasticsearch. https://mattermost.atlassian.net/browse/MM-18496 ```release-note Private channels will now appear in channel autocomplete. If you are using Bleve or ElasticSearch, you will have to reindex the channels again to populate them with the new attributes. ``` A large chunk of this work has been based on the earlier effort at https://github.com/mattermost/mattermost-server/pull/17804. Full credit goes to https://github.com/arvinDarmawan. * Add comment ```release-note NONE ``` * Adding more tests ```release-note NONE ``` * fix more tests ```release-note NONE ``` * tmp ```release-note NONE ``` * more fixes ```release-note NONE ``` * add tests ```release-note NONE ``` * Add review comments from previous PR ```release-note NONE ``` * Add API to return all channels from all team ```release-note NONE ``` * Added support for bleve and ES ```release-note NONE ``` * Streaming response for GetAllChannels ```release-note NONE ``` * Fix tests ```release-note NONE ``` * Trigger CI ```release-note NONE ``` * fix tests ```release-note NONE ``` * Addressing review comments ```release-note NONE ``` * Fix lint ```release-note NONE ``` * Removing flaky test ```release-note NONE ``` * Address comments ```release-note NONE ``` * Trigger CI ```release-note NONE ``` * Added /users/<userid>/channel_members endpoint ```release-note NONE ``` * Minor edit ```release-note NONE ``` * Improve embedding ```release-note NONE ``` * Fix lint error ```release-note NONE ``` Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
2021-10-26 11:30:59 +05:30
--- a/store/searchlayer/channel_layer.go
+++ b/store/searchlayer/channel_layer.go
@@ -36,17 +36,31 @@ func (c *SearchChannelStore) deleteChannelIndex(channel *model.Channel) {
 }

 func (c *SearchChannelStore) indexChannel(channel *model.Channel) {
-	if channel.Type == model.ChannelTypeOpen {
-		for _, engine := range c.rootStore.searchEngine.GetActiveEngines() {
-			if engine.IsIndexingEnabled() {
-				runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
-					if err := engineCopy.IndexChannel(channel); err != nil {
-						mlog.Warn("Encountered error indexing channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
-						return
-					}
-					mlog.Debug("Indexed channel in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id))
-				})
-			}
+	var userIDs, teamMemberIDs []string
+	var err error
+	if channel.Type == model.ChannelTypePrivate {
+		userIDs, err = c.GetAllChannelMembersById(channel.Id)
+		if err != nil {
+			mlog.Warn("Encountered error while indexing channel", mlog.String("channel_id", channel.Id), mlog.Err(err))
+			return
+		}
+	}
+
+	teamMemberIDs, err = c.GetTeamMembersForChannel(channel.Id)
+	if err != nil {
+		mlog.Warn("Encountered error while indexing channel", mlog.String("channel_id", channel.Id), mlog.Err(err))
+		return
+	}
+
+	for _, engine := range c.rootStore.searchEngine.GetActiveEngines() {
+		if engine.IsIndexingEnabled() {
+			runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
+				if err := engineCopy.IndexChannel(channel, userIDs, teamMemberIDs); err != nil {
+					mlog.Warn("Encountered error indexing channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
+					return
+				}
+				mlog.Debug("Indexed channel in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id))
+			})
 		}
 	}
 }
@@ -75,6 +89,7 @@ func (c *SearchChannelStore) UpdateMember(cm *model.ChannelMember) (*model.Chann
 		if channelErr != nil {
 			mlog.Warn("Encountered error indexing user in channel", mlog.String("channel_id", member.ChannelId), mlog.Err(channelErr))
 		} else {
+			c.indexChannel(channel)
 			c.rootStore.indexUserFromID(channel.CreatorId)
 		}
 	}
@@ -89,25 +104,37 @@ func (c *SearchChannelStore) SaveMember(cm *model.ChannelMember) (*model.Channel
 		if channelErr != nil {
 			mlog.Warn("Encountered error indexing user in channel", mlog.String("channel_id", member.ChannelId), mlog.Err(channelErr))
 		} else {
+			c.indexChannel(channel)
 			c.rootStore.indexUserFromID(channel.CreatorId)
 		}
 	}
 	return member, err
 }

-func (c *SearchChannelStore) RemoveMember(channelId, userIdToRemove string) error {
-	err := c.ChannelStore.RemoveMember(channelId, userIdToRemove)
+func (c *SearchChannelStore) RemoveMember(channelID, userIdToRemove string) error {
+	err := c.ChannelStore.RemoveMember(channelID, userIdToRemove)
 	if err == nil {
 		c.rootStore.indexUserFromID(userIdToRemove)
 	}
+
+	channel, err := c.ChannelStore.Get(channelID, true)
+	if err == nil {
+		c.indexChannel(channel)
+	}
+
 	return err
 }

-func (c *SearchChannelStore) RemoveMembers(channelId string, userIds []string) error {
-	if err := c.ChannelStore.RemoveMembers(channelId, userIds); err != nil {
+func (c *SearchChannelStore) RemoveMembers(channelID string, userIds []string) error {
+	if err := c.ChannelStore.RemoveMembers(channelID, userIds); err != nil {
 		return err
 	}

+	channel, err := c.ChannelStore.Get(channelID, true)
+	if err == nil {
+		c.indexChannel(channel)
+	}
+
 	for _, uid := range userIds {
 		c.rootStore.indexUserFromID(uid)
 	}
@@ -119,27 +146,29 @@ func (c *SearchChannelStore) CreateDirectChannel(user *model.User, otherUser *mo
 	if err == nil {
 		c.rootStore.indexUserFromID(user.Id)
 		c.rootStore.indexUserFromID(otherUser.Id)
+		c.indexChannel(channel)
 	}
 	return channel, err
 }

 func (c *SearchChannelStore) SaveDirectChannel(directchannel *model.Channel, member1 *model.ChannelMember, member2 *model.ChannelMember) (*model.Channel, error) {
 	channel, err := c.ChannelStore.SaveDirectChannel(directchannel, member1, member2)
-	if err != nil {
+	if err == nil {
 		c.rootStore.indexUserFromID(member1.UserId)
 		c.rootStore.indexUserFromID(member2.UserId)
+		c.indexChannel(channel)
 	}
 	return channel, err
 }

-func (c *SearchChannelStore) AutocompleteInTeam(teamId string, term string, includeDeleted bool) (model.ChannelList, error) {
-	var channelList model.ChannelList
+func (c *SearchChannelStore) Autocomplete(userID, term string, includeDeleted bool) (model.ChannelListWithTeamData, error) {
+	var channelList model.ChannelListWithTeamData
 	var err error

 	allFailed := true
 	for _, engine := range c.rootStore.searchEngine.GetActiveEngines() {
 		if engine.IsAutocompletionEnabled() {
-			channelList, err = c.searchAutocompleteChannels(engine, teamId, term, includeDeleted)
+			channelList, err = c.searchAutocompleteChannelsAllTeams(engine, userID, term, includeDeleted)
 			if err != nil {
 				mlog.Warn("Encountered error on AutocompleteChannels through SearchEngine. Falling back to default autocompletion.", mlog.String("search_engine", engine.GetName()), mlog.Err(err))
 				continue
@@ -152,7 +181,7 @@ func (c *SearchChannelStore) AutocompleteInTeam(teamId string, term string, incl

 	if allFailed {
 		mlog.Debug("Using database search because no other search engine is available")
-		channelList, err = c.ChannelStore.AutocompleteInTeam(teamId, term, includeDeleted)
+		channelList, err = c.ChannelStore.Autocomplete(userID, term, includeDeleted)
 		if err != nil {
 			return nil, errors.Wrap(err, "Failed to autocomplete channels in team")
 		}
@@ -165,21 +194,69 @@ func (c *SearchChannelStore) AutocompleteInTeam(teamId string, term string, incl
 	return channelList, nil
 }

-func (c *SearchChannelStore) searchAutocompleteChannels(engine searchengine.SearchEngineInterface, teamId, term string, includeDeleted bool) (model.ChannelList, error) {
-	channelIds, err := engine.SearchChannels(teamId, term)
+func (c *SearchChannelStore) AutocompleteInTeam(teamID, userID, term string, includeDeleted bool) (model.ChannelList, error) {
+	var channelList model.ChannelList
+	var err error
+
+	allFailed := true
+	for _, engine := range c.rootStore.searchEngine.GetActiveEngines() {
+		if engine.IsAutocompletionEnabled() {
+			channelList, err = c.searchAutocompleteChannels(engine, teamID, userID, term, includeDeleted)
+			if err != nil {
+				mlog.Warn("Encountered error on AutocompleteChannels through SearchEngine. Falling back to default autocompletion.", mlog.String("search_engine", engine.GetName()), mlog.Err(err))
+				continue
+			}
+			allFailed = false
+			mlog.Debug("Using the first available search engine", mlog.String("search_engine", engine.GetName()))
+			break
+		}
+	}
+
+	if allFailed {
+		mlog.Debug("Using database search because no other search engine is available")
+		channelList, err = c.ChannelStore.AutocompleteInTeam(teamID, userID, term, includeDeleted)
+		if err != nil {
+			return nil, errors.Wrap(err, "Failed to autocomplete channels in team")
+		}
+	}
+
+	if err != nil {
+		return channelList, err
+	}
+
+	return channelList, nil
+}
+
+func (c *SearchChannelStore) searchAutocompleteChannels(engine searchengine.SearchEngineInterface, teamId, userID, term string, includeDeleted bool) (model.ChannelList, error) {
+	channelIds, err := engine.SearchChannels(teamId, userID, term)
 	if err != nil {
 		return nil, err
 	}

 	channelList := model.ChannelList{}
+	var nErr error
 	if len(channelIds) > 0 {
-		channels, err := c.ChannelStore.GetChannelsByIds(channelIds, includeDeleted)
-		if err != nil {
-			return nil, errors.Wrap(err, "Failed to get channels by ids")
+		channelList, nErr = c.ChannelStore.GetChannelsByIds(channelIds, includeDeleted)
+		if nErr != nil {
+			return nil, errors.Wrap(nErr, "Failed to get channels by ids")
 		}
+	}

-		for _, ch := range channels {
-			channelList = append(channelList, ch)
+	return channelList, nil
+}
+
+func (c *SearchChannelStore) searchAutocompleteChannelsAllTeams(engine searchengine.SearchEngineInterface, userID, term string, includeDeleted bool) (model.ChannelListWithTeamData, error) {
+	channelIds, err := engine.SearchChannels("", userID, term)
+	if err != nil {
+		return nil, err
+	}
+
+	channelList := model.ChannelListWithTeamData{}
+	var nErr error
+	if len(channelIds) > 0 {
+		channelList, nErr = c.ChannelStore.GetChannelsWithTeamDataByIds(channelIds, includeDeleted)
+		if nErr != nil {
+			return nil, errors.Wrap(nErr, "Failed to get channels by ids")
 		}
 	}

@@ -187,10 +264,21 @@ func (c *SearchChannelStore) searchAutocompleteChannels(engine searchengine.Sear
 }

 func (c *SearchChannelStore) PermanentDeleteMembersByUser(userId string) error {
+	channels, errGetChannels := c.ChannelStore.GetChannelsByUser(userId, false, 0, -1, "")
+	if errGetChannels != nil {
+		mlog.Warn("Encountered error indexing channel after removing user", mlog.String("user_id", userId), mlog.Err(errGetChannels))
+	}
+
 	err := c.ChannelStore.PermanentDeleteMembersByUser(userId)
 	if err == nil {
 		c.rootStore.indexUserFromID(userId)
+		if errGetChannels == nil {
+			for _, ch := range channels {
+				c.indexChannel(ch)
+			}
+		}
 	}
+
 	return err
 }