[MM-18946] [MM-26721] [MM-6842] Cross team search+private channel autocomplete (#18468)

* Show private channels in autocomplete This is supported in all Engines: MySQL, Postgres, Bleve, Elasticsearch. https://mattermost.atlassian.net/browse/MM-18496 ```release-note Private channels will now appear in channel autocomplete. If you are using Bleve or ElasticSearch, you will have to reindex the channels again to populate them with the new attributes. ``` A large chunk of this work has been based on the earlier effort at https://github.com/mattermost/mattermost-server/pull/17804. Full credit goes to https://github.com/arvinDarmawan. * Add comment ```release-note NONE ``` * Adding more tests ```release-note NONE ``` * fix more tests ```release-note NONE ``` * tmp ```release-note NONE ``` * more fixes ```release-note NONE ``` * add tests ```release-note NONE ``` * Add review comments from previous PR ```release-note NONE ``` * Add API to return all channels from all team ```release-note NONE ``` * Added support for bleve and ES ```release-note NONE ``` * Streaming response for GetAllChannels ```release-note NONE ``` * Fix tests ```release-note NONE ``` * Trigger CI ```release-note NONE ``` * fix tests ```release-note NONE ``` * Addressing review comments ```release-note NONE ``` * Fix lint ```release-note NONE ``` * Removing flaky test ```release-note NONE ``` * Address comments ```release-note NONE ``` * Trigger CI ```release-note NONE ``` * Added /users/<userid>/channel_members endpoint ```release-note NONE ``` * Minor edit ```release-note NONE ``` * Improve embedding ```release-note NONE ``` * Fix lint error ```release-note NONE ``` Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
2021-10-26 11:30:59 +05:30
--- a/api4/channel.go
+++ b/api4/channel.go
@@ -35,6 +35,7 @@ func (api *API) InitChannel() {
 	api.BaseRoutes.ChannelsForTeam.Handle("/autocomplete", api.APISessionRequired(autocompleteChannelsForTeam)).Methods("GET")
 	api.BaseRoutes.ChannelsForTeam.Handle("/search_autocomplete", api.APISessionRequired(autocompleteChannelsForTeamForSearch)).Methods("GET")
 	api.BaseRoutes.User.Handle("/teams/{team_id:[A-Za-z0-9]+}/channels", api.APISessionRequired(getChannelsForTeamForUser)).Methods("GET")
+	api.BaseRoutes.User.Handle("/channels", api.APISessionRequired(getChannelsForUser)).Methods("GET")

 	api.BaseRoutes.ChannelCategories.Handle("", api.APISessionRequired(getCategoriesForTeamForUser)).Methods("GET")
 	api.BaseRoutes.ChannelCategories.Handle("", api.APISessionRequired(createCategoryForTeamForUser)).Methods("POST")
@@ -66,7 +67,7 @@ func (api *API) InitChannel() {
 	api.BaseRoutes.ChannelMembers.Handle("", api.APISessionRequired(getChannelMembers)).Methods("GET")
 	api.BaseRoutes.ChannelMembers.Handle("/ids", api.APISessionRequired(getChannelMembersByIds)).Methods("POST")
 	api.BaseRoutes.ChannelMembers.Handle("", api.APISessionRequired(addChannelMember)).Methods("POST")
-	api.BaseRoutes.ChannelMembersForUser.Handle("", api.APISessionRequired(getChannelMembersForUser)).Methods("GET")
+	api.BaseRoutes.ChannelMembersForUser.Handle("", api.APISessionRequired(getChannelMembersForTeamForUser)).Methods("GET")
 	api.BaseRoutes.ChannelMember.Handle("", api.APISessionRequired(getChannelMember)).Methods("GET")
 	api.BaseRoutes.ChannelMember.Handle("", api.APISessionRequired(removeChannelMember)).Methods("DELETE")
 	api.BaseRoutes.ChannelMember.Handle("/roles", api.APISessionRequired(updateChannelMemberRoles)).Methods("PUT")
@@ -877,7 +878,7 @@ func getChannelsForTeamForUser(c *Context, w http.ResponseWriter, r *http.Reques
 		return
 	}

-	channels, err := c.App.GetChannelsForUser(c.Params.TeamId, c.Params.UserId, c.Params.IncludeDeleted, lastDeleteAt)
+	channels, err := c.App.GetChannelsForTeamForUser(c.Params.TeamId, c.Params.UserId, c.Params.IncludeDeleted, lastDeleteAt)
 	if err != nil {
 		c.Err = err
 		return
@@ -899,6 +900,79 @@ func getChannelsForTeamForUser(c *Context, w http.ResponseWriter, r *http.Reques
 	}
 }

+func getChannelsForUser(c *Context, w http.ResponseWriter, r *http.Request) {
+	c.RequireUserId()
+	if c.Err != nil {
+		return
+	}
+
+	if !c.App.SessionHasPermissionToUser(*c.AppContext.Session(), c.Params.UserId) {
+		c.SetPermissionError(model.PermissionEditOtherUsers)
+		return
+	}
+
+	query := r.URL.Query()
+	lastDeleteAt, nErr := strconv.Atoi(query.Get("last_delete_at"))
+	if nErr != nil {
+		lastDeleteAt = 0
+	}
+	if lastDeleteAt < 0 {
+		c.SetInvalidURLParam("last_delete_at")
+		return
+	}
+
+	pageSize := 100
+	fromChannelID := ""
+	// We have to write `[` and `]` separately because we want to stream the response.
+	// The internal API is paginated, but the client always needs to get the full data.
+	// Therefore, to avoid forcing the client to go through all the pages,
+	// we stream the full data from server side itself.
+	//
+	// Note that this means if an error occurs in mid-stream, the response won't be
+	// fully JSON.
+	w.Write([]byte(`[`))
+	enc := json.NewEncoder(w)
+	for {
+		channels, err := c.App.GetChannelsForUser(c.Params.UserId, c.Params.IncludeDeleted, lastDeleteAt, pageSize, fromChannelID)
+		if err != nil {
+			// If the page size was a perfect multiple of the total number of results,
+			// then the last query will always return zero results.
+			if fromChannelID != "" && err.Id == "app.channel.get_channels.not_found.app_error" {
+				break
+			}
+			c.Err = err
+			return
+		}
+
+		err = c.App.FillInChannelsProps(channels)
+		if err != nil {
+			c.Err = err
+			return
+		}
+
+		// intermediary comma between sets
+		if fromChannelID != "" {
+			w.Write([]byte(`,`))
+		}
+
+		for i, ch := range channels {
+			if err := enc.Encode(ch); err != nil {
+				mlog.Warn("Error while writing response", mlog.Err(err))
+			}
+			if i < len(channels)-1 {
+				w.Write([]byte(`,`))
+			}
+		}
+
+		if len(channels) < pageSize {
+			break
+		}
+
+		fromChannelID = channels[len(channels)-1].Id
+	}
+	w.Write([]byte(`]`))
+}
+
 func autocompleteChannelsForTeam(c *Context, w http.ResponseWriter, r *http.Request) {
 	c.RequireTeamId()
 	if c.Err != nil {
@@ -912,7 +986,7 @@ func autocompleteChannelsForTeam(c *Context, w http.ResponseWriter, r *http.Requ

 	name := r.URL.Query().Get("name")

-	channels, err := c.App.AutocompleteChannels(c.Params.TeamId, name)
+	channels, err := c.App.AutocompleteChannelsForTeam(c.Params.TeamId, c.AppContext.Session().UserId, name)
 	if err != nil {
 		c.Err = err
 		return
@@ -1029,6 +1103,31 @@ func searchAllChannels(c *Context, w http.ResponseWriter, r *http.Request) {
 		c.SetInvalidParam("channel_search")
 		return
 	}
+
+	fromSysConsole := true
+	if val := r.URL.Query().Get("system_console"); val != "" {
+		fromSysConsole, err = strconv.ParseBool(val)
+		if err != nil {
+			c.SetInvalidParam("system_console")
+			return
+		}
+	}
+
+	if !fromSysConsole {
+		// If the request is not coming from system_console, only show the user level channels
+		// from all teams.
+		channels, err := c.App.AutocompleteChannels(c.AppContext.Session().UserId, props.Term)
+		if err != nil {
+			c.Err = err
+			return
+		}
+
+		if err := json.NewEncoder(w).Encode(channels); err != nil {
+			mlog.Warn("Error while writing response", mlog.Err(err))
+		}
+		return
+	}
+
 	// Only system managers may use the ExcludePolicyConstrained field
 	if props.ExcludePolicyConstrained && !c.App.SessionHasPermissionTo(*c.AppContext.Session(), model.PermissionSysconsoleReadComplianceDataRetentionPolicy) {
 		c.SetPermissionError(model.PermissionSysconsoleReadComplianceDataRetentionPolicy)
@@ -1039,6 +1138,7 @@ func searchAllChannels(c *Context, w http.ResponseWriter, r *http.Request) {
 		c.SetPermissionError(model.PermissionSysconsoleReadUserManagementChannels)
 		return
 	}
+
 	includeDeleted, _ := strconv.ParseBool(r.URL.Query().Get("include_deleted"))
 	includeDeleted = includeDeleted || props.IncludeDeleted

@@ -1297,7 +1397,7 @@ func getChannelMember(c *Context, w http.ResponseWriter, r *http.Request) {
 	}
 }

-func getChannelMembersForUser(c *Context, w http.ResponseWriter, r *http.Request) {
+func getChannelMembersForTeamForUser(c *Context, w http.ResponseWriter, r *http.Request) {
 	c.RequireUserId().RequireTeamId()
 	if c.Err != nil {
 		return
--- a/api4/channel_category.go
+++ b/api4/channel_category.go
@@ -210,7 +210,7 @@ func updateCategoriesForTeamForUser(c *Context, w http.ResponseWriter, r *http.R
 }

 func validateSidebarCategory(c *Context, teamId, userId string, category *model.SidebarCategoryWithChannels) *model.AppError {
-	channels, err := c.App.GetChannelsForUser(teamId, userId, true, 0)
+	channels, err := c.App.GetChannelsForTeamForUser(teamId, userId, true, 0)
 	if err != nil {
 		return model.NewAppError("validateSidebarCategory", "api.invalid_channel", nil, err.Error(), http.StatusBadRequest)
 	}
@@ -221,7 +221,7 @@ func validateSidebarCategory(c *Context, teamId, userId string, category *model.
 }

 func validateSidebarCategories(c *Context, teamId, userId string, categories []*model.SidebarCategoryWithChannels) *model.AppError {
-	channels, err := c.App.GetChannelsForUser(teamId, userId, true, 0)
+	channels, err := c.App.GetChannelsForTeamForUser(teamId, userId, true, 0)
 	if err != nil {
 		return model.NewAppError("validateSidebarCategory", "api.invalid_channel", nil, err.Error(), http.StatusBadRequest)
 	}
--- a/api4/channel_test.go
+++ b/api4/channel_test.go
@@ -1034,6 +1034,58 @@ func TestGetChannelsForTeamForUser(t *testing.T) {
 	})
 }

+func TestGetChannelsForUser(t *testing.T) {
+	th := Setup(t).InitBasic()
+	defer th.TearDown()
+
+	client := th.Client
+
+	// Adding another team with more channels (public and private)
+	myTeam := th.CreateTeam()
+	ch1 := th.CreateChannelWithClientAndTeam(client, model.ChannelTypeOpen, myTeam.Id)
+	ch2 := th.CreateChannelWithClientAndTeam(client, model.ChannelTypePrivate, myTeam.Id)
+	th.LinkUserToTeam(th.BasicUser, myTeam)
+	th.App.AddUserToChannel(th.BasicUser, ch1, false)
+	th.App.AddUserToChannel(th.BasicUser, ch2, false)
+
+	channels, _, err := client.GetChannelsForUserWithLastDeleteAt(th.BasicUser.Id, 0)
+	require.NoError(t, err)
+
+	numPrivate := 0
+	numPublic := 0
+	numOffTopic := 0
+	numTownSquare := 0
+	for _, ch := range channels {
+		if ch.Type == model.ChannelTypeOpen {
+			numPublic++
+		} else if ch.Type == model.ChannelTypePrivate {
+			numPrivate++
+		}
+
+		if ch.DisplayName == "Off-Topic" {
+			numOffTopic++
+		} else if ch.DisplayName == "Town Square" {
+			numTownSquare++
+		}
+	}
+
+	assert.Len(t, channels, 9)
+	assert.Equal(t, 2, numPrivate)
+	assert.Equal(t, 7, numPublic)
+	assert.Equal(t, 2, numOffTopic)
+	assert.Equal(t, 2, numTownSquare)
+
+	// Creating some more channels to be exactly 100 to test page size boundaries.
+	for i := 0; i < 91; i++ {
+		ch1 = th.CreateChannelWithClientAndTeam(client, model.ChannelTypeOpen, myTeam.Id)
+		th.App.AddUserToChannel(th.BasicUser, ch1, false)
+	}
+
+	channels, _, err = client.GetChannelsForUserWithLastDeleteAt(th.BasicUser.Id, 0)
+	require.NoError(t, err)
+	assert.Len(t, channels, 100)
+}
+
 func TestGetAllChannels(t *testing.T) {
 	th := Setup(t).InitBasic()
 	defer th.TearDown()
@@ -1383,6 +1435,16 @@ func TestSearchAllChannels(t *testing.T) {
 	require.NoError(t, err)

 	team := th.CreateTeam()
+	privateChannel2, _, err := th.SystemAdminClient.CreateChannel(&model.Channel{
+		DisplayName: "dn_private2",
+		Name:        "private2",
+		Type:        model.ChannelTypePrivate,
+		TeamId:      team.Id,
+	})
+	require.NoError(t, err)
+	th.LinkUserToTeam(th.SystemAdminUser, team)
+	th.LinkUserToTeam(th.SystemAdminUser, th.BasicTeam)
+
 	groupConstrainedChannel, _, err := th.SystemAdminClient.CreateChannel(&model.Channel{
 		DisplayName:      "SearchAllChannels-groupConstrained-1",
 		Name:             "groupconstrained1",
@@ -1450,7 +1512,7 @@ func TestSearchAllChannels(t *testing.T) {
 		{
 			"Search with private channel filter",
 			&model.ChannelSearch{Private: true},
-			[]string{th.BasicPrivateChannel.Id, th.BasicPrivateChannel2.Id, privateChannel.Id, groupConstrainedChannel.Id},
+			[]string{th.BasicPrivateChannel.Id, privateChannel2.Id, th.BasicPrivateChannel2.Id, privateChannel.Id, groupConstrainedChannel.Id},
 		},
 		{
 			"Search with public channel filter",
@@ -1517,6 +1579,14 @@ func TestSearchAllChannels(t *testing.T) {
 		})
 	}

+	userChannels, _, err := th.SystemAdminClient.SearchAllChannelsForUser("private")
+	require.NoError(t, err)
+	assert.Len(t, userChannels, 2)
+
+	userChannels, _, err = th.SystemAdminClient.SearchAllChannelsForUser("FOOBARDISPLAYNAME")
+	require.NoError(t, err)
+	assert.Len(t, userChannels, 1)
+
 	// Searching with no terms returns all default channels
 	allChannels, _, err := th.SystemAdminClient.SearchAllChannels(&model.ChannelSearch{Term: ""})
 	require.NoError(t, err)
@@ -3238,7 +3308,7 @@ func TestAutocompleteChannels(t *testing.T) {
 	th := Setup(t).InitBasic()
 	defer th.TearDown()

-	// A private channel to make sure private channels are not used
+	// A private channel to make sure private channels are used.
 	ptown, _, _ := th.Client.CreateChannel(&model.Channel{
 		DisplayName: "Town",
 		Name:        "town",
@@ -3267,8 +3337,8 @@ func TestAutocompleteChannels(t *testing.T) {
 			"Basic town-square",
 			th.BasicTeam.Id,
 			"town",
-			[]string{"town-square"},
-			[]string{"off-topic", "town", "tower"},
+			[]string{"town-square", "town"},
+			[]string{"off-topic", "tower"},
 		},
 		{
 			"Basic off-topic",
@@ -3281,8 +3351,8 @@ func TestAutocompleteChannels(t *testing.T) {
 			"Basic town square and off topic",
 			th.BasicTeam.Id,
 			"tow",
-			[]string{"town-square", "tower"},
-			[]string{"off-topic", "town"},
+			[]string{"town-square", "tower", "town"},
+			[]string{"off-topic"},
 		},
 	} {
 		t.Run(tc.description, func(t *testing.T) {
--- a/api4/user.go
+++ b/api4/user.go
@@ -90,6 +90,7 @@ func (api *API) InitUser() {
 	api.BaseRoutes.Users.Handle("/migrate_auth/saml", api.APISessionRequired(migrateAuthToSaml)).Methods("POST")

 	api.BaseRoutes.User.Handle("/uploads", api.APISessionRequired(getUploadsForUser)).Methods("GET")
+	api.BaseRoutes.User.Handle("/channel_members", api.APISessionRequired(getChannelMembersForUser)).Methods("GET")

 	api.BaseRoutes.UserThreads.Handle("", api.APISessionRequired(getThreadsForUser)).Methods("GET")
 	api.BaseRoutes.UserThreads.Handle("/read", api.APISessionRequired(updateReadStateAllThreadsByUser)).Methods("PUT")
@@ -2800,6 +2801,28 @@ func getUploadsForUser(c *Context, w http.ResponseWriter, r *http.Request) {
 	w.Write(js)
 }

+func getChannelMembersForUser(c *Context, w http.ResponseWriter, r *http.Request) {
+	c.RequireUserId()
+	if c.Err != nil {
+		return
+	}
+
+	if !c.App.SessionHasPermissionToUser(*c.AppContext.Session(), c.Params.UserId) {
+		c.SetPermissionError(model.PermissionEditOtherUsers)
+		return
+	}
+
+	members, err := c.App.GetChannelMembersWithTeamDataForUserWithPagination(c.Params.UserId, c.Params.Page, c.Params.PerPage)
+	if err != nil {
+		c.Err = err
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(members); err != nil {
+		mlog.Warn("Error while writing response", mlog.Err(err))
+	}
+}
+
 func migrateAuthToLDAP(c *Context, w http.ResponseWriter, r *http.Request) {
 	props := model.StringInterfaceFromJSON(r.Body)
 	from, ok := props["from"].(string)
--- a/api4/user_test.go
+++ b/api4/user_test.go
@@ -5443,6 +5443,19 @@ func TestConvertUserToBot(t *testing.T) {
 	})
 }

+func TestGetChannelMembersWithTeamData(t *testing.T) {
+	th := Setup(t).InitBasic()
+	defer th.TearDown()
+
+	channels, resp, err := th.Client.GetChannelMembersWithTeamData(th.BasicUser.Id, 0, 5)
+	require.NoError(t, err)
+	CheckOKStatus(t, resp)
+	assert.Len(t, channels, 5)
+	for _, ch := range channels {
+		assert.Equal(t, th.BasicTeam.DisplayName, ch.TeamDisplayName)
+	}
+}
+
 func TestMigrateAuthToLDAP(t *testing.T) {
 	th := Setup(t).InitBasic()
 	defer th.TearDown()