Custom groups (#18839)

* WIP

* adding initial creategroup endpoint

* fetching by group source

* fixing startup error

* updating create endpoint to take an array of user_ids, this will allow us to create the group with one request

* adding delete group endpoint and appropriate test

* adding source param for getGroups

* adding add members and delete members endpoints

* locking down crud endpoints to only be allowed for custom groups

* user search stuff

* allowing remoteid be null by changing field to pointer

* code cleanup and store level tests

* adding new tests and removing unused endpoint

* resolving conflicts

* Adds authz check for group.

* Adds authz checks to groups APIs.

* Updated create group authz tests.

* Updates delete group tests.

* Tests create group.

* Adds some tests and validations.

* adding new parameter so I can get users not in a group

* Fixed all lint warnings.

* Fix type.

* fixing search users not in group

* Fixes some lint errors.

* Moves entry in JSON array.

* Fixed SQL query.

* Fixes permission migration test.

* Fixes migration test.

* Fixes some group store tests.

* Fix test.

* Fix test.

* Revert lint change.

* Migrated CreateWithUserIds to sqlx.

* Adds tests for GetMember; migrates implementation to sqlx.

* Tests GetNonMemberUsersPage and hanles wrong group id.

* Fixes test.

* Switches GetMaster to GetMasterX.

* Switches GetReplica to GetReplicaX.

* Fixes logic.

* Fixes shadow declaration.

* Adds include_member_count to get group API endpoint.

* Adds filter_has_member param to getGroups.

* Fixes.

* Removes array of group sources.

* fixing error

* Testing reverting CreateWithUserIds back to gorp.

* Added websocket event for CreateGroupWithUserIds.

* Changed a few response status codes. Switched to correct permission.

* Added member count to ws payload for group when updating or creating.

* Adds feature flag checks for custom groups.

* Added middleware function to require license. Added config to disable custom groups.

* Change for function signature change of executePossiblyEmptyQuery.

* Lint fixes.

* Adds telemetry none comment.

* Adds translations.

* Migrated to sqlx.

* Temp. removal of translation.

* Fixed typo.

* Added an intermediary model to query with a field that is now ignored by sqlx on read queries.

* Re-used existing store struct.

* Inludes member count.

* Fix for merge error.'

* Require license for group endpoints.

* Updates translations.

* Fix shadow declaration.

* Renames permissions. Switches to new method to retrieve remoteid.

* Added WS events for upsert and delete member(s).

* Added new store error type ErrUniqueConstraint.

* Added EnableCustonGroups to the client config.

* Sanitized some user records.

* Added parameter to include_total_count for listing groups.

* Added translations.

* adding deleteAt field to getByUsers query

* Revert sanitize.

* Added uniqueness constraint error to UpdateGroup.

* Removed the FutureFeatures flag so that the feature is not enabled on old Enterprise licenses.

* Renamed function.

* Updates authz check for user search related to groups.

* Removed debug statement.

* Removed unused app method.

* Added telemetry for enable_custom_groups.

* Returns early from nil license.

* Updates test.

* Returned early to avoid nesting in (*SqlGroupStore).checkUserExist. Switched to reading from replica in (*SqlGroupStore).GetMember. Handled JSON marshal error in (*Client4).UpsertGroupMembers

* Switched to SanitizeProfile.

* Switched to model.NewInt.

* Switched from status NotImplemented to Forbidden for missing license.

* Removed deactivated users from 'exists' set.

* Revert gotool update.

* Ignored lint error that I think is invalid.

* Added the approprate access tag for disabling custom groups.

* Revert change to response status.

* Fixed refactor mistake.

* Limited the group member WS events to individual users.

* Removed WS event of deleted groups.

* Updated license check for searchUsers endpoint.

* Switched from license feature to license sku.

* Update app/group.go

Co-authored-by: Claudio Costa <cstcld91@gmail.com>

* Update app/group.go

Co-authored-by: Claudio Costa <cstcld91@gmail.com>

* Remove linter ignore comment.

* Added function to create sku-specific license.

* Fixed typo. Removed comment.

* Fixed for wrong type.

* Added missing param to client. Removed unnecessary props setting. Added test for retrieving groups by source.

* Updated some tests now that we're validating group membership not created for deactivated user.

* Fix for groups endpoint returning all group types by default.

* Changes constant names. Adds migration for all users to manage custom group members.

* Removes requirement for manage_system permission to filter user search by group.

* Added migration mock.

* Removes default permissions from custom_group_user role.

* Fixes migration.

* Fixes emoji migration test.

* fixing issue with member counts

* fixing search issue for deleted members

Co-authored-by: Benjamin Cooke <benjamincooke@Benjamins-MacBook-Pro.local>
Co-authored-by: Benjamin Cooke <benjamincooke@Benjamins-MBP.ht.home>
Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Co-authored-by: Benjamin Cooke <benjamincooke@Benjamins-MacBook-Pro.fritz.box>
Co-authored-by: Claudio Costa <cstcld91@gmail.com>

api4/user.go

@@ -19,6 +19,7 @@ import (
 	"github.com/mattermost/mattermost-server/v6/shared/mlog"
 	"github.com/mattermost/mattermost-server/v6/store"
 	"github.com/mattermost/mattermost-server/v6/utils"
+	"github.com/mattermost/mattermost-server/v6/web"
 )
 
 func (api *API) InitUser() {
@@ -641,6 +642,7 @@ func getUsers(c *Context, w http.ResponseWriter, r *http.Request) {
 	notInTeamId := r.URL.Query().Get("not_in_team")
 	inChannelId := r.URL.Query().Get("in_channel")
 	inGroupId := r.URL.Query().Get("in_group")
+	notInGroupId := r.URL.Query().Get("not_in_group")
 	notInChannelId := r.URL.Query().Get("not_in_channel")
 	groupConstrained := r.URL.Query().Get("group_constrained")
 	withoutTeam := r.URL.Query().Get("without_team")
@@ -664,7 +666,7 @@ func getUsers(c *Context, w http.ResponseWriter, r *http.Request) {
 
 	// Currently only supports sorting on a team
 	// or sort="status" on inChannelId
-	if (sort == "last_activity_at" || sort == "create_at") && (inTeamId == "" || notInTeamId != "" || inChannelId != "" || notInChannelId != "" || withoutTeam != "" || inGroupId != "") {
+	if (sort == "last_activity_at" || sort == "create_at") && (inTeamId == "" || notInTeamId != "" || inChannelId != "" || notInChannelId != "" || withoutTeam != "" || inGroupId != "" || notInGroupId != "") {
 		c.SetInvalidURLParam("sort")
 		return
 	}
@@ -720,6 +722,7 @@ func getUsers(c *Context, w http.ResponseWriter, r *http.Request) {
 		NotInTeamId:      notInTeamId,
 		NotInChannelId:   notInChannelId,
 		InGroupId:        inGroupId,
+		NotInGroupId:     notInGroupId,
 		GroupConstrained: groupConstrainedBool,
 		WithoutTeam:      withoutTeamBool,
 		Inactive:         inactiveBool,
@@ -807,13 +810,9 @@ func getUsers(c *Context, w http.ResponseWriter, r *http.Request) {
 			profiles, err = c.App.GetUsersInChannelPage(userGetOptions, c.IsSystemAdmin())
 		}
 	} else if inGroupId != "" {
-		if c.App.Srv().License() == nil || !*c.App.Srv().License().Features.LDAPGroups {
-			c.Err = model.NewAppError("Api4.getUsersInGroup", "api.ldap_groups.license_error", nil, "", http.StatusNotImplemented)
-			return
-		}
-
-		if !c.App.SessionHasPermissionTo(*c.AppContext.Session(), model.PermissionSysconsoleReadUserManagementGroups) {
-			c.SetPermissionError(model.PermissionSysconsoleReadUserManagementGroups)
+		if gErr := requireGroupAccess(c, inGroupId); gErr != nil {
+			gErr.Where = "Api.getUsers"
+			c.Err = gErr
 			return
 		}
 
@@ -822,6 +821,18 @@ func getUsers(c *Context, w http.ResponseWriter, r *http.Request) {
 			c.Err = err
 			return
 		}
+	} else if notInGroupId != "" {
+		if gErr := requireGroupAccess(c, notInGroupId); gErr != nil {
+			gErr.Where = "Api.getUsers"
+			c.Err = gErr
+			return
+		}
+
+		profiles, err = c.App.GetUsersNotInGroupPage(notInGroupId, c.Params.Page, c.Params.PerPage)
+		if err != nil {
+			c.Err = err
+			return
+		}
 	} else {
 		userGetOptions, err = c.App.RestrictUsersGetByPermissions(c.AppContext.Session().UserId, userGetOptions)
 		if err != nil {
@@ -850,6 +861,25 @@ func getUsers(c *Context, w http.ResponseWriter, r *http.Request) {
 	w.Write(js)
 }
 
+func requireGroupAccess(c *web.Context, groupID string) *model.AppError {
+	group, err := c.App.GetGroup(groupID, nil)
+	if err != nil {
+		return err
+	}
+
+	if lcErr := licensedAndConfiguredForGroupBySource(c.App, group.Source); lcErr != nil {
+		return lcErr
+	}
+
+	if group.Source == model.GroupSourceLdap {
+		if !c.App.SessionHasPermissionTo(*c.AppContext.Session(), model.PermissionSysconsoleReadUserManagementGroups) {
+			return c.App.MakePermissionError(c.AppContext.Session(), []*model.Permission{model.PermissionSysconsoleReadUserManagementGroups})
+		}
+	}
+
+	return nil
+}
+
 func getUsersByIds(c *Context, w http.ResponseWriter, r *http.Request) {
 	userIds := model.ArrayFromJSON(r.Body)
 
@@ -958,13 +988,17 @@ func searchUsers(c *Context, w http.ResponseWriter, r *http.Request) {
 	}
 
 	if props.InGroupId != "" {
-		if c.App.Srv().License() == nil || !*c.App.Srv().License().Features.LDAPGroups {
-			c.Err = model.NewAppError("Api4.searchUsers", "api.ldap_groups.license_error", nil, "", http.StatusNotImplemented)
+		if gErr := requireGroupAccess(c, props.InGroupId); gErr != nil {
+			gErr.Where = "Api.searchUsers"
+			c.Err = gErr
 			return
 		}
+	}
 
-		if !c.App.SessionHasPermissionTo(*c.AppContext.Session(), model.PermissionManageSystem) {
-			c.SetPermissionError(model.PermissionManageSystem)
+	if props.NotInGroupId != "" {
+		if gErr := requireGroupAccess(c, props.NotInGroupId); gErr != nil {
+			gErr.Where = "Api.searchUsers"
+			c.Err = gErr
 			return
 		}
 	}
@@ -1904,7 +1938,7 @@ func loginCWS(c *Context, w http.ResponseWriter, r *http.Request) {
 	if err != nil {
 		c.LogAuditWithUserId("", "failure - login_id="+loginID)
 		c.LogErrorByCode(err)
-		http.Redirect(w, r, *c.App.Config().ServiceSettings.SiteURL, 302)
+		http.Redirect(w, r, *c.App.Config().ServiceSettings.SiteURL, http.StatusNotFound)
 		return
 	}
 	auditRec.AddMeta("user", user)
@@ -1912,12 +1946,12 @@ func loginCWS(c *Context, w http.ResponseWriter, r *http.Request) {
 	err = c.App.DoLogin(c.AppContext, w, r, user, "", false, false, false)
 	if err != nil {
 		c.LogErrorByCode(err)
-		http.Redirect(w, r, *c.App.Config().ServiceSettings.SiteURL, 302)
+		http.Redirect(w, r, *c.App.Config().ServiceSettings.SiteURL, http.StatusNotFound)
 		return
 	}
 	c.LogAuditWithUserId(user.Id, "success")
 	c.App.AttachSessionCookies(c.AppContext, w, r)
-	http.Redirect(w, r, *c.App.Config().ServiceSettings.SiteURL, 302)
+	http.Redirect(w, r, *c.App.Config().ServiceSettings.SiteURL, http.StatusNotFound)
 }
 
 func logout(c *Context, w http.ResponseWriter, r *http.Request) {