Custom groups (#18839)

* WIP

* adding initial creategroup endpoint

* fetching by group source

* fixing startup error

* updating create endpoint to take an array of user_ids, this will allow us to create the group with one request

* adding delete group endpoint and appropriate test

* adding source param for getGroups

* adding add members and delete members endpoints

* locking down crud endpoints to only be allowed for custom groups

* user search stuff

* allowing remoteid be null by changing field to pointer

* code cleanup and store level tests

* adding new tests and removing unused endpoint

* resolving conflicts

* Adds authz check for group.

* Adds authz checks to groups APIs.

* Updated create group authz tests.

* Updates delete group tests.

* Tests create group.

* Adds some tests and validations.

* adding new parameter so I can get users not in a group

* Fixed all lint warnings.

* Fix type.

* fixing search users not in group

* Fixes some lint errors.

* Moves entry in JSON array.

* Fixed SQL query.

* Fixes permission migration test.

* Fixes migration test.

* Fixes some group store tests.

* Fix test.

* Fix test.

* Revert lint change.

* Migrated CreateWithUserIds to sqlx.

* Adds tests for GetMember; migrates implementation to sqlx.

* Tests GetNonMemberUsersPage and hanles wrong group id.

* Fixes test.

* Switches GetMaster to GetMasterX.

* Switches GetReplica to GetReplicaX.

* Fixes logic.

* Fixes shadow declaration.

* Adds include_member_count to get group API endpoint.

* Adds filter_has_member param to getGroups.

* Fixes.

* Removes array of group sources.

* fixing error

* Testing reverting CreateWithUserIds back to gorp.

* Added websocket event for CreateGroupWithUserIds.

* Changed a few response status codes. Switched to correct permission.

* Added member count to ws payload for group when updating or creating.

* Adds feature flag checks for custom groups.

* Added middleware function to require license. Added config to disable custom groups.

* Change for function signature change of executePossiblyEmptyQuery.

* Lint fixes.

* Adds telemetry none comment.

* Adds translations.

* Migrated to sqlx.

* Temp. removal of translation.

* Fixed typo.

* Added an intermediary model to query with a field that is now ignored by sqlx on read queries.

* Re-used existing store struct.

* Inludes member count.

* Fix for merge error.'

* Require license for group endpoints.

* Updates translations.

* Fix shadow declaration.

* Renames permissions. Switches to new method to retrieve remoteid.

* Added WS events for upsert and delete member(s).

* Added new store error type ErrUniqueConstraint.

* Added EnableCustonGroups to the client config.

* Sanitized some user records.

* Added parameter to include_total_count for listing groups.

* Added translations.

* adding deleteAt field to getByUsers query

* Revert sanitize.

* Added uniqueness constraint error to UpdateGroup.

* Removed the FutureFeatures flag so that the feature is not enabled on old Enterprise licenses.

* Renamed function.

* Updates authz check for user search related to groups.

* Removed debug statement.

* Removed unused app method.

* Added telemetry for enable_custom_groups.

* Returns early from nil license.

* Updates test.

* Returned early to avoid nesting in (*SqlGroupStore).checkUserExist. Switched to reading from replica in (*SqlGroupStore).GetMember. Handled JSON marshal error in (*Client4).UpsertGroupMembers

* Switched to SanitizeProfile.

* Switched to model.NewInt.

* Switched from status NotImplemented to Forbidden for missing license.

* Removed deactivated users from 'exists' set.

* Revert gotool update.

* Ignored lint error that I think is invalid.

* Added the approprate access tag for disabling custom groups.

* Revert change to response status.

* Fixed refactor mistake.

* Limited the group member WS events to individual users.

* Removed WS event of deleted groups.

* Updated license check for searchUsers endpoint.

* Switched from license feature to license sku.

* Update app/group.go

Co-authored-by: Claudio Costa <cstcld91@gmail.com>

* Update app/group.go

Co-authored-by: Claudio Costa <cstcld91@gmail.com>

* Remove linter ignore comment.

* Added function to create sku-specific license.

* Fixed typo. Removed comment.

* Fixed for wrong type.

* Added missing param to client. Removed unnecessary props setting. Added test for retrieving groups by source.

* Updated some tests now that we're validating group membership not created for deactivated user.

* Fix for groups endpoint returning all group types by default.

* Changes constant names. Adds migration for all users to manage custom group members.

* Removes requirement for manage_system permission to filter user search by group.

* Added migration mock.

* Removes default permissions from custom_group_user role.

* Fixes migration.

* Fixes emoji migration test.

* fixing issue with member counts

* fixing search issue for deleted members

Co-authored-by: Benjamin Cooke <benjamincooke@Benjamins-MacBook-Pro.local>
Co-authored-by: Benjamin Cooke <benjamincooke@Benjamins-MBP.ht.home>
Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Co-authored-by: Benjamin Cooke <benjamincooke@Benjamins-MacBook-Pro.fritz.box>
Co-authored-by: Claudio Costa <cstcld91@gmail.com>

app/app_iface.go

@@ -450,6 +450,7 @@ type AppIface interface {
 	CreateEmoji(sessionUserId string, emoji *model.Emoji, multiPartImageData *multipart.Form) (*model.Emoji, *model.AppError)
 	CreateGroup(group *model.Group) (*model.Group, *model.AppError)
 	CreateGroupChannel(userIDs []string, creatorId string) (*model.Channel, *model.AppError)
+	CreateGroupWithUserIds(group *model.GroupWithUserIds) (*model.Group, *model.AppError)
 	CreateIncomingWebhookForChannel(creatorId string, channel *model.Channel, hook *model.IncomingWebhook) (*model.IncomingWebhook, *model.AppError)
 	CreateJob(job *model.Job) (*model.Job, *model.AppError)
 	CreateOAuthApp(app *model.OAuthApp) (*model.OAuthApp, *model.AppError)
@@ -491,6 +492,7 @@ type AppIface interface {
 	DeleteExport(name string) *model.AppError
 	DeleteGroup(groupID string) (*model.Group, *model.AppError)
 	DeleteGroupMember(groupID string, userID string) (*model.GroupMember, *model.AppError)
+	DeleteGroupMembers(groupID string, userIDs []string) ([]*model.GroupMember, *model.AppError)
 	DeleteGroupSyncable(groupID string, syncableID string, syncableType model.GroupSyncableType) (*model.GroupSyncable, *model.AppError)
 	DeleteIncomingWebhook(hookID string) *model.AppError
 	DeleteOAuthApp(appID string) *model.AppError
@@ -604,7 +606,7 @@ type AppIface interface {
 	GetFlaggedPostsForChannel(userID, channelID string, offset int, limit int) (*model.PostList, *model.AppError)
 	GetFlaggedPostsForTeam(userID, teamID string, offset int, limit int) (*model.PostList, *model.AppError)
 	GetGlobalRetentionPolicy() (*model.GlobalRetentionPolicy, *model.AppError)
-	GetGroup(id string) (*model.Group, *model.AppError)
+	GetGroup(id string, opts *model.GetGroupOpts) (*model.Group, *model.AppError)
 	GetGroupByName(name string, opts model.GroupSearchOpts) (*model.Group, *model.AppError)
 	GetGroupByRemoteID(remoteID string, groupSource model.GroupSource) (*model.Group, *model.AppError)
 	GetGroupChannel(userIDs []string) (*model.Channel, *model.AppError)
@@ -782,6 +784,7 @@ type AppIface interface {
 	GetUsersNotInChannel(teamID string, channelID string, groupConstrained bool, offset int, limit int, viewRestrictions *model.ViewUsersRestrictions) ([]*model.User, *model.AppError)
 	GetUsersNotInChannelMap(teamID string, channelID string, groupConstrained bool, offset int, limit int, asAdmin bool, viewRestrictions *model.ViewUsersRestrictions) (map[string]*model.User, *model.AppError)
 	GetUsersNotInChannelPage(teamID string, channelID string, groupConstrained bool, page int, perPage int, asAdmin bool, viewRestrictions *model.ViewUsersRestrictions) ([]*model.User, *model.AppError)
+	GetUsersNotInGroupPage(groupID string, page int, perPage int) ([]*model.User, *model.AppError)
 	GetUsersNotInTeam(teamID string, groupConstrained bool, offset int, limit int, viewRestrictions *model.ViewUsersRestrictions) ([]*model.User, *model.AppError)
 	GetUsersNotInTeamEtag(teamID string, restrictionsHash string) string
 	GetUsersNotInTeamPage(teamID string, groupConstrained bool, page int, perPage int, asAdmin bool, viewRestrictions *model.ViewUsersRestrictions) ([]*model.User, *model.AppError)
@@ -960,6 +963,7 @@ type AppIface interface {
 	SearchUsersInGroup(groupID string, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError)
 	SearchUsersInTeam(teamID, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError)
 	SearchUsersNotInChannel(teamID string, channelID string, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError)
+	SearchUsersNotInGroup(groupID string, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError)
 	SearchUsersNotInTeam(notInTeamId string, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError)
 	SearchUsersWithoutTeam(term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError)
 	SendAckToPushProxy(ack *model.PushNotificationAck) error
@@ -979,6 +983,7 @@ type AppIface interface {
 	SessionHasPermissionToChannel(session model.Session, channelID string, permission *model.Permission) bool
 	SessionHasPermissionToChannelByPost(session model.Session, postID string, permission *model.Permission) bool
 	SessionHasPermissionToCreateJob(session model.Session, job *model.Job) (bool, *model.Permission)
+	SessionHasPermissionToGroup(session model.Session, groupID string, permission *model.Permission) bool
 	SessionHasPermissionToReadJob(session model.Session, jobType string) (bool, *model.Permission)
 	SessionHasPermissionToTeam(session model.Session, teamID string, permission *model.Permission) bool
 	SessionHasPermissionToUser(session model.Session, userID string) bool
@@ -1082,6 +1087,7 @@ type AppIface interface {
 	UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppError
 	UploadMultipartFiles(c *request.Context, teamID string, channelID string, userID string, fileHeaders []*multipart.FileHeader, clientIds []string, now time.Time) (*model.FileUploadResponse, *model.AppError)
 	UpsertGroupMember(groupID string, userID string) (*model.GroupMember, *model.AppError)
+	UpsertGroupMembers(groupID string, userIDs []string) ([]*model.GroupMember, *model.AppError)
 	UpsertGroupSyncable(groupSyncable *model.GroupSyncable) (*model.GroupSyncable, *model.AppError)
 	UserCanSeeOtherUser(userID string, otherUserId string) (bool, *model.AppError)
 	VerifyEmailFromToken(userSuppliedTokenString string) *model.AppError

app/app_test.go

@@ -164,6 +164,10 @@ func TestDoAdvancedPermissionsMigration(t *testing.T) {
 			model.PermissionCreateGroupChannel.Id,
 			model.PermissionViewMembers.Id,
 			model.PermissionCreateTeam.Id,
+			model.PermissionCreateCustomGroup.Id,
+			model.PermissionEditCustomGroup.Id,
+			model.PermissionDeleteCustomGroup.Id,
+			model.PermissionManageCustomGroupMembers.Id,
 		},
 		"system_post_all": {
 			model.PermissionCreatePost.Id,
@@ -219,6 +223,10 @@ func TestDoEmojisPermissionsMigration(t *testing.T) {
 	role3, err3 := th.App.GetRoleByName(context.Background(), model.SystemUserRoleId)
 	assert.Nil(t, err3)
 	expected3 := []string{
+		model.PermissionCreateCustomGroup.Id,
+		model.PermissionEditCustomGroup.Id,
+		model.PermissionDeleteCustomGroup.Id,
+		model.PermissionManageCustomGroupMembers.Id,
 		model.PermissionListPublicTeams.Id,
 		model.PermissionJoinPublicTeams.Id,
 		model.PermissionCreateDirectChannel.Id,

app/authorization.go

@@ -5,6 +5,8 @@ package app
 
 import (
 	"context"
+	"database/sql"
+	"errors"
 	"net/http"
 	"strings"
 
@@ -88,6 +90,25 @@ func (a *App) SessionHasPermissionToChannel(session model.Session, channelID str
 	return a.SessionHasPermissionTo(session, permission)
 }
 
+func (a *App) SessionHasPermissionToGroup(session model.Session, groupID string, permission *model.Permission) bool {
+	groupMember, err := a.Srv().Store.Group().GetMember(groupID, session.UserId)
+	// don't reject immediately on ErrNoRows error because there's further authz logic below for non-groupmembers
+	if err != nil && !errors.Is(err, sql.ErrNoRows) {
+		return false
+	}
+
+	// each member of a group is implicitly considered to have the 'custom_group_user' role in that group, so if the user is a member of the
+	// group and custom_group_user on their system has the requested permission then return true
+	if groupMember != nil && a.RolesGrantPermission([]string{model.CustomGroupUserRoleId}, permission.Id) {
+		return true
+	}
+
+	// Not implemented: group-override schemes.
+
+	// ...otherwise check their system roles to see if they have the requested permission system-wide
+	return a.SessionHasPermissionTo(session, permission)
+}
+
 func (a *App) SessionHasPermissionToChannelByPost(session model.Session, postID string, permission *model.Permission) bool {
 	if channelMember, err := a.Srv().Store.Channel().GetMemberForPost(postID, session.UserId); err == nil {
 

app/authorization_test.go

@@ -4,7 +4,13 @@
 package app
 
 import (
+	"context"
+	"encoding/csv"
 	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -118,3 +124,87 @@ func TestHasPermissionToCategory(t *testing.T) {
 	require.Nil(t, err)
 	require.False(t, th.App.SessionHasPermissionToCategory(*session, th.BasicUser.Id, th.BasicTeam.Id, categories2.Order[0]))
 }
+
+func TestSessionHasPermissionToGroup(t *testing.T) {
+	th := Setup(t).InitBasic()
+	defer th.TearDown()
+
+	file, e := os.Open("tests/group-role-has-permission.csv")
+	require.NoError(t, e)
+	defer file.Close()
+
+	b, e := ioutil.ReadAll(file)
+	require.NoError(t, e)
+
+	r := csv.NewReader(strings.NewReader(string(b)))
+	records, e := r.ReadAll()
+	require.NoError(t, e)
+
+	systemRole, err := th.App.GetRoleByName(context.Background(), model.SystemUserRoleId)
+	require.Nil(t, err)
+
+	groupRole, err := th.App.GetRoleByName(context.Background(), model.CustomGroupUserRoleId)
+	require.Nil(t, err)
+
+	group, err := th.App.CreateGroup(&model.Group{
+		Name:           model.NewString(model.NewId()),
+		DisplayName:    model.NewId(),
+		Source:         model.GroupSourceCustom,
+		AllowReference: true,
+	})
+	require.Nil(t, err)
+
+	permission := model.PermissionDeleteCustomGroup
+
+	for i, row := range records {
+		// skip csv header
+		if i == 0 {
+			continue
+		}
+
+		systemRoleHasPermission, e := strconv.ParseBool(row[0])
+		require.NoError(t, e)
+
+		isGroupMember, e := strconv.ParseBool(row[1])
+		require.NoError(t, e)
+
+		groupRoleHasPermission, e := strconv.ParseBool(row[2])
+		require.NoError(t, e)
+
+		permissionShouldBeGranted, e := strconv.ParseBool(row[3])
+		require.NoError(t, e)
+
+		if systemRoleHasPermission {
+			th.AddPermissionToRole(permission.Id, systemRole.Name)
+		} else {
+			th.RemovePermissionFromRole(permission.Id, systemRole.Name)
+		}
+
+		if isGroupMember {
+			_, err := th.App.UpsertGroupMember(group.Id, th.BasicUser.Id)
+			require.Nil(t, err)
+		} else {
+			_, err := th.App.DeleteGroupMember(group.Id, th.BasicUser.Id)
+			if err != nil && err.Id != "app.group.no_rows" {
+				t.Error(err)
+			}
+		}
+
+		if groupRoleHasPermission {
+			th.AddPermissionToRole(permission.Id, groupRole.Name)
+		} else {
+			th.RemovePermissionFromRole(permission.Id, groupRole.Name)
+		}
+
+		session, err := th.App.CreateSession(&model.Session{UserId: th.BasicUser.Id, Props: model.StringMap{}, Roles: systemRole.Name})
+		require.Nil(t, err)
+
+		result := th.App.SessionHasPermissionToGroup(*session, group.Id, permission)
+
+		if permissionShouldBeGranted {
+			require.True(t, result, fmt.Sprintf("row: %v", row))
+		} else {
+			require.False(t, result, fmt.Sprintf("row: %v", row))
+		}
+	}
+}

app/group.go

@@ -13,7 +13,7 @@ import (
 	"github.com/mattermost/mattermost-server/v6/store"
 )
 
-func (a *App) GetGroup(id string) (*model.Group, *model.AppError) {
+func (a *App) GetGroup(id string, opts *model.GetGroupOpts) (*model.Group, *model.AppError) {
 	group, err := a.Srv().Store.Group().Get(id)
 	if err != nil {
 		var nfErr *store.ErrNotFound
@@ -25,6 +25,14 @@ func (a *App) GetGroup(id string) (*model.Group, *model.AppError) {
 		}
 	}
 
+	if opts != nil && opts.IncludeMemberCount {
+		memberCount, err := a.Srv().Store.Group().GetMemberCount(id)
+		if err != nil {
+			return nil, model.NewAppError("GetGroup", "app.member_count", nil, err.Error(), http.StatusInternalServerError)
+		}
+		group.MemberCount = model.NewInt(int(memberCount))
+	}
+
 	return group, nil
 }
 
@@ -94,10 +102,49 @@ func (a *App) CreateGroup(group *model.Group) (*model.Group, *model.AppError) {
 	return group, nil
 }
 
+func (a *App) CreateGroupWithUserIds(group *model.GroupWithUserIds) (*model.Group, *model.AppError) {
+	newGroup, err := a.Srv().Store.Group().CreateWithUserIds(group)
+	if err != nil {
+		var invErr *store.ErrInvalidInput
+		var appErr *model.AppError
+		var dupKey *store.ErrUniqueConstraint
+		switch {
+		case errors.As(err, &appErr):
+			return nil, appErr
+		case errors.As(err, &invErr):
+			return nil, model.NewAppError("CreateGroupWithUserIds", "app.group.id.app_error", nil, invErr.Error(), http.StatusBadRequest)
+		case errors.As(err, &dupKey):
+			return nil, model.NewAppError("CreateGroup", "app.custom_group.unique_name", nil, dupKey.Error(), http.StatusBadRequest)
+		default:
+			return nil, model.NewAppError("CreateGroupWithUserIds", "app.insert_error", nil, err.Error(), http.StatusInternalServerError)
+		}
+	}
+
+	messageWs := model.NewWebSocketEvent(model.WebsocketEventReceivedGroup, "", "", "", nil)
+	count, err := a.Srv().Store.Group().GetMemberCount(newGroup.Id)
+	if err != nil {
+		return nil, model.NewAppError("CreateGroupWithUserIds", "app.group.id.app_error", nil, err.Error(), http.StatusBadRequest)
+	}
+	group.MemberCount = model.NewInt(int(count))
+	groupJSON, jsonErr := json.Marshal(newGroup)
+	if jsonErr != nil {
+		mlog.Warn("Failed to encode group to JSON", mlog.Err(jsonErr))
+	}
+	messageWs.Add("group", string(groupJSON))
+	a.Publish(messageWs)
+
+	return newGroup, nil
+}
+
 func (a *App) UpdateGroup(group *model.Group) (*model.Group, *model.AppError) {
 	updatedGroup, err := a.Srv().Store.Group().Update(group)
 
 	if err == nil {
+		count, countErr := a.Srv().Store.Group().GetMemberCount(updatedGroup.Id)
+		if countErr != nil {
+			return nil, model.NewAppError("CreateGroupWithUserIds", "app.group.id.app_error", nil, countErr.Error(), http.StatusBadRequest)
+		}
+		updatedGroup.MemberCount = model.NewInt(int(count))
 		messageWs := model.NewWebSocketEvent(model.WebsocketEventReceivedGroup, "", "", "", nil)
 		groupJSON, jsonErr := json.Marshal(updatedGroup)
 		if jsonErr != nil {
@@ -110,11 +157,14 @@ func (a *App) UpdateGroup(group *model.Group) (*model.Group, *model.AppError) {
 	if err != nil {
 		var nfErr *store.ErrNotFound
 		var appErr *model.AppError
+		var dupKey *store.ErrUniqueConstraint
 		switch {
 		case errors.As(err, &appErr):
 			return nil, appErr
 		case errors.As(err, &nfErr):
 			return nil, model.NewAppError("UpdateGroup", "app.group.no_rows", nil, nfErr.Error(), http.StatusNotFound)
+		case errors.As(err, &dupKey):
+			return nil, model.NewAppError("CreateGroup", "app.custom_group.unique_name", nil, dupKey.Error(), http.StatusBadRequest)
 		default:
 			return nil, model.NewAppError("UpdateGroup", "app.select_error", nil, err.Error(), http.StatusInternalServerError)
 		}
@@ -125,17 +175,6 @@ func (a *App) UpdateGroup(group *model.Group) (*model.Group, *model.AppError) {
 
 func (a *App) DeleteGroup(groupID string) (*model.Group, *model.AppError) {
 	deletedGroup, err := a.Srv().Store.Group().Delete(groupID)
-
-	if err == nil {
-		messageWs := model.NewWebSocketEvent(model.WebsocketEventReceivedGroup, "", "", "", nil)
-		groupJSON, jsonErr := json.Marshal(deletedGroup)
-		if jsonErr != nil {
-			mlog.Warn("Failed to encode group to JSON", mlog.Err(jsonErr))
-		}
-		messageWs.Add("group", string(groupJSON))
-		a.Publish(messageWs)
-	}
-
 	if err != nil {
 		var nfErr *store.ErrNotFound
 		switch {
@@ -177,7 +216,15 @@ func (a *App) GetGroupMemberUsersPage(groupID string, page int, perPage int) ([]
 	if appErr != nil {
 		return nil, 0, appErr
 	}
-	return members, int(count), nil
+	return a.sanitizeProfiles(members, false), int(count), nil
+}
+func (a *App) GetUsersNotInGroupPage(groupID string, page int, perPage int) ([]*model.User, *model.AppError) {
+	members, err := a.Srv().Store.Group().GetNonMemberUsersPage(groupID, page, perPage)
+	if err != nil {
+		return nil, model.NewAppError("GetUsersNotInGroupPage", "app.select_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+
+	return a.sanitizeProfiles(members, false), nil
 }
 
 func (a *App) UpsertGroupMember(groupID string, userID string) (*model.GroupMember, *model.AppError) {
@@ -195,6 +242,8 @@ func (a *App) UpsertGroupMember(groupID string, userID string) (*model.GroupMemb
 		}
 	}
 
+	a.publishGroupMemberEvent(model.WebsocketEventGroupMemberAdd, groupMember)
+
 	return groupMember, nil
 }
 
@@ -210,6 +259,8 @@ func (a *App) DeleteGroupMember(groupID string, userID string) (*model.GroupMemb
 		}
 	}
 
+	a.publishGroupMemberEvent(model.WebsocketEventGroupMemberDelete, groupMember)
+
 	return groupMember, nil
 }
 
@@ -515,6 +566,10 @@ func (a *App) TeamMembersMinusGroupMembers(teamID string, groupIDs []string, pag
 		return nil, 0, model.NewAppError("TeamMembersMinusGroupMembers", "app.select_error", nil, err.Error(), http.StatusInternalServerError)
 	}
 
+	for _, u := range users {
+		a.SanitizeProfile(&u.User, false)
+	}
+
 	// parse all group ids of all users
 	allUsersGroupIDMap := map[string]bool{}
 	for _, user := range users {
@@ -579,6 +634,10 @@ func (a *App) ChannelMembersMinusGroupMembers(channelID string, groupIDs []strin
 		return nil, 0, model.NewAppError("ChannelMembersMinusGroupMembers", "app.select_error", nil, err.Error(), http.StatusInternalServerError)
 	}
 
+	for _, u := range users {
+		a.SanitizeProfile(&u.User, false)
+	}
+
 	// parse all group ids of all users
 	allUsersGroupIDMap := map[string]bool{}
 	for _, user := range users {
@@ -637,3 +696,57 @@ func (a *App) UserIsInAdminRoleGroup(userID, syncableID string, syncableType mod
 
 	return true, nil
 }
+
+func (a *App) UpsertGroupMembers(groupID string, userIDs []string) ([]*model.GroupMember, *model.AppError) {
+	members, err := a.Srv().Store.Group().UpsertMembers(groupID, userIDs)
+	if err != nil {
+		var invErr *store.ErrInvalidInput
+		var appErr *model.AppError
+		switch {
+		case errors.As(err, &appErr):
+			return nil, appErr
+		case errors.As(err, &invErr):
+			return nil, model.NewAppError("UpsertGroupMembers", "app.group.uniqueness_error", nil, invErr.Error(), http.StatusBadRequest)
+		default:
+			return nil, model.NewAppError("UpsertGroupMembers", "app.update_error", nil, err.Error(), http.StatusInternalServerError)
+		}
+	}
+
+	for _, groupMember := range members {
+		a.publishGroupMemberEvent(model.WebsocketEventGroupMemberAdd, groupMember)
+	}
+
+	return members, nil
+}
+
+func (a *App) DeleteGroupMembers(groupID string, userIDs []string) ([]*model.GroupMember, *model.AppError) {
+	members, err := a.Srv().Store.Group().DeleteMembers(groupID, userIDs)
+	if err != nil {
+		var invErr *store.ErrInvalidInput
+		var appErr *model.AppError
+		switch {
+		case errors.As(err, &appErr):
+			return nil, appErr
+		case errors.As(err, &invErr):
+			return nil, model.NewAppError("DeleteGroupMember", "app.group.uniqueness_error", nil, invErr.Error(), http.StatusBadRequest)
+		default:
+			return nil, model.NewAppError("DeleteGroupMember", "app.update_error", nil, err.Error(), http.StatusInternalServerError)
+		}
+	}
+
+	for _, groupMember := range members {
+		a.publishGroupMemberEvent(model.WebsocketEventGroupMemberDelete, groupMember)
+	}
+
+	return members, nil
+}
+
+func (a *App) publishGroupMemberEvent(eventName string, groupMember *model.GroupMember) {
+	messageWs := model.NewWebSocketEvent(eventName, "", "", groupMember.UserId, nil)
+	groupMemberJSON, jsonErr := json.Marshal(groupMember)
+	if jsonErr != nil {
+		mlog.Warn("failed to encode group member to JSON", mlog.Err(jsonErr))
+	}
+	messageWs.Add("group_member", string(groupMemberJSON))
+	a.Publish(messageWs)
+}

app/group_test.go

@@ -17,13 +17,21 @@ func TestGetGroup(t *testing.T) {
 	defer th.TearDown()
 	group := th.CreateGroup()
 
-	group, err := th.App.GetGroup(group.Id)
+	group, err := th.App.GetGroup(group.Id, nil)
 	require.Nil(t, err)
 	require.NotNil(t, group)
 
-	group, err = th.App.GetGroup(model.NewId())
+	nilGroup, err := th.App.GetGroup(model.NewId(), nil)
 	require.NotNil(t, err)
-	require.Nil(t, group)
+	require.Nil(t, nilGroup)
+
+	group, err = th.App.GetGroup(group.Id, &model.GetGroupOpts{IncludeMemberCount: false})
+	require.Nil(t, err)
+	require.Nil(t, group.MemberCount)
+
+	group, err = th.App.GetGroup(group.Id, &model.GetGroupOpts{IncludeMemberCount: true})
+	require.Nil(t, err)
+	require.NotNil(t, group.MemberCount)
 }
 
 func TestGetGroupByRemoteID(t *testing.T) {
@@ -31,7 +39,7 @@ func TestGetGroupByRemoteID(t *testing.T) {
 	defer th.TearDown()
 	group := th.CreateGroup()
 
-	g, err := th.App.GetGroupByRemoteID(group.RemoteId, model.GroupSourceLdap)
+	g, err := th.App.GetGroupByRemoteID(*group.RemoteId, model.GroupSourceLdap)
 	require.Nil(t, err)
 	require.NotNil(t, g)
 
@@ -65,7 +73,7 @@ func TestCreateGroup(t *testing.T) {
 		DisplayName: "dn_" + id,
 		Name:        model.NewString("name" + id),
 		Source:      model.GroupSourceLdap,
-		RemoteId:    model.NewId(),
+		RemoteId:    model.NewString(model.NewId()),
 	}
 
 	g, err := th.App.CreateGroup(group)

app/helper_test.go

@@ -479,7 +479,7 @@ func (th *TestHelper) CreateGroup() *model.Group {
 		Name:        model.NewString("name" + id),
 		Source:      model.GroupSourceLdap,
 		Description: "description_" + id,
-		RemoteId:    model.NewId(),
+		RemoteId:    model.NewString(model.NewId()),
 	}
 
 	var err *model.AppError

app/notification.go

@@ -1060,7 +1060,7 @@ func (a *App) allowGroupMentions(post *model.Post) bool {
 func (a *App) getGroupsAllowedForReferenceInChannel(channel *model.Channel, team *model.Team) (map[string]*model.Group, error) {
 	var err error
 	groupsMap := make(map[string]*model.Group)
-	opts := model.GroupSearchOpts{FilterAllowReference: true}
+	opts := model.GroupSearchOpts{FilterAllowReference: true, IncludeMemberCount: true}
 
 	if channel.IsGroupConstrained() || (team != nil && team.IsGroupConstrained()) {
 		var groups []*model.GroupWithSchemeAdmin

app/opentracing/opentracing_layer.go

@@ -2044,6 +2044,28 @@ func (a *OpenTracingAppLayer) CreateGroupChannel(userIDs []string, creatorId str
 	return resultVar0, resultVar1
 }
 
+func (a *OpenTracingAppLayer) CreateGroupWithUserIds(group *model.GroupWithUserIds) (*model.Group, *model.AppError) {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.CreateGroupWithUserIds")
+
+	a.ctx = newCtx
+	a.app.Srv().Store.SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store.SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0, resultVar1 := a.app.CreateGroupWithUserIds(group)
+
+	if resultVar1 != nil {
+		span.LogFields(spanlog.Error(resultVar1))
+		ext.Error.Set(span, true)
+	}
+
+	return resultVar0, resultVar1
+}
+
 func (a *OpenTracingAppLayer) CreateGuest(c *request.Context, user *model.User) (*model.User, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.CreateGuest")
@@ -3044,6 +3066,28 @@ func (a *OpenTracingAppLayer) DeleteGroupMember(groupID string, userID string) (
 	return resultVar0, resultVar1
 }
 
+func (a *OpenTracingAppLayer) DeleteGroupMembers(groupID string, userIDs []string) ([]*model.GroupMember, *model.AppError) {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.DeleteGroupMembers")
+
+	a.ctx = newCtx
+	a.app.Srv().Store.SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store.SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0, resultVar1 := a.app.DeleteGroupMembers(groupID, userIDs)
+
+	if resultVar1 != nil {
+		span.LogFields(spanlog.Error(resultVar1))
+		ext.Error.Set(span, true)
+	}
+
+	return resultVar0, resultVar1
+}
+
 func (a *OpenTracingAppLayer) DeleteGroupSyncable(groupID string, syncableID string, syncableType model.GroupSyncableType) (*model.GroupSyncable, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.DeleteGroupSyncable")
@@ -5975,7 +6019,7 @@ func (a *OpenTracingAppLayer) GetGlobalRetentionPolicy() (*model.GlobalRetention
 	return resultVar0, resultVar1
 }
 
-func (a *OpenTracingAppLayer) GetGroup(id string) (*model.Group, *model.AppError) {
+func (a *OpenTracingAppLayer) GetGroup(id string, opts *model.GetGroupOpts) (*model.Group, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetGroup")
 
@@ -5987,7 +6031,7 @@ func (a *OpenTracingAppLayer) GetGroup(id string) (*model.Group, *model.AppError
 	}()
 
 	defer span.Finish()
-	resultVar0, resultVar1 := a.app.GetGroup(id)
+	resultVar0, resultVar1 := a.app.GetGroup(id, opts)
 
 	if resultVar1 != nil {
 		span.LogFields(spanlog.Error(resultVar1))
@@ -10185,6 +10229,28 @@ func (a *OpenTracingAppLayer) GetUsersNotInChannelPage(teamID string, channelID
 	return resultVar0, resultVar1
 }
 
+func (a *OpenTracingAppLayer) GetUsersNotInGroupPage(groupID string, page int, perPage int) ([]*model.User, *model.AppError) {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetUsersNotInGroupPage")
+
+	a.ctx = newCtx
+	a.app.Srv().Store.SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store.SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0, resultVar1 := a.app.GetUsersNotInGroupPage(groupID, page, perPage)
+
+	if resultVar1 != nil {
+		span.LogFields(spanlog.Error(resultVar1))
+		ext.Error.Set(span, true)
+	}
+
+	return resultVar0, resultVar1
+}
+
 func (a *OpenTracingAppLayer) GetUsersNotInTeam(teamID string, groupConstrained bool, offset int, limit int, viewRestrictions *model.ViewUsersRestrictions) ([]*model.User, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetUsersNotInTeam")
@@ -14224,6 +14290,28 @@ func (a *OpenTracingAppLayer) SearchUsersNotInChannel(teamID string, channelID s
 	return resultVar0, resultVar1
 }
 
+func (a *OpenTracingAppLayer) SearchUsersNotInGroup(groupID string, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError) {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SearchUsersNotInGroup")
+
+	a.ctx = newCtx
+	a.app.Srv().Store.SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store.SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0, resultVar1 := a.app.SearchUsersNotInGroup(groupID, term, options)
+
+	if resultVar1 != nil {
+		span.LogFields(spanlog.Error(resultVar1))
+		ext.Error.Set(span, true)
+	}
+
+	return resultVar0, resultVar1
+}
+
 func (a *OpenTracingAppLayer) SearchUsersNotInTeam(notInTeamId string, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SearchUsersNotInTeam")
@@ -14644,6 +14732,23 @@ func (a *OpenTracingAppLayer) SessionHasPermissionToCreateJob(session model.Sess
 	return resultVar0, resultVar1
 }
 
+func (a *OpenTracingAppLayer) SessionHasPermissionToGroup(session model.Session, groupID string, permission *model.Permission) bool {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SessionHasPermissionToGroup")
+
+	a.ctx = newCtx
+	a.app.Srv().Store.SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store.SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0 := a.app.SessionHasPermissionToGroup(session, groupID, permission)
+
+	return resultVar0
+}
+
 func (a *OpenTracingAppLayer) SessionHasPermissionToManageBot(session model.Session, botUserId string) *model.AppError {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SessionHasPermissionToManageBot")
@@ -17163,6 +17268,28 @@ func (a *OpenTracingAppLayer) UpsertGroupMember(groupID string, userID string) (
 	return resultVar0, resultVar1
 }
 
+func (a *OpenTracingAppLayer) UpsertGroupMembers(groupID string, userIDs []string) ([]*model.GroupMember, *model.AppError) {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.UpsertGroupMembers")
+
+	a.ctx = newCtx
+	a.app.Srv().Store.SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store.SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0, resultVar1 := a.app.UpsertGroupMembers(groupID, userIDs)
+
+	if resultVar1 != nil {
+		span.LogFields(spanlog.Error(resultVar1))
+		ext.Error.Set(span, true)
+	}
+
+	return resultVar0, resultVar1
+}
+
 func (a *OpenTracingAppLayer) UpsertGroupSyncable(groupSyncable *model.GroupSyncable) (*model.GroupSyncable, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.UpsertGroupSyncable")

app/permissions_migrations.go

@@ -915,6 +915,29 @@ func (a *App) getAddTestEmailAncillaryPermission() (permissionsMap, error) {
 	return transformations, nil
 }
 
+func (a *App) getAddCustomUserGroupsPermissions() (permissionsMap, error) {
+	t := []permissionTransformation{}
+
+	customGroupPermissions := []string{
+		model.PermissionCreateCustomGroup.Id,
+		model.PermissionManageCustomGroupMembers.Id,
+		model.PermissionEditCustomGroup.Id,
+		model.PermissionDeleteCustomGroup.Id,
+	}
+
+	t = append(t, permissionTransformation{
+		On:  isRole(model.SystemUserRoleId),
+		Add: customGroupPermissions,
+	})
+
+	t = append(t, permissionTransformation{
+		On:  isRole(model.SystemAdminRoleId),
+		Add: customGroupPermissions,
+	})
+
+	return t, nil
+}
+
 func (a *App) getAddPlaybooksPermissions() (permissionsMap, error) {
 	transformations := []permissionTransformation{}
 
@@ -989,6 +1012,7 @@ func (s *Server) doPermissionsMigrations() error {
 		{Key: model.MigrationKeyAddReportingSubsectionPermissions, Migration: a.getAddReportingSubsectionPermissions},
 		{Key: model.MigrationKeyAddTestEmailAncillaryPermission, Migration: a.getAddTestEmailAncillaryPermission},
 		{Key: model.MigrationKeyAddPlaybooksPermissions, Migration: a.getAddPlaybooksPermissions},
+		{Key: model.MigrationKeyAddCustomUserGroupsPermissions, Migration: a.getAddCustomUserGroupsPermissions},
 	}
 
 	roles, err := s.Store.Role().GetAll()

app/plugin_api.go

@@ -597,7 +597,7 @@ func (api *PluginAPI) DeleteChannelMember(channelID, userID string) *model.AppEr
 }
 
 func (api *PluginAPI) GetGroup(groupId string) (*model.Group, *model.AppError) {
-	return api.app.GetGroup(groupId)
+	return api.app.GetGroup(groupId, nil)
 }
 
 func (api *PluginAPI) GetGroupByName(name string) (*model.Group, *model.AppError) {

app/syncables_test.go

@@ -59,7 +59,7 @@ func TestCreateDefaultMemberships(t *testing.T) {
 	gleeGroup, err := th.App.CreateGroup(&model.Group{
 		Name:        model.NewString(model.NewId()),
 		DisplayName: "Glee Club",
-		RemoteId:    model.NewId(),
+		RemoteId:    model.NewString(model.NewId()),
 		Source:      model.GroupSourceLdap,
 	})
 	if err != nil {
@@ -69,7 +69,7 @@ func TestCreateDefaultMemberships(t *testing.T) {
 	scienceGroup, err := th.App.CreateGroup(&model.Group{
 		Name:        model.NewString(model.NewId()),
 		DisplayName: "Science Club",
-		RemoteId:    model.NewId(),
+		RemoteId:    model.NewString(model.NewId()),
 		Source:      model.GroupSourceLdap,
 	})
 	if err != nil {

app/user.go

@@ -1735,6 +1735,9 @@ func (a *App) SearchUsers(props *model.UserSearch, options *model.UserSearchOpti
 	if props.InGroupId != "" {
 		return a.SearchUsersInGroup(props.InGroupId, props.Term, options)
 	}
+	if props.NotInGroupId != "" {
+		return a.SearchUsersNotInGroup(props.NotInGroupId, props.Term, options)
+	}
 	return a.SearchUsersInTeam(props.TeamId, props.Term, options)
 }
 
@@ -1822,6 +1825,20 @@ func (a *App) SearchUsersInGroup(groupID string, term string, options *model.Use
 	return users, nil
 }
 
+func (a *App) SearchUsersNotInGroup(groupID string, term string, options *model.UserSearchOptions) ([]*model.User, *model.AppError) {
+	term = strings.TrimSpace(term)
+	users, err := a.Srv().Store.User().SearchNotInGroup(groupID, term, options)
+	if err != nil {
+		return nil, model.NewAppError("SearchUsersNotInGroup", "app.user.search.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+
+	for _, user := range users {
+		a.SanitizeProfile(user, options.IsAdmin)
+	}
+
+	return users, nil
+}
+
 func (a *App) AutocompleteUsersInChannel(teamID string, channelID string, term string, options *model.UserSearchOptions) (*model.UserAutocompleteInChannel, *model.AppError) {
 	term = strings.TrimSpace(term)