Custom groups (#18839)

* WIP * adding initial creategroup endpoint * fetching by group source * fixing startup error * updating create endpoint to take an array of user_ids, this will allow us to create the group with one request * adding delete group endpoint and appropriate test * adding source param for getGroups * adding add members and delete members endpoints * locking down crud endpoints to only be allowed for custom groups * user search stuff * allowing remoteid be null by changing field to pointer * code cleanup and store level tests * adding new tests and removing unused endpoint * resolving conflicts * Adds authz check for group. * Adds authz checks to groups APIs. * Updated create group authz tests. * Updates delete group tests. * Tests create group. * Adds some tests and validations. * adding new parameter so I can get users not in a group * Fixed all lint warnings. * Fix type. * fixing search users not in group * Fixes some lint errors. * Moves entry in JSON array. * Fixed SQL query. * Fixes permission migration test. * Fixes migration test. * Fixes some group store tests. * Fix test. * Fix test. * Revert lint change. * Migrated CreateWithUserIds to sqlx. * Adds tests for GetMember; migrates implementation to sqlx. * Tests GetNonMemberUsersPage and hanles wrong group id. * Fixes test. * Switches GetMaster to GetMasterX. * Switches GetReplica to GetReplicaX. * Fixes logic. * Fixes shadow declaration. * Adds include_member_count to get group API endpoint. * Adds filter_has_member param to getGroups. * Fixes. * Removes array of group sources. * fixing error * Testing reverting CreateWithUserIds back to gorp. * Added websocket event for CreateGroupWithUserIds. * Changed a few response status codes. Switched to correct permission. * Added member count to ws payload for group when updating or creating. * Adds feature flag checks for custom groups. * Added middleware function to require license. Added config to disable custom groups. * Change for function signature change of executePossiblyEmptyQuery. * Lint fixes. * Adds telemetry none comment. * Adds translations. * Migrated to sqlx. * Temp. removal of translation. * Fixed typo. * Added an intermediary model to query with a field that is now ignored by sqlx on read queries. * Re-used existing store struct. * Inludes member count. * Fix for merge error.' * Require license for group endpoints. * Updates translations. * Fix shadow declaration. * Renames permissions. Switches to new method to retrieve remoteid. * Added WS events for upsert and delete member(s). * Added new store error type ErrUniqueConstraint. * Added EnableCustonGroups to the client config. * Sanitized some user records. * Added parameter to include_total_count for listing groups. * Added translations. * adding deleteAt field to getByUsers query * Revert sanitize. * Added uniqueness constraint error to UpdateGroup. * Removed the FutureFeatures flag so that the feature is not enabled on old Enterprise licenses. * Renamed function. * Updates authz check for user search related to groups. * Removed debug statement. * Removed unused app method. * Added telemetry for enable_custom_groups. * Returns early from nil license. * Updates test. * Returned early to avoid nesting in (*SqlGroupStore).checkUserExist. Switched to reading from replica in (*SqlGroupStore).GetMember. Handled JSON marshal error in (*Client4).UpsertGroupMembers * Switched to SanitizeProfile. * Switched to model.NewInt. * Switched from status NotImplemented to Forbidden for missing license. * Removed deactivated users from 'exists' set. * Revert gotool update. * Ignored lint error that I think is invalid. * Added the approprate access tag for disabling custom groups. * Revert change to response status. * Fixed refactor mistake. * Limited the group member WS events to individual users. * Removed WS event of deleted groups. * Updated license check for searchUsers endpoint. * Switched from license feature to license sku. * Update app/group.go Co-authored-by: Claudio Costa <cstcld91@gmail.com> * Update app/group.go Co-authored-by: Claudio Costa <cstcld91@gmail.com> * Remove linter ignore comment. * Added function to create sku-specific license. * Fixed typo. Removed comment. * Fixed for wrong type. * Added missing param to client. Removed unnecessary props setting. Added test for retrieving groups by source. * Updated some tests now that we're validating group membership not created for deactivated user. * Fix for groups endpoint returning all group types by default. * Changes constant names. Adds migration for all users to manage custom group members. * Removes requirement for manage_system permission to filter user search by group. * Added migration mock. * Removes default permissions from custom_group_user role. * Fixes migration. * Fixes emoji migration test. * fixing issue with member counts * fixing search issue for deleted members Co-authored-by: Benjamin Cooke <benjamincooke@Benjamins-MacBook-Pro.local> Co-authored-by: Benjamin Cooke <benjamincooke@Benjamins-MBP.ht.home> Co-authored-by: Mattermod <mattermod@users.noreply.github.com> Co-authored-by: Benjamin Cooke <benjamincooke@Benjamins-MacBook-Pro.fritz.box> Co-authored-by: Claudio Costa <cstcld91@gmail.com>
2022-02-17 12:34:39 -05:00
--- a/store/sqlstore/channel_store.go
+++ b/store/sqlstore/channel_store.go
@@ -2173,7 +2173,7 @@ func (s SqlChannelStore) GetMemberCountsByGroup(ctx context.Context, channelID s
 	query := s.getQueryBuilder().
 		Select(selectStr).
 		From("ChannelMembers").
-		Join("GroupMembers ON GroupMembers.UserId = ChannelMembers.UserId")
+		Join("GroupMembers ON GroupMembers.UserId = ChannelMembers.UserId AND GroupMembers.DeleteAt = 0")

 	if includeTimezones {
 		query = query.Join("Users ON Users.Id = GroupMembers.UserId")
--- a/store/sqlstore/group_store.go
+++ b/store/sqlstore/group_store.go
@@ -81,6 +81,137 @@ func (s *SqlGroupStore) Create(group *model.Group) (*model.Group, error) {
 	return group, nil
 }

+func (s *SqlGroupStore) CreateWithUserIds(g *model.GroupWithUserIds) (*model.Group, error) {
+	if g.Id != "" {
+		return nil, store.NewErrInvalidInput("Group", "id", g.Id)
+	}
+
+	// Check if group values are formatted correctly
+	if err := g.IsValidForCreate(); err != nil {
+		return nil, err
+	}
+
+	// Check Users exist
+	if err := s.checkUsersExist(g.UserIds); err != nil {
+		return nil, err
+	}
+
+	g.Id = model.NewId()
+	g.CreateAt = model.GetMillis()
+	g.UpdateAt = g.CreateAt
+
+	groupInsertQuery, groupInsertArgs, err := s.getQueryBuilder().
+		Insert("UserGroups").
+		Columns("Id", "Name", "DisplayName", "Description", "Source", "RemoteId", "CreateAt", "UpdateAt", "DeleteAt", "AllowReference").
+		Values(g.Id, g.Name, g.DisplayName, g.Description, g.Source, g.RemoteId, g.CreateAt, g.UpdateAt, 0, g.AllowReference).
+		ToSql()
+	if err != nil {
+		return nil, err
+	}
+
+	usersInsertQuery, usersInsertArgs, err := s.buildInsertGroupUsersQuery(g.Id, g.UserIds)
+	if err != nil {
+		return nil, err
+	}
+
+	txn, err := s.GetMasterX().Beginx()
+	if err != nil {
+		return nil, err
+	}
+	defer finalizeTransactionX(txn)
+	// Create a new usergroup
+	if _, err = txn.Exec(groupInsertQuery, groupInsertArgs...); err != nil {
+		if IsUniqueConstraintError(err, []string{"Name", "groups_name_key"}) {
+			return nil, store.NewErrUniqueConstraint("Name")
+		}
+		return nil, errors.Wrap(err, "failed to save Group")
+	}
+	// Insert the Group Members
+	if _, err = executePossiblyEmptyQuery(txn, usersInsertQuery, usersInsertArgs...); err != nil {
+		return nil, err
+	}
+
+	// Get the new Group along with the member count
+	groupGroupQuery := `
+		SELECT
+			UserGroups.*,
+			A.Count AS MemberCount
+		FROM
+			UserGroups
+			INNER JOIN (
+				SELECT
+					UserGroups.Id,
+					COUNT(GroupMembers.UserId) AS Count
+				FROM
+					UserGroups
+					LEFT JOIN GroupMembers ON UserGroups.Id = GroupMembers.GroupId
+				WHERE
+					UserGroups.Id = ?
+				GROUP BY
+					UserGroups.Id
+				ORDER BY
+					UserGroups.DisplayName,
+					UserGroups.Id
+				LIMIT
+					? OFFSET ?
+			) AS A ON UserGroups.Id = A.Id
+		ORDER BY
+			UserGroups.CreateAt DESC`
+	var newGroup group
+	if err = txn.Get(&newGroup, groupGroupQuery, g.Id, 1, 0); err != nil {
+		return nil, err
+	}
+	if err = txn.Commit(); err != nil {
+		return nil, err
+	}
+	return newGroup.ToModel(), nil
+}
+
+func (s *SqlGroupStore) checkUsersExist(userIDs []string) error {
+	if len(userIDs) == 0 {
+		return nil
+	}
+	usersSelectQuery, usersSelectArgs, err := s.getQueryBuilder().
+		Select("Id").
+		From("Users").
+		Where(sq.Eq{"Id": userIDs, "DeleteAt": 0}).
+		ToSql()
+	if err != nil {
+		return err
+	}
+	var rows []string
+	err = s.GetReplicaX().Select(&rows, usersSelectQuery, usersSelectArgs...)
+	if err != nil {
+		return err
+	}
+	if len(rows) == len(userIDs) {
+		return nil
+	}
+	retrievedIDs := make(map[string]bool)
+	for _, userID := range rows {
+		retrievedIDs[userID] = true
+	}
+	for _, userID := range userIDs {
+		if _, ok := retrievedIDs[userID]; !ok {
+			return store.NewErrNotFound("User", userID)
+		}
+	}
+	return nil
+}
+
+func (s *SqlGroupStore) buildInsertGroupUsersQuery(groupId string, userIds []string) (query string, args []interface{}, err error) {
+	if len(userIds) > 0 {
+		builder := s.getQueryBuilder().
+			Insert("GroupMembers").
+			Columns("GroupId", "UserId", "CreateAt", "DeleteAt")
+		for _, userId := range userIds {
+			builder = builder.Values(groupId, userId, model.GetMillis(), 0)
+		}
+		query, args, err = builder.ToSql()
+	}
+	return
+}
+
 func (s *SqlGroupStore) Get(groupId string) (*model.Group, error) {
 	var group model.Group
 	if err := s.GetReplicaX().Get(&group, "SELECT * from UserGroups WHERE Id = ?", groupId); err != nil {
@@ -228,6 +359,26 @@ func (s *SqlGroupStore) Delete(groupID string) (*model.Group, error) {
 	return &group, nil
 }

+func (s *SqlGroupStore) GetMember(groupID, userID string) (*model.GroupMember, error) {
+	query, args, err := s.getQueryBuilder().
+		Select("*").
+		From("GroupMembers").
+		Where(sq.Eq{"UserId": userID}).
+		Where(sq.Eq{"GroupId": groupID}).
+		Where(sq.Eq{"DeleteAt": 0}).
+		ToSql()
+	if err != nil {
+		return nil, errors.Wrap(err, "get_member_query")
+	}
+	var groupMember model.GroupMember
+	err = s.GetReplicaX().Get(&groupMember, query, args...)
+	if err != nil {
+		return nil, errors.Wrap(err, "GetMember")
+	}
+
+	return &groupMember, nil
+}
+
 func (s *SqlGroupStore) GetMemberUsers(groupID string) ([]*model.User, error) {
 	groupMembers := []*model.User{}

@@ -276,6 +427,37 @@ func (s *SqlGroupStore) GetMemberUsersPage(groupID string, page int, perPage int
 	return groupMembers, nil
 }

+func (s *SqlGroupStore) GetNonMemberUsersPage(groupID string, page int, perPage int) ([]*model.User, error) {
+	groupMembers := []*model.User{}
+
+	if err := s.GetReplicaX().Get(&model.Group{}, "SELECT * FROM UserGroups WHERE Id = ?", groupID); err != nil {
+		return nil, errors.Wrap(err, "GetNonMemberUsersPage")
+	}
+
+	query := `
+	SELECT 
+		Users.*
+	FROM
+		Users
+	LEFT JOIN 
+		GroupMembers ON (GroupMembers.UserId = Users.Id AND GroupMembers.GroupId = ?)
+	WHERE
+		Users.DeleteAt = 0
+		AND ( GroupMembers.UserId IS NULL OR GroupMembers.DeleteAt != 0)
+	ORDER BY
+		GroupMembers.CreateAt DESC
+	LIMIT
+		?
+	OFFSET
+		?`
+
+	if err := s.GetReplicaX().Select(&groupMembers, query, groupID, perPage, page*perPage); err != nil {
+		return nil, errors.Wrapf(err, "failed to find member Users for Group with id=%s", groupID)
+	}
+
+	return groupMembers, nil
+}
+
 func (s *SqlGroupStore) GetMemberCount(groupID string) (int64, error) {
 	query := `
 		SELECT
@@ -285,7 +467,8 @@ func (s *SqlGroupStore) GetMemberCount(groupID string) (int64, error) {
 			JOIN Users ON Users.Id = GroupMembers.UserId
 		WHERE
 			GroupMembers.GroupId = ?
-			AND Users.DeleteAt = 0`
+			AND Users.DeleteAt = 0
+			AND GroupMembers.DeleteAt = 0`

 	var count int64
 	err := s.GetReplicaX().Get(&count, query, groupID)
@@ -361,64 +544,26 @@ func (s *SqlGroupStore) GetMemberUsersNotInChannel(groupID string, channelID str
 }

 func (s *SqlGroupStore) UpsertMember(groupID string, userID string) (*model.GroupMember, error) {
-	member := &model.GroupMember{
-		GroupId:  groupID,
-		UserId:   userID,
-		CreateAt: model.GetMillis(),
-		DeleteAt: 0,
-	}
-
-	if err := member.IsValid(); err != nil {
+	members, query, args, err := s.buildUpsertMembersQuery(groupID, []string{userID})
+	if err != nil {
 		return nil, err
 	}
-
-	var retrievedGroup model.Group
-	if err := s.GetReplicaX().Get(&retrievedGroup, "SELECT * FROM UserGroups WHERE Id = ?", groupID); err != nil {
-		return nil, errors.Wrapf(err, "failed to get UserGroup with groupId=%s and userId=%s", groupID, userID)
-	}
-
-	query := s.getQueryBuilder().
-		Insert("GroupMembers").
-		Columns("GroupId", "UserId", "CreateAt", "DeleteAt").
-		Values(member.GroupId, member.UserId, member.CreateAt, member.DeleteAt)
-
-	if s.DriverName() == model.DatabaseDriverMysql {
-		query = query.SuffixExpr(sq.Expr("ON DUPLICATE KEY UPDATE CreateAt = ?, DeleteAt = ?", member.CreateAt, member.DeleteAt))
-	} else if s.DriverName() == model.DatabaseDriverPostgres {
-		query = query.SuffixExpr(sq.Expr("ON CONFLICT (groupid, userid) DO UPDATE SET CreateAt = ?, DeleteAt = ?", member.CreateAt, member.DeleteAt))
-	}
-
-	queryString, args, err := query.ToSql()
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to generate sqlquery")
-	}
-
-	if _, err = s.GetMasterX().Exec(queryString, args...); err != nil {
+	if _, err = s.GetMasterX().Exec(query, args...); err != nil {
 		return nil, errors.Wrap(err, "failed to save GroupMember")
 	}
-	return member, nil
+	return members[0], nil
 }

 func (s *SqlGroupStore) DeleteMember(groupID string, userID string) (*model.GroupMember, error) {
-	var retrievedMember model.GroupMember
-	if err := s.GetReplicaX().Get(&retrievedMember, "SELECT * FROM GroupMembers WHERE GroupId = ? AND UserId = ? AND DeleteAt = 0", groupID, userID); err != nil {
-		if err == sql.ErrNoRows {
-			return nil, store.NewErrNotFound("GroupMember", fmt.Sprintf("groupId=%s, userId=%s", groupID, userID))
-		}
-		return nil, errors.Wrapf(err, "failed to get GroupMember with groupId=%s and userId=%s", groupID, userID)
+	members, query, args, err := s.buildDeleteMembersQuery(groupID, []string{userID})
+	if err != nil {
+		return nil, err
 	}
-
-	retrievedMember.DeleteAt = model.GetMillis()
-
-	if _, err := s.GetMasterX().NamedExec(`UPDATE GroupMembers
-		SET DeleteAt=:DeleteAt
-		WHERE GroupId=:GroupId
-		AND UserId=:UserId
-		AND DeleteAt=0`, retrievedMember); err != nil {
+	if _, err = s.GetMasterX().Exec(query, args...); err != nil {
 		return nil, errors.Wrapf(err, "failed to update GroupMember with groupId=%s and userId=%s", groupID, userID)
 	}

-	return &retrievedMember, nil
+	return members[0], nil
 }

 func (s *SqlGroupStore) PermanentDeleteMembersByUser(userId string) error {
@@ -887,7 +1032,7 @@ type group struct {
 	DisplayName    string
 	Description    string
 	Source         model.GroupSource
-	RemoteId       string
+	RemoteId       *string
 	CreateAt       int64
 	UpdateAt       int64
 	DeleteAt       int64
@@ -1255,6 +1400,13 @@ func (s *SqlGroupStore) GetGroups(page, perPage int, opts model.GroupSearchOpts)
 			LeftJoin("(SELECT GroupMembers.GroupId, COUNT(*) AS MemberCount FROM GroupMembers LEFT JOIN Users ON Users.Id = GroupMembers.UserId WHERE GroupMembers.DeleteAt = 0 AND Users.DeleteAt = 0 GROUP BY GroupId) AS Members ON Members.GroupId = g.Id")
 	}

+	if opts.FilterHasMember != "" {
+		groupsQuery = groupsQuery.
+			LeftJoin("GroupMembers ON GroupMembers.GroupId = g.Id").
+			Where("GroupMembers.UserId = ?", opts.FilterHasMember).
+			Where("GroupMembers.DeleteAt = 0")
+	}
+
 	groupsQuery = groupsQuery.
 		From("UserGroups g").
 		OrderBy("g.DisplayName")
@@ -1350,6 +1502,10 @@ func (s *SqlGroupStore) GetGroups(page, perPage int, opts model.GroupSearchOpts)
 		`, opts.NotAssociatedToChannel, opts.NotAssociatedToChannel)
 	}

+	if opts.Source != "" {
+		groupsQuery = groupsQuery.Where("g.Source = ?", opts.Source)
+	}
+
 	queryString, args, err := groupsQuery.ToSql()
 	if err != nil {
 		return nil, errors.Wrap(err, "get_groups_tosql")
@@ -1608,3 +1764,117 @@ func (s *SqlGroupStore) countTableWithSelectAndWhere(selectStr, tableName string

 	return count, nil
 }
+
+func (s *SqlGroupStore) UpsertMembers(groupID string, userIDs []string) ([]*model.GroupMember, error) {
+	members, query, args, err := s.buildUpsertMembersQuery(groupID, userIDs)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err = s.GetMasterX().Exec(query, args...); err != nil {
+		return nil, errors.Wrap(err, "failed to save GroupMember")
+	}
+
+	return members, err
+}
+
+func (s *SqlGroupStore) buildUpsertMembersQuery(groupID string, userIDs []string) (members []*model.GroupMember, query string, args []interface{}, err error) {
+	var retrievedGroup model.Group
+	// Check Group exists
+	if err = s.GetReplicaX().Get(&retrievedGroup, "SELECT * FROM UserGroups WHERE Id = ?", groupID); err != nil {
+		err = errors.Wrapf(err, "failed to get UserGroup with groupId=%s", groupID)
+		return
+	}
+
+	// Check Users exist
+	if err = s.checkUsersExist(userIDs); err != nil {
+		return
+	}
+
+	builder := s.getQueryBuilder().
+		Insert("GroupMembers").
+		Columns("GroupId", "UserId", "CreateAt", "DeleteAt")
+
+	members = make([]*model.GroupMember, 0, len(userIDs))
+	createAt := model.GetMillis()
+	for _, userId := range userIDs {
+		member := &model.GroupMember{
+			GroupId:  groupID,
+			UserId:   userId,
+			CreateAt: createAt,
+			DeleteAt: 0,
+		}
+		builder = builder.Values(member.GroupId, member.UserId, member.CreateAt, member.DeleteAt)
+		members = append(members, member)
+	}
+
+	if s.DriverName() == model.DatabaseDriverMysql {
+		builder = builder.SuffixExpr(sq.Expr("ON DUPLICATE KEY UPDATE CreateAt = ?, DeleteAt = ?", createAt, 0))
+	} else if s.DriverName() == model.DatabaseDriverPostgres {
+		builder = builder.SuffixExpr(sq.Expr("ON CONFLICT (groupid, userid) DO UPDATE SET CreateAt = ?, DeleteAt = ?", createAt, 0))
+	}
+
+	query, args, err = builder.ToSql()
+	return
+}
+
+func (s *SqlGroupStore) DeleteMembers(groupID string, userIDs []string) ([]*model.GroupMember, error) {
+	members, query, args, err := s.buildDeleteMembersQuery(groupID, userIDs)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err = s.GetMasterX().Exec(query, args...); err != nil {
+		return nil, errors.Wrap(err, "failed to delete GroupMembers")
+	}
+	return members, err
+}
+
+func (s *SqlGroupStore) buildDeleteMembersQuery(groupID string, userIDs []string) (members []*model.GroupMember, query string, args []interface{}, err error) {
+	membersSelectQuery, membersSelectArgs, err := s.getQueryBuilder().
+		Select("*").
+		From("GroupMembers").
+		Where(sq.And{
+			sq.Eq{"GroupId": groupID},
+			sq.Eq{"UserId": userIDs},
+			sq.Eq{"DeleteAt": 0},
+		}).
+		ToSql()
+	if err != nil {
+		return
+	}
+
+	err = s.GetReplicaX().Select(&members, membersSelectQuery, membersSelectArgs...)
+	if err != nil {
+		return
+	}
+	if len(members) != len(userIDs) {
+		retrievedRecords := make(map[string]bool)
+		for _, member := range members {
+			retrievedRecords[member.UserId] = true
+		}
+		for _, userID := range userIDs {
+			if _, ok := retrievedRecords[userID]; !ok {
+				err = store.NewErrNotFound("User", userID)
+				return
+			}
+		}
+	}
+
+	deleteAt := model.GetMillis()
+
+	for _, member := range members {
+		member.DeleteAt = deleteAt
+	}
+
+	builder := s.getQueryBuilder().
+		Update("GroupMembers").
+		Set("DeleteAt", deleteAt).
+		Where(sq.And{
+			sq.Eq{"GroupId": groupID},
+			sq.Eq{"UserId": userIDs},
+		})
+
+	query, args, err = builder.ToSql()
+	return
+}
--- a/store/sqlstore/user_store.go
+++ b/store/sqlstore/user_store.go
@@ -1438,7 +1438,17 @@ func (us SqlUserStore) SearchInChannel(channelId string, term string, options *m

 func (us SqlUserStore) SearchInGroup(groupID string, term string, options *model.UserSearchOptions) ([]*model.User, error) {
 	query := us.usersQuery.
-		Join("GroupMembers gm ON ( gm.UserId = u.Id AND gm.GroupId = ? )", groupID).
+		Join("GroupMembers gm ON ( gm.UserId = u.Id AND gm.GroupId = ? AND gm.DeleteAt = 0 )", groupID).
+		OrderBy("Username ASC").
+		Limit(uint64(options.Limit))
+
+	return us.performSearch(query, term, options)
+}
+
+func (us SqlUserStore) SearchNotInGroup(groupID string, term string, options *model.UserSearchOptions) ([]*model.User, error) {
+	query := us.usersQuery.
+		LeftJoin("GroupMembers gm ON ( gm.UserId = u.Id AND gm.GroupId = ? )", groupID).
+		Where("gm.UserId IS NULL").
 		OrderBy("Username ASC").
 		Limit(uint64(options.Limit))

@@ -2070,9 +2080,11 @@ func (us SqlUserStore) GetUsersWithInvalidEmails(page int, perPage int, restrict
 			query = query.Where("u.Email NOT LIKE LOWER(?)", wildcardSearchTerm(d))
 		}
 	}
+
 	query = query.Offset(uint64(page * perPage)).Limit(uint64(perPage))

 	queryString, args, err := query.ToSql()
+
 	if err != nil {
 		return nil, errors.Wrap(err, "users_get_many_tosql")
 	}