MM-54201 Refactor mention parsing in preparation for multi-word mentions (#25030)

* MM-54201 Move ExplicitMentions to its own file and rename it (#24932) * MM-54201 Move ExplicitMentions to its own file and rename it * Fix vet * MM-54201 Refactor current mention parsing into MentionParserStandard (#24936) * MM-54201 Refactor current mention parsing into MentionParserStandard * Fix vet * MM-54201 Unify user and group mention parsing logic (#24937) * MM-54201 Add MentionKeywords type * MM-54201 Move group mentions into MentionKeywords * Fix flaky test caused by random iteration order * Update server/channels/app/mention_results.go Co-authored-by: Jesse Hallam <jesse.hallam@gmail.com> * Address feedback --------- Co-authored-by: Jesse Hallam <jesse.hallam@gmail.com> --------- Co-authored-by: Jesse Hallam <jesse.hallam@gmail.com>
2023-10-23 12:37:58 -04:00
--- a/server/channels/app/notification.go
+++ b/server/channels/app/notification.go
@@ -10,8 +10,6 @@ import (
 	"sort"
 	"strings"
 	"sync"
-	"unicode"
-	"unicode/utf8"

 	"github.com/pkg/errors"

@@ -126,14 +124,15 @@ func (a *App) SendNotifications(c request.CTX, post *model.Post, team *model.Tea
 	var allActivityPushUserIds []string
 	if channel.Type != model.ChannelTypeDirect {
 		// Iterate through all groups that were mentioned and insert group members into the list of mentions or potential mentions
-		for _, group := range mentions.GroupMentions {
+		for groupID := range mentions.GroupMentions {
+			group := groups[groupID]
 			anyUsersMentionedByGroup, err := a.insertGroupMentions(group, channel, profileMap, mentions)
 			if err != nil {
 				return nil, err
 			}

 			if !anyUsersMentionedByGroup {
-				a.sendNoUsersNotifiedByGroupInChannel(c, sender, post, channel, group)
+				a.sendNoUsersNotifiedByGroupInChannel(c, sender, post, channel, groups[groupID])
 			}
 		}

@@ -165,14 +164,14 @@ func (a *App) SendNotifications(c request.CTX, post *model.Post, team *model.Tea
 	membershipsMutex := &sync.Mutex{}
 	followersMutex := &sync.Mutex{}
 	if *a.Config().ServiceSettings.ThreadAutoFollow && post.RootId != "" {
-		var rootMentions *ExplicitMentions
+		var rootMentions *MentionResults
 		if parentPostList != nil {
 			rootPost := parentPostList.Posts[parentPostList.Order[0]]
 			if rootPost.GetProp("from_webhook") != "true" {
 				threadParticipants[rootPost.UserId] = true
 			}
 			if channel.Type != model.ChannelTypeDirect {
-				rootMentions = getExplicitMentions(rootPost, keywords, groups)
+				rootMentions = getExplicitMentions(rootPost, keywords)
 				for id := range rootMentions.Mentions {
 					threadParticipants[id] = true
 				}
@@ -676,9 +675,9 @@ func (a *App) RemoveNotifications(c request.CTX, post *model.Post, channel *mode
 		mentions, _ := a.getExplicitMentionsAndKeywords(c, post, channel, profileMap, groups, channelMemberNotifyPropsMap, nil)

 		userIDs := []string{}
-		for _, group := range mentions.GroupMentions {
+		for groupID := range mentions.GroupMentions {
 			for page := 0; ; page++ {
-				groupMemberPage, count, appErr := a.GetGroupMemberUsersPage(group.Id, page, 100, &model.ViewUsersRestrictions{Channels: []string{channel.Id}})
+				groupMemberPage, count, appErr := a.GetGroupMemberUsersPage(groupID, page, 100, &model.ViewUsersRestrictions{Channels: []string{channel.Id}})
 				if appErr != nil {
 					return appErr
 				}
@@ -752,10 +751,10 @@ func (a *App) RemoveNotifications(c request.CTX, post *model.Post, channel *mode
 	return nil
 }

-func (a *App) getExplicitMentionsAndKeywords(c request.CTX, post *model.Post, channel *model.Channel, profileMap map[string]*model.User, groups map[string]*model.Group, channelMemberNotifyPropsMap map[string]model.StringMap, parentPostList *model.PostList) (*ExplicitMentions, map[string][]string) {
-	mentions := &ExplicitMentions{}
+func (a *App) getExplicitMentionsAndKeywords(c request.CTX, post *model.Post, channel *model.Channel, profileMap map[string]*model.User, groups map[string]*model.Group, channelMemberNotifyPropsMap map[string]model.StringMap, parentPostList *model.PostList) (*MentionResults, MentionKeywords) {
+	mentions := &MentionResults{}
 	var allowChannelMentions bool
-	var keywords map[string][]string
+	var keywords MentionKeywords

 	if channel.Type == model.ChannelTypeDirect {
 		otherUserId := channel.GetOtherUserIdForDM(post.UserId)
@@ -770,9 +769,9 @@ func (a *App) getExplicitMentionsAndKeywords(c request.CTX, post *model.Post, ch
 		}
 	} else {
 		allowChannelMentions = a.allowChannelMentions(c, post, len(profileMap))
-		keywords = a.getMentionKeywordsInChannel(profileMap, allowChannelMentions, channelMemberNotifyPropsMap)
+		keywords = a.getMentionKeywordsInChannel(profileMap, allowChannelMentions, channelMemberNotifyPropsMap, groups)

-		mentions = getExplicitMentions(post, keywords, groups)
+		mentions = getExplicitMentions(post, keywords)

 		// Add a GM mention to all members of a GM channel
 		if channel.Type == model.ChannelTypeGroup {
@@ -1045,141 +1044,39 @@ func splitAtFinal(items []string) (preliminary []string, final string) {
 	return
 }

-type ExplicitMentions struct {
-	// Mentions contains the ID of each user that was mentioned and how they were mentioned.
-	Mentions map[string]MentionType
-
-	// Contains a map of groups that were mentioned
-	GroupMentions map[string]*model.Group
-
-	// OtherPotentialMentions contains a list of strings that looked like mentions, but didn't have
-	// a corresponding keyword.
-	OtherPotentialMentions []string
-
-	// HereMentioned is true if the message contained @here.
-	HereMentioned bool
-
-	// AllMentioned is true if the message contained @all.
-	AllMentioned bool
-
-	// ChannelMentioned is true if the message contained @channel.
-	ChannelMentioned bool
-}
-
-type MentionType int
-
-const (
-	// Different types of mentions ordered by their priority from lowest to highest
-
-	// A placeholder that should never be used in practice
-	NoMention MentionType = iota
-
-	// The post is in a GM
-	GMMention
-
-	// The post is in a thread that the user has commented on
-	ThreadMention
-
-	// The post is a comment on a thread started by the user
-	CommentMention
-
-	// The post contains an at-channel, at-all, or at-here
-	ChannelMention
-
-	// The post is a DM
-	DMMention
-
-	// The post contains an at-mention for the user
-	KeywordMention
-
-	// The post contains a group mention for the user
-	GroupMention
-)
-
-func (m *ExplicitMentions) isUserMentioned(userID string) bool {
-	if _, ok := m.Mentions[userID]; ok {
-		return true
-	}
-
-	if _, ok := m.GroupMentions[userID]; ok {
-		return true
-	}
-
-	return m.HereMentioned || m.AllMentioned || m.ChannelMentioned
-}
-
-func (m *ExplicitMentions) addMention(userID string, mentionType MentionType) {
-	if m.Mentions == nil {
-		m.Mentions = make(map[string]MentionType)
-	}
-
-	if currentType, ok := m.Mentions[userID]; ok && currentType >= mentionType {
-		return
-	}
-
-	m.Mentions[userID] = mentionType
-}
-
-func (m *ExplicitMentions) addGroupMention(word string, groups map[string]*model.Group) bool {
-	if strings.HasPrefix(word, "@") {
-		word = word[1:]
-	} else {
-		// Only allow group mentions when mentioned directly with @group-name
-		return false
-	}
-
-	group, groupFound := groups[word]
-	if !groupFound {
-		group = groups[strings.ToLower(word)]
-	}
-
-	if group == nil {
-		return false
-	}
-
-	if m.GroupMentions == nil {
-		m.GroupMentions = make(map[string]*model.Group)
-	}
-
-	if group.Name != nil {
-		m.GroupMentions[*group.Name] = group
-	}
-
-	return true
-}
-
-func (m *ExplicitMentions) addMentions(userIDs []string, mentionType MentionType) {
-	for _, userID := range userIDs {
-		m.addMention(userID, mentionType)
-	}
-}
-
-func (m *ExplicitMentions) removeMention(userID string) {
-	delete(m.Mentions, userID)
-}
-
 // Given a message and a map mapping mention keywords to the users who use them, returns a map of mentioned
 // users and a slice of potential mention users not in the channel and whether or not @here was mentioned.
-func getExplicitMentions(post *model.Post, keywords map[string][]string, groups map[string]*model.Group) *ExplicitMentions {
-	ret := &ExplicitMentions{}
+func getExplicitMentions(post *model.Post, keywords MentionKeywords) *MentionResults {
+	parser := makeStandardMentionParser(keywords)

 	buf := ""
 	mentionsEnabledFields := getMentionsEnabledFields(post)
 	for _, message := range mentionsEnabledFields {
+		// Parse the text as Markdown, combining adjacent Text nodes into a single string for processing
 		markdown.Inspect(message, func(node any) bool {
 			text, ok := node.(*markdown.Text)
 			if !ok {
-				ret.processText(buf, keywords, groups)
+				// This node isn't a string so process any accumulated text in the buffer
+				if buf != "" {
+					parser.ProcessText(buf)
+				}
+
 				buf = ""
 				return true
 			}
+
+			// This node is a string, so add it to buf and continue onto the next node to see if it's more text
 			buf += text.Text
 			return false
 		})
 	}
-	ret.processText(buf, keywords, groups)

-	return ret
+	// Process any left over text
+	if buf != "" {
+		parser.ProcessText(buf)
+	}
+
+	return parser.Results()
 }

 // Given a post returns the values of the fields in which mentions are possible.
@@ -1251,7 +1148,7 @@ func (a *App) getGroupsAllowedForReferenceInChannel(channel *model.Channel, team
 		}
 		for _, group := range groups {
 			if group.Group.Name != nil {
-				groupsMap[*group.Group.Name] = &group.Group
+				groupsMap[group.Id] = &group.Group
 			}
 		}
 		return groupsMap, nil
@@ -1263,7 +1160,7 @@ func (a *App) getGroupsAllowedForReferenceInChannel(channel *model.Channel, team
 	}
 	for _, group := range groups {
 		if group.Name != nil {
-			groupsMap[*group.Name] = group
+			groupsMap[group.Id] = group
 		}
 	}

@@ -1272,12 +1169,11 @@ func (a *App) getGroupsAllowedForReferenceInChannel(channel *model.Channel, team

 // Given a map of user IDs to profiles, returns a list of mention
 // keywords for all users in the channel.
-func (a *App) getMentionKeywordsInChannel(profiles map[string]*model.User, allowChannelMentions bool, channelMemberNotifyPropsMap map[string]model.StringMap) map[string][]string {
-	keywords := make(map[string][]string)
+func (a *App) getMentionKeywordsInChannel(profiles map[string]*model.User, allowChannelMentions bool, channelMemberNotifyPropsMap map[string]model.StringMap, groups map[string]*model.Group) MentionKeywords {
+	keywords := make(MentionKeywords)

 	for _, profile := range profiles {
-		addMentionKeywordsForUser(
-			keywords,
+		keywords.AddUser(
 			profile,
 			channelMemberNotifyPropsMap[profile.Id],
 			a.GetStatusFromCache(profile.Id),
@@ -1285,12 +1181,14 @@ func (a *App) getMentionKeywordsInChannel(profiles map[string]*model.User, allow
 		)
 	}

+	keywords.AddGroupsMap(groups)
+
 	return keywords
 }

 // insertGroupMentions adds group members in the channel to Mentions, adds group members not in the channel to OtherPotentialMentions
 // returns false if no group members present in the team that the channel belongs to
-func (a *App) insertGroupMentions(group *model.Group, channel *model.Channel, profileMap map[string]*model.User, mentions *ExplicitMentions) (bool, *model.AppError) {
+func (a *App) insertGroupMentions(group *model.Group, channel *model.Channel, profileMap map[string]*model.User, mentions *MentionResults) (bool, *model.AppError) {
 	var err error
 	var groupMembers []*model.User
 	outOfChannelGroupMembers := []*model.User{}
@@ -1331,44 +1229,6 @@ func (a *App) insertGroupMentions(group *model.Group, channel *model.Channel, pr
 	return isGroupOrDirect || len(groupMembers) > 0, nil
 }

-// addMentionKeywordsForUser adds the mention keywords for a given user to the given keyword map. Returns the provided keyword map.
-func addMentionKeywordsForUser(keywords map[string][]string, profile *model.User, channelNotifyProps map[string]string, status *model.Status, allowChannelMentions bool) map[string][]string {
-	userMention := "@" + strings.ToLower(profile.Username)
-	keywords[userMention] = append(keywords[userMention], profile.Id)
-
-	// Add all the user's mention keys
-	for _, k := range profile.GetMentionKeys() {
-		// note that these are made lower case so that we can do a case insensitive check for them
-		key := strings.ToLower(k)
-
-		if key != "" {
-			keywords[key] = append(keywords[key], profile.Id)
-		}
-	}
-
-	// If turned on, add the user's case sensitive first name
-	if profile.NotifyProps[model.FirstNameNotifyProp] == "true" && profile.FirstName != "" {
-		keywords[profile.FirstName] = append(keywords[profile.FirstName], profile.Id)
-	}
-
-	// Add @channel and @all to keywords if user has them turned on and the server allows them
-	if allowChannelMentions {
-		// Ignore channel mentions if channel is muted and channel mention setting is default
-		ignoreChannelMentions := channelNotifyProps[model.IgnoreChannelMentionsNotifyProp] == model.IgnoreChannelMentionsOn || (channelNotifyProps[model.MarkUnreadNotifyProp] == model.UserNotifyMention && channelNotifyProps[model.IgnoreChannelMentionsNotifyProp] == model.IgnoreChannelMentionsDefault)
-
-		if profile.NotifyProps[model.ChannelMentionsNotifyProp] == "true" && !ignoreChannelMentions {
-			keywords["@channel"] = append(keywords["@channel"], profile.Id)
-			keywords["@all"] = append(keywords["@all"], profile.Id)
-
-			if status != nil && status.Status == model.StatusOnline {
-				keywords["@here"] = append(keywords["@here"], profile.Id)
-			}
-		}
-	}
-
-	return keywords
-}
-
 // Represents either an email or push notification and contains the fields required to send it to any user.
 type PostNotification struct {
 	Channel    *model.Channel
@@ -1418,127 +1278,6 @@ func (n *PostNotification) GetSenderName(userNameFormat string, overridesAllowed
 	return n.Sender.GetDisplayNameWithPrefix(userNameFormat, "@")
 }

-// checkForMention checks if there is a mention to a specific user or to the keywords here / channel / all
-func (m *ExplicitMentions) checkForMention(word string, keywords map[string][]string, groups map[string]*model.Group) bool {
-	var mentionType MentionType
-
-	switch strings.ToLower(word) {
-	case "@here":
-		m.HereMentioned = true
-		mentionType = ChannelMention
-	case "@channel":
-		m.ChannelMentioned = true
-		mentionType = ChannelMention
-	case "@all":
-		m.AllMentioned = true
-		mentionType = ChannelMention
-	default:
-		mentionType = KeywordMention
-	}
-
-	m.addGroupMention(word, groups)
-
-	if ids, match := keywords[strings.ToLower(word)]; match {
-		m.addMentions(ids, mentionType)
-		return true
-	}
-
-	// Case-sensitive check for first name
-	if ids, match := keywords[word]; match {
-		m.addMentions(ids, mentionType)
-		return true
-	}
-
-	return false
-}
-
-// isKeywordMultibyte checks if a word containing a multibyte character contains a multibyte keyword
-func isKeywordMultibyte(keywords map[string][]string, word string) ([]string, bool) {
-	ids := []string{}
-	match := false
-	var multibyteKeywords []string
-	for keyword := range keywords {
-		if len(keyword) != utf8.RuneCountInString(keyword) {
-			multibyteKeywords = append(multibyteKeywords, keyword)
-		}
-	}
-
-	if len(word) != utf8.RuneCountInString(word) {
-		for _, key := range multibyteKeywords {
-			if strings.Contains(word, key) {
-				ids, match = keywords[key]
-			}
-		}
-	}
-	return ids, match
-}
-
-// Processes text to filter mentioned users and other potential mentions
-func (m *ExplicitMentions) processText(text string, keywords map[string][]string, groups map[string]*model.Group) {
-	systemMentions := map[string]bool{"@here": true, "@channel": true, "@all": true}
-
-	for _, word := range strings.FieldsFunc(text, func(c rune) bool {
-		// Split on any whitespace or punctuation that can't be part of an at mention or emoji pattern
-		return !(c == ':' || c == '.' || c == '-' || c == '_' || c == '@' || unicode.IsLetter(c) || unicode.IsNumber(c))
-	}) {
-		// skip word with format ':word:' with an assumption that it is an emoji format only
-		if word[0] == ':' && word[len(word)-1] == ':' {
-			continue
-		}
-
-		word = strings.TrimLeft(word, ":.-_")
-
-		if m.checkForMention(word, keywords, groups) {
-			continue
-		}
-
-		foundWithoutSuffix := false
-		wordWithoutSuffix := word
-
-		for wordWithoutSuffix != "" && strings.LastIndexAny(wordWithoutSuffix, ".-:_") == (len(wordWithoutSuffix)-1) {
-			wordWithoutSuffix = wordWithoutSuffix[0 : len(wordWithoutSuffix)-1]
-
-			if m.checkForMention(wordWithoutSuffix, keywords, groups) {
-				foundWithoutSuffix = true
-				break
-			}
-		}
-
-		if foundWithoutSuffix {
-			continue
-		}
-
-		if _, ok := systemMentions[word]; !ok && strings.HasPrefix(word, "@") {
-			// No need to bother about unicode as we are looking for ASCII characters.
-			last := word[len(word)-1]
-			switch last {
-			// If the word is possibly at the end of a sentence, remove that character.
-			case '.', '-', ':':
-				word = word[:len(word)-1]
-			}
-			m.OtherPotentialMentions = append(m.OtherPotentialMentions, word[1:])
-		} else if strings.ContainsAny(word, ".-:") {
-			// This word contains a character that may be the end of a sentence, so split further
-			splitWords := strings.FieldsFunc(word, func(c rune) bool {
-				return c == '.' || c == '-' || c == ':'
-			})
-
-			for _, splitWord := range splitWords {
-				if m.checkForMention(splitWord, keywords, groups) {
-					continue
-				}
-				if _, ok := systemMentions[splitWord]; !ok && strings.HasPrefix(splitWord, "@") {
-					m.OtherPotentialMentions = append(m.OtherPotentialMentions, splitWord[1:])
-				}
-			}
-		}
-
-		if ids, match := isKeywordMultibyte(keywords, word); match {
-			m.addMentions(ids, KeywordMention)
-		}
-	}
-}
-
 func (a *App) GetNotificationNameFormat(user *model.User) string {
 	if !*a.Config().PrivacySettings.ShowFullName {
 		return model.ShowUsername
@@ -1563,7 +1302,7 @@ type CRTNotifiers struct {
 	Push model.StringArray
 }

-func (c *CRTNotifiers) addFollowerToNotify(user *model.User, mentions *ExplicitMentions, channelMemberNotificationProps model.StringMap, channel *model.Channel) {
+func (c *CRTNotifiers) addFollowerToNotify(user *model.User, mentions *MentionResults, channelMemberNotificationProps model.StringMap, channel *model.Channel) {
 	_, userWasMentioned := mentions.Mentions[user.Id]
 	notifyDesktop, notifyPush, notifyEmail := shouldUserNotifyCRT(user, userWasMentioned)
 	notifyChannelDesktop, notifyChannelPush := shouldChannelMemberNotifyCRT(channelMemberNotificationProps, userWasMentioned)