MM-23489 Fix poor Unicode handling in display names (#14214)

Filter blacklisted Unicode characters from:

user: first name, last name, nickname, bot description, username
team: name, display name, description, company name
channel: name, display name

model/channel.go

@@ -230,6 +230,9 @@ func (o *Channel) PreSave() {
 		o.Id = NewId()
 	}
 
+	o.Name = SanitizeUnicode(o.Name)
+	o.DisplayName = SanitizeUnicode(o.DisplayName)
+
 	o.CreateAt = GetMillis()
 	o.UpdateAt = o.CreateAt
 	o.ExtraUpdateAt = 0
@@ -237,6 +240,8 @@ func (o *Channel) PreSave() {
 
 func (o *Channel) PreUpdate() {
 	o.UpdateAt = GetMillis()
+	o.Name = SanitizeUnicode(o.Name)
+	o.DisplayName = SanitizeUnicode(o.DisplayName)
 }
 
 func (o *Channel) IsGroupOrDirect() bool {

model/team.go

@@ -203,6 +203,11 @@ func (o *Team) PreSave() {
 	o.CreateAt = GetMillis()
 	o.UpdateAt = o.CreateAt
 
+	o.Name = SanitizeUnicode(o.Name)
+	o.DisplayName = SanitizeUnicode(o.DisplayName)
+	o.Description = SanitizeUnicode(o.Description)
+	o.CompanyName = SanitizeUnicode(o.CompanyName)
+
 	if len(o.InviteId) == 0 {
 		o.InviteId = NewId()
 	}
@@ -210,6 +215,10 @@ func (o *Team) PreSave() {
 
 func (o *Team) PreUpdate() {
 	o.UpdateAt = GetMillis()
+	o.Name = SanitizeUnicode(o.Name)
+	o.DisplayName = SanitizeUnicode(o.DisplayName)
+	o.Description = SanitizeUnicode(o.Description)
+	o.CompanyName = SanitizeUnicode(o.CompanyName)
 }
 
 func IsReservedTeamName(s string) bool {

model/user.go

@@ -329,6 +329,11 @@ func (u *User) PreSave() {
 		u.AuthData = nil
 	}
 
+	u.Username = SanitizeUnicode(u.Username)
+	u.FirstName = SanitizeUnicode(u.FirstName)
+	u.LastName = SanitizeUnicode(u.LastName)
+	u.Nickname = SanitizeUnicode(u.Nickname)
+
 	u.Username = NormalizeUsername(u.Username)
 	u.Email = NormalizeEmail(u.Email)
 
@@ -362,10 +367,21 @@ func (u *User) PreSave() {
 
 // PreUpdate should be run before updating the user in the db.
 func (u *User) PreUpdate() {
+	u.Username = SanitizeUnicode(u.Username)
+	u.FirstName = SanitizeUnicode(u.FirstName)
+	u.LastName = SanitizeUnicode(u.LastName)
+	u.Nickname = SanitizeUnicode(u.Nickname)
+	u.BotDescription = SanitizeUnicode(u.BotDescription)
+
 	u.Username = NormalizeUsername(u.Username)
 	u.Email = NormalizeEmail(u.Email)
 	u.UpdateAt = GetMillis()
 
+	u.FirstName = SanitizeUnicode(u.FirstName)
+	u.LastName = SanitizeUnicode(u.LastName)
+	u.Nickname = SanitizeUnicode(u.Nickname)
+	u.BotDescription = SanitizeUnicode(u.BotDescription)
+
 	if u.AuthData != nil && *u.AuthData == "" {
 		u.AuthData = nil
 	}

model/utils.go

@@ -647,3 +647,48 @@ func AsStringBoolMap(list []string) map[string]bool {
 	}
 	return listMap
 }
+
+// SanitizeUnicode will remove undesirable Unicode characters from a string.
+func SanitizeUnicode(s string) string {
+	return strings.Map(filterBlacklist, s)
+}
+
+// filterBlacklist returns `r` if it is not in the blacklist, otherwise drop (-1).
+// Blacklist is taken from https://www.w3.org/TR/unicode-xml/#Charlist
+func filterBlacklist(r rune) rune {
+	const drop = -1
+	switch r {
+	case '\u0340', '\u0341': // clones of grave and acute; deprecated in Unicode
+		return drop
+	case '\u17A3', '\u17D3': // obsolete characters for Khmer; deprecated in Unicode
+		return drop
+	case '\u2028', '\u2029': // line and paragraph separator
+		return drop
+	case '\u202A', '\u202B', '\u202C', '\u202D', '\u202E': // BIDI embedding controls
+		return drop
+	case '\u206A', '\u206B': // activate/inhibit symmetric swapping; deprecated in Unicode
+		return drop
+	case '\u206C', '\u206D': // activate/inhibit Arabic form shaping; deprecated in Unicode
+		return drop
+	case '\u206E', '\u206F': // activate/inhibit national digit shapes; deprecated in Unicode
+		return drop
+	case '\uFFF9', '\uFFFA', '\uFFFB': // interlinear annotation characters
+		return drop
+	case '\uFEFF': // byte order mark
+		return drop
+	case '\uFFFC': // object replacement character
+		return drop
+	}
+
+	// Scoping for musical notation
+	if r >= 0x0001D173 && r <= 0x0001D17A {
+		return drop
+	}
+
+	// Language tag code points
+	if r >= 0x000E0000 && r <= 0x000E007F {
+		return drop
+	}
+
+	return r
+}

model/utils_test.go

@@ -4,6 +4,7 @@
 package model
 
 import (
+	"bytes"
 	"fmt"
 	"net/http"
 	"reflect"
@@ -736,3 +737,33 @@ func checkNowhereNil(t *testing.T, name string, value interface{}) bool {
 		return true
 	}
 }
+
+func TestSanitizeUnicode(t *testing.T) {
+	buf := bytes.Buffer{}
+	buf.WriteString("Hello")
+	buf.WriteRune(0x1d173)
+	buf.WriteRune(0x1d17a)
+	buf.WriteString(" there.")
+
+	musicArg := buf.String()
+	musicWant := "Hello there."
+
+	tests := []struct {
+		name string
+		arg  string
+		want string
+	}{
+		{name: "empty string", arg: "", want: ""},
+		{name: "ascii only", arg: "Hello There", want: "Hello There"},
+		{name: "allowed unicode", arg: "Ādam likes Iñtërnâtiônàližætiøn", want: "Ādam likes Iñtërnâtiônàližætiøn"},
+		{name: "allowed unicode escaped", arg: "\u00eaI like hats\u00e2", want: "êI like hatsâ"},
+		{name: "blacklist char, don't reverse string", arg: "\u202E2resu", want: "2resu"},
+		{name: "blacklist chars, scoping musical notation", arg: musicArg, want: musicWant},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := SanitizeUnicode(tt.arg)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}