MM-44576 autocomplete names including utf 8 chars (#20367)

* Added unaccent extension

* Changed comment

* Deletes migrations, changes query method

* Mionr changes

* Creates test for multilingual queries

* Minor formatting changes

* Adds two more tests

* Changes name in test function

* Minor change

* Retriggers tests

* Removes % from Postgres query

* Lint fix

* Changes variable name

* Removes SetDefaultTextSearchConfig method

* Removes mocks

* Adds error handling on test raw queries

* Lint fix

* Deletes unused generated file

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

store/sqlstore/user_store.go

@@ -1500,17 +1500,20 @@ func generateSearchQuery(query sq.SelectBuilder, terms []string, fields []string
 	for _, term := range terms {
 		searchFields := []string{}
 		termArgs := []any{}
+		var dbSpecificTerm string
+
 		for _, field := range fields {
 			if isPostgreSQL {
-				searchFields = append(searchFields, fmt.Sprintf("lower(%s) LIKE lower(?) escape '*' ", field))
+				searchFields = append(searchFields, fmt.Sprintf("to_tsvector(lower(%[1]s)) @@ to_tsquery(concat(lower(?),':*'))", field))
+				dbSpecificTerm = strings.TrimLeft(term, "@")
 			} else {
 				searchFields = append(searchFields, fmt.Sprintf("%s LIKE ? escape '*' ", field))
+				dbSpecificTerm = fmt.Sprintf("%s%%", strings.TrimLeft(term, "@"))
 			}
-			termArgs = append(termArgs, fmt.Sprintf("%s%%", strings.TrimLeft(term, "@")))
+			termArgs = append(termArgs, dbSpecificTerm)
 		}
 		query = query.Where(fmt.Sprintf("(%s)", strings.Join(searchFields, " OR ")), termArgs...)
 	}
-
 	return query
 }
 

store/storetest/user_store.go

@@ -94,6 +94,7 @@ func TestUserStore(t *testing.T, ss store.Store, s SqlStore) {
 	t.Run("ResetLastPictureUpdate", func(t *testing.T) { testUserStoreResetLastPictureUpdate(t, ss) })
 	t.Run("GetKnownUsers", func(t *testing.T) { testGetKnownUsers(t, ss) })
 	t.Run("GetUsersWithInvalidEmails", func(t *testing.T) { testGetUsersWithInvalidEmails(t, ss) })
+	t.Run("SearchMultilingual", func(t *testing.T) { testUserStoreSearchUsersMultilingual(t, ss, s) })
 }
 
 func testUserStoreSave(t *testing.T, ss store.Store) {
@@ -6010,3 +6011,174 @@ func testGetUsersWithInvalidEmails(t *testing.T, ss store.Store) {
 	require.NoError(t, err)
 	assert.Len(t, users, 1)
 }
+
+func testUserStoreSearchUsersMultilingual(t *testing.T, ss store.Store, s SqlStore) {
+	u1 := &model.User{
+		Username:  "test1" + model.NewId(),
+		FirstName: "Inígo",
+		LastName:  "Martínez",
+		Nickname:  "Berridi",
+		Email:     MakeEmail(),
+	}
+	_, err := ss.User().Save(u1)
+	require.NoError(t, err)
+	defer func() { require.NoError(t, ss.User().PermanentDelete(u1.Id)) }()
+
+	u2 := &model.User{
+		Username:  "test2" + model.NewId(),
+		FirstName: "Zinëdìne",
+		LastName:  "Zidanë",
+		Email:     MakeEmail(),
+	}
+	_, err = ss.User().Save(u2)
+	require.NoError(t, err)
+	defer func() { require.NoError(t, ss.User().PermanentDelete(u2.Id)) }()
+
+	u3 := &model.User{
+		Username:  "test3" + model.NewId(),
+		FirstName: "Thomas ",
+		LastName:  "Müller",
+		Nickname:  "Fußballspieler",
+		Email:     MakeEmail(),
+	}
+	_, err = ss.User().Save(u3)
+	require.NoError(t, err)
+	defer func() { require.NoError(t, ss.User().PermanentDelete(u3.Id)) }()
+
+	u4 := &model.User{
+		Username:  "test4" + model.NewId(),
+		FirstName: "Jérémie ",
+		LastName:  "Jéry",
+		Email:     MakeEmail(),
+	}
+	_, err = ss.User().Save(u4)
+	require.NoError(t, err)
+	defer func() { require.NoError(t, ss.User().PermanentDelete(u4.Id)) }()
+
+	// The users returned from the database will have AuthData as an empty string.
+	nilAuthData := new(string)
+	*nilAuthData = ""
+	u1.AuthData = nilAuthData
+	u2.AuthData = nilAuthData
+	u3.AuthData = nilAuthData
+	u4.AuthData = nilAuthData
+
+	testCases := []struct {
+		Description      string
+		Term             string
+		Options          *model.UserSearchOptions
+		ExpectedPostgres []*model.User
+		ExpectedMysql    []*model.User
+		Language         string
+	}{
+		{
+			"search test1 player",
+			"inig",
+			&model.UserSearchOptions{
+				AllowFullNames: true,
+				Limit:          model.UserSearchDefaultLimit,
+			},
+			[]*model.User{u1},
+			[]*model.User{u1},
+			"spanish",
+		},
+		{
+			"search test2 player",
+			"zine",
+			&model.UserSearchOptions{
+				AllowFullNames: true,
+				Limit:          model.UserSearchDefaultLimit,
+			},
+			[]*model.User{u2},
+			[]*model.User{u2},
+			"french",
+		},
+		{
+			"search test2 player",
+			"zidane",
+			&model.UserSearchOptions{
+				AllowFullNames: true,
+				Limit:          model.UserSearchDefaultLimit,
+			},
+			[]*model.User{u2},
+			[]*model.User{u2},
+			"french",
+		},
+		{
+			"search test3 player",
+			"muller",
+			&model.UserSearchOptions{
+				AllowFullNames: true,
+				Limit:          model.UserSearchDefaultLimit,
+			},
+			[]*model.User{u3},
+			[]*model.User{u3},
+			"german",
+		},
+		{
+			"search test3 player",
+			"muller",
+			&model.UserSearchOptions{
+				AllowFullNames: true,
+				Limit:          model.UserSearchDefaultLimit,
+			},
+			[]*model.User{},
+			[]*model.User{u3},
+			"english",
+		},
+		{
+			"search test4 player",
+			"jere",
+			&model.UserSearchOptions{
+				AllowFullNames: true,
+				Limit:          model.UserSearchDefaultLimit,
+			},
+			[]*model.User{u4},
+			[]*model.User{u4},
+			"spanish",
+		},
+		{
+			"search test4 player",
+			"jere",
+			&model.UserSearchOptions{
+				AllowFullNames: true,
+				Limit:          model.UserSearchDefaultLimit,
+			},
+			[]*model.User{},
+			[]*model.User{u4},
+			"english",
+		},
+	}
+
+	var initialDefaultTextSearchConfig string
+	if s.DriverName() == model.DatabaseDriverPostgres {
+		error := s.GetMasterX().Get(&initialDefaultTextSearchConfig, `SHOW default_text_search_config`)
+		require.NoError(t, error)
+	}
+
+	for _, testCase := range testCases {
+		if s.DriverName() == model.DatabaseDriverPostgres {
+			_, error := s.GetMasterX().Exec("SET default_text_search_config TO '" + testCase.Language + "'")
+			require.NoError(t, error)
+		}
+		t.Run(testCase.Description, func(t *testing.T) {
+			users, err := ss.User().SearchWithoutTeam(
+				testCase.Term,
+				testCase.Options,
+			)
+
+			if s.DriverName() != model.DatabaseDriverPostgres {
+				require.NoError(t, err)
+				assertUsers(t, testCase.ExpectedMysql, users)
+			} else {
+				require.NoError(t, err)
+				assertUsers(t, testCase.ExpectedPostgres, users)
+			}
+		})
+	}
+
+	if s.DriverName() == model.DatabaseDriverPostgres {
+		_, error := s.GetMasterX().Exec("SET default_text_search_config TO '" + initialDefaultTextSearchConfig + "'")
+		require.NoError(t, error)
+	}
+}