MM-54556: Integrate wip i18n languages (#24308)

* merge languages from https://github.com/mattermost/i18n-wip

* allow only supported server locales

* Revert "merge languages from https://github.com/mattermost/i18n-wip"

This reverts commit 36de545102faffe1e8ba943decdab04bc4f5a55c. We'll let
weblate populate these on start instead.

* copy fileutils to public/utils

---------

Co-authored-by: Mattermost Build <build@mattermost.com>

server/config/utils.go

@@ -116,14 +116,14 @@ func FixInvalidLocales(cfg *model.Config) bool {
 
 	locales := i18n.GetSupportedLocales()
 	if _, ok := locales[*cfg.LocalizationSettings.DefaultServerLocale]; !ok {
+		mlog.Warn("DefaultServerLocale must be one of the supported locales. Setting DefaultServerLocale to en as default value.", mlog.String("locale", *cfg.LocalizationSettings.DefaultServerLocale))
 		*cfg.LocalizationSettings.DefaultServerLocale = model.DefaultLocale
-		mlog.Warn("DefaultServerLocale must be one of the supported locales. Setting DefaultServerLocale to en as default value.")
 		changed = true
 	}
 
 	if _, ok := locales[*cfg.LocalizationSettings.DefaultClientLocale]; !ok {
+		mlog.Warn("DefaultClientLocale must be one of the supported locales. Setting DefaultClientLocale to en as default value.", mlog.String("locale", *cfg.LocalizationSettings.DefaultClientLocale))
 		*cfg.LocalizationSettings.DefaultClientLocale = model.DefaultLocale
-		mlog.Warn("DefaultClientLocale must be one of the supported locales. Setting DefaultClientLocale to en as default value.")
 		changed = true
 	}
 

server/public/shared/i18n/i18n.go

@@ -33,6 +33,32 @@ var T TranslateFunc
 var TDefault TranslateFunc
 
 var locales = make(map[string]string)
+
+// supportedLocales is a hard-coded list of locales considered ready for production use. It must
+// be kept in sync with ../../../../webapp/channels/src/i18n/i18n.jsx.
+var supportedLocales = []string{
+	"de",
+	"en",
+	"en-AU",
+	"es",
+	"fr",
+	"it",
+	"hu",
+	"nl",
+	"pl",
+	"pt-BR",
+	"ro",
+	"sv",
+	"tr",
+	"bg",
+	"ru",
+	"uk",
+	"fa",
+	"ko",
+	"zh-CN",
+	"zh-TW",
+	"ja",
+}
 var defaultServerLocale string
 var defaultClientLocale string
 
@@ -67,7 +93,13 @@ func initTranslationsWithDir(dir string) error {
 	for _, f := range files {
 		if filepath.Ext(f.Name()) == ".json" {
 			filename := f.Name()
-			locales[strings.Split(filename, ".")[0]] = filepath.Join(dir, filename)
+
+			locale := strings.Split(filename, ".")[0]
+			if !isSupportedLocale(locale) {
+				continue
+			}
+
+			locales[locale] = filepath.Join(dir, filename)
 
 			if err := i18n.LoadTranslationFile(filepath.Join(dir, filename)); err != nil {
 				return err
@@ -89,8 +121,13 @@ func GetTranslationFuncForDir(dir string) (TranslationFuncByLocal, error) {
 			continue
 		}
 
+		locale := strings.Split(f.Name(), ".")[0]
+		if !isSupportedLocale(locale) {
+			continue
+		}
+
 		filename := f.Name()
-		availableLocals[strings.Split(filename, ".")[0]] = filepath.Join(dir, filename)
+		availableLocals[locale] = filepath.Join(dir, filename)
 		if err := bundle.LoadTranslationFile(filepath.Join(dir, filename)); err != nil {
 			return nil, err
 		}
@@ -116,7 +153,12 @@ func GetTranslationFuncForDir(dir string) (TranslationFuncByLocal, error) {
 func getTranslationsBySystemLocale() (TranslateFunc, error) {
 	locale := defaultServerLocale
 	if _, ok := locales[locale]; !ok {
-		mlog.Warn("Failed to load system translations for", mlog.String("locale", locale), mlog.String("attempting to fall back to default locale", defaultLocale))
+		mlog.Warn("Failed to load system translations for selected locale, attempting to fall back to default", mlog.String("locale", locale), mlog.String("default_locale", defaultLocale))
+		locale = defaultLocale
+	}
+
+	if !isSupportedLocale(locale) {
+		mlog.Warn("Selected locale is unsupported, attempting to fall back to default", mlog.String("locale", locale), mlog.String("default_locale", defaultLocale))
 		locale = defaultLocale
 	}
 
@@ -222,3 +264,13 @@ func IdentityTfunc() TranslateFunc {
 		return translationID
 	}
 }
+
+func isSupportedLocale(locale string) bool {
+	for _, supportedLocale := range supportedLocales {
+		if locale == supportedLocale {
+			return true
+		}
+	}
+
+	return false
+}

server/public/shared/i18n/i18n_test.go

@@ -4,12 +4,17 @@
 package i18n
 
 import (
+	"fmt"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/mattermost/go-i18n/i18n/bundle"
 	"github.com/mattermost/go-i18n/i18n/language"
 	"github.com/mattermost/go-i18n/i18n/translation"
+	"github.com/mattermost/mattermost/server/public/utils"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 var htmlTestTranslationBundle *bundle.Bundle
@@ -67,3 +72,155 @@ func TestEscapeForHTML(t *testing.T) {
 		})
 	}
 }
+
+func TestInitTranslationsWithDir(t *testing.T) {
+	i18nDir, found := utils.FindDir("server/i18n")
+	require.True(t, found, "unable to find i18n dir")
+
+	setup := func(t *testing.T, localesToCopy map[string]string) string {
+		tempDir, err := os.MkdirTemp(os.TempDir(), "TestGetTranslationFuncForDir")
+		require.NoError(t, err, "unable to create temporary directory")
+
+		t.Cleanup(func() {
+			err = os.RemoveAll(tempDir)
+			require.NoError(t, err)
+		})
+
+		for locale, fromLocale := range localesToCopy {
+			err = utils.CopyFile(
+				filepath.Join(i18nDir, fmt.Sprintf("%s.json", fromLocale)),
+				filepath.Join(tempDir, fmt.Sprintf("%s.json", locale)),
+			)
+			require.NoError(t, err)
+		}
+
+		return tempDir
+	}
+
+	t.Run("unsupported locale ignored", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"en": "en", "fr": "fr", "zz": "en"})
+
+		err := initTranslationsWithDir(tempDir)
+		require.NoError(t, err)
+
+		_, found := locales["zz"]
+		require.False(t, found, "should have ignored unsupported locale")
+	})
+
+	t.Run("malformed, unsupported locale ignored", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"en": "en", "fr": "fr", "zz": "en"})
+
+		err := os.WriteFile(filepath.Join(tempDir, "xx.json"), []byte{'{'}, os.ModePerm)
+		require.NoError(t, err)
+
+		err = initTranslationsWithDir(tempDir)
+		require.NoError(t, err)
+
+		_, found := locales["xx"]
+		require.False(t, found, "should have ignored malformed, unsupported locale")
+	})
+
+	t.Run("malformed, supported locale causes error", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"fr": "fr", "zz": "en"})
+
+		err := os.WriteFile(filepath.Join(tempDir, "en.json"), []byte{'{'}, os.ModePerm)
+		require.NoError(t, err)
+
+		err = initTranslationsWithDir(tempDir)
+		require.Error(t, err, "should have failed to load malformed, supported locale")
+	})
+
+	t.Run("known locales loaded ", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"en": "en", "fr": "fr"})
+
+		err := initTranslationsWithDir(tempDir)
+		require.NoError(t, err)
+
+		_, found := locales["en"]
+		require.True(t, found, "should have found en locale")
+		_, found = locales["fr"]
+		require.True(t, found, "should have found fr locale")
+		_, found = locales["es"]
+		require.False(t, found, "should not have found unloaded es locale")
+	})
+}
+
+func TestGetTranslationFuncForDir(t *testing.T) {
+	i18nDir, found := utils.FindDir("server/i18n")
+	require.True(t, found, "unable to find i18n dir")
+
+	setup := func(t *testing.T, localesToCopy map[string]string) string {
+		tempDir, err := os.MkdirTemp(os.TempDir(), "TestGetTranslationFuncForDir")
+		require.NoError(t, err, "unable to create temporary directory")
+
+		t.Cleanup(func() {
+			err = os.RemoveAll(tempDir)
+			require.NoError(t, err)
+		})
+
+		for locale, fromLocale := range localesToCopy {
+			err = utils.CopyFile(
+				filepath.Join(i18nDir, fmt.Sprintf("%s.json", fromLocale)),
+				filepath.Join(tempDir, fmt.Sprintf("%s.json", locale)),
+			)
+			require.NoError(t, err)
+		}
+
+		return tempDir
+	}
+
+	t.Run("unknown locale falls back to english", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"en": "en", "fr": "fr", "zz": "en"})
+
+		translationFunc, err := GetTranslationFuncForDir(tempDir)
+		require.NoError(t, err)
+		require.NotNil(t, translationFunc)
+
+		require.Equal(t, "December", translationFunc("unknown")("December"))
+	})
+
+	t.Run("unsupported locale falls back to english", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"en": "en", "fr": "fr", "zz": "en"})
+
+		translationFunc, err := GetTranslationFuncForDir(tempDir)
+		require.NoError(t, err)
+		require.NotNil(t, translationFunc)
+
+		require.Equal(t, "December", translationFunc("zz")("December"))
+	})
+
+	t.Run("malformed, unsupported locale ignored and falls back to english", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"en": "en", "fr": "fr", "zz": "en"})
+
+		err := os.WriteFile(filepath.Join(tempDir, "xx.json"), []byte{'{'}, os.ModePerm)
+		require.NoError(t, err)
+
+		translationFunc, err := GetTranslationFuncForDir(tempDir)
+		require.NoError(t, err)
+		require.NotNil(t, translationFunc)
+
+		require.Equal(t, "December", translationFunc("xx")("December"))
+	})
+
+	t.Run("malformed, supported locale causes error", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"fr": "fr", "zz": "en"})
+
+		err := os.WriteFile(filepath.Join(tempDir, "en.json"), []byte{'{'}, os.ModePerm)
+		require.NoError(t, err)
+
+		translationFunc, err := GetTranslationFuncForDir(tempDir)
+		require.Error(t, err)
+		require.Nil(t, translationFunc)
+	})
+
+	t.Run("known locale matches", func(t *testing.T) {
+		tempDir := setup(t, map[string]string{"en": "en", "fr": "fr"})
+
+		translationFunc, err := GetTranslationFuncForDir(tempDir)
+		require.NoError(t, err)
+		require.NotNil(t, translationFunc)
+
+		require.Equal(t, "Décembre", translationFunc("fr")("December"))
+		require.Equal(t, "December", translationFunc("en")("December"))
+	})
+}

server/public/utils/fileutils.go

@@ -0,0 +1,118 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package utils
+
+import (
+	"os"
+	"path/filepath"
+)
+
+func CommonBaseSearchPaths() []string {
+	paths := []string{
+		".",
+		"..",
+		"../..",
+		"../../..",
+		"../../../..",
+	}
+
+	// this enables the server to be used in tests from a different repository
+	if mmPath := os.Getenv("MM_SERVER_PATH"); mmPath != "" {
+		paths = append(paths, mmPath)
+	}
+
+	return paths
+}
+
+func findPath(path string, baseSearchPaths []string, workingDirFirst bool, filter func(os.FileInfo) bool) string {
+	if filepath.IsAbs(path) {
+		if _, err := os.Stat(path); err == nil {
+			return path
+		}
+
+		return ""
+	}
+
+	searchPaths := []string{}
+	if workingDirFirst {
+		searchPaths = append(searchPaths, baseSearchPaths...)
+	}
+
+	// Attempt to search relative to the location of the running binary either before
+	// or after searching relative to the working directory, depending on `workingDirFirst`.
+	var binaryDir string
+	if exe, err := os.Executable(); err == nil {
+		if exe, err = filepath.EvalSymlinks(exe); err == nil {
+			if exe, err = filepath.Abs(exe); err == nil {
+				binaryDir = filepath.Dir(exe)
+			}
+		}
+	}
+	if binaryDir != "" {
+		for _, baseSearchPath := range baseSearchPaths {
+			searchPaths = append(
+				searchPaths,
+				filepath.Join(binaryDir, baseSearchPath),
+			)
+		}
+	}
+
+	if !workingDirFirst {
+		searchPaths = append(searchPaths, baseSearchPaths...)
+	}
+
+	for _, parent := range searchPaths {
+		found, err := filepath.Abs(filepath.Join(parent, path))
+		if err != nil {
+			continue
+		} else if fileInfo, err := os.Stat(found); err == nil {
+			if filter != nil {
+				if filter(fileInfo) {
+					return found
+				}
+			} else {
+				return found
+			}
+		}
+	}
+
+	return ""
+}
+
+func FindPath(path string, baseSearchPaths []string, filter func(os.FileInfo) bool) string {
+	return findPath(path, baseSearchPaths, true, filter)
+}
+
+// FindFile looks for the given file in nearby ancestors relative to the current working
+// directory as well as the directory of the executable.
+func FindFile(path string) string {
+	return FindPath(path, CommonBaseSearchPaths(), func(fileInfo os.FileInfo) bool {
+		return !fileInfo.IsDir()
+	})
+}
+
+// fileutils.FindDir looks for the given directory in nearby ancestors relative to the current working
+// directory as well as the directory of the executable, falling back to `./` if not found.
+func FindDir(dir string) (string, bool) {
+	found := FindPath(dir, CommonBaseSearchPaths(), func(fileInfo os.FileInfo) bool {
+		return fileInfo.IsDir()
+	})
+	if found == "" {
+		return "./", false
+	}
+
+	return found, true
+}
+
+// FindDirRelBinary looks for the given directory in nearby ancestors relative to the
+// directory of the executable, then relative to the working directory, falling back to `./` if not found.
+func FindDirRelBinary(dir string) (string, bool) {
+	found := findPath(dir, CommonBaseSearchPaths(), false, func(fileInfo os.FileInfo) bool {
+		return fileInfo.IsDir()
+	})
+	if found == "" {
+		return "./", false
+	}
+	return found, true
+}

server/public/utils/fileutils_test.go

@@ -0,0 +1,123 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package utils
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFindFile(t *testing.T) {
+	t.Run("files from various paths", func(t *testing.T) {
+		// Create the following directory structure:
+		// tmpDir1/
+		//   file1.json
+		//   file2.xml
+		//   other.txt
+		//   tmpDir2/
+		//     other.txt/ [directory]
+		//     tmpDir3/
+		//       tmpDir4/
+		//         tmpDir5/
+		tmpDir1, err := os.MkdirTemp("", "")
+		require.NoError(t, err)
+		defer os.RemoveAll(tmpDir1)
+
+		tmpDir2, err := os.MkdirTemp(tmpDir1, "")
+		require.NoError(t, err)
+
+		err = os.Mkdir(filepath.Join(tmpDir2, "other.txt"), 0700)
+		require.NoError(t, err)
+
+		tmpDir3, err := os.MkdirTemp(tmpDir2, "")
+		require.NoError(t, err)
+
+		tmpDir4, err := os.MkdirTemp(tmpDir3, "")
+		require.NoError(t, err)
+
+		tmpDir5, err := os.MkdirTemp(tmpDir4, "")
+		require.NoError(t, err)
+
+		type testCase struct {
+			Description string
+			Cwd         *string
+			FileName    string
+			Expected    string
+		}
+
+		testCases := []testCase{}
+
+		for _, fileName := range []string{"file1.json", "file2.xml", "other.txt"} {
+			filePath := filepath.Join(tmpDir1, fileName)
+			require.NoError(t, os.WriteFile(filePath, []byte("{}"), 0600))
+
+			// Relative paths end up getting symlinks fully resolved, so use this below as necessary.
+			filePathResolved, err := filepath.EvalSymlinks(filePath)
+			require.NoError(t, err)
+
+			testCases = append(testCases, []testCase{
+				{
+					fmt.Sprintf("absolute path to %s", fileName),
+					nil,
+					filePath,
+					filePath,
+				},
+				{
+					fmt.Sprintf("absolute path to %s from containing directory", fileName),
+					&tmpDir1,
+					filePath,
+					filePath,
+				},
+				{
+					fmt.Sprintf("relative path to %s from containing directory", fileName),
+					&tmpDir1,
+					fileName,
+					filePathResolved,
+				},
+				{
+					fmt.Sprintf("%s: subdirectory of containing directory", fileName),
+					&tmpDir2,
+					fileName,
+					filePathResolved,
+				},
+				{
+					fmt.Sprintf("%s: twice-nested subdirectory of containing directory", fileName),
+					&tmpDir3,
+					fileName,
+					filePathResolved,
+				},
+				{
+					fmt.Sprintf("%s: thrice-nested subdirectory of containing directory", fileName),
+					&tmpDir4,
+					fileName,
+					filePathResolved,
+				},
+				{
+					fmt.Sprintf("%s: quadruple-nested subdirectory of containing directory", fileName),
+					&tmpDir5,
+					fileName,
+					filePathResolved,
+				},
+			}...)
+		}
+
+		for _, testCase := range testCases {
+			t.Run(testCase.Description, func(t *testing.T) {
+				if testCase.Cwd != nil {
+					prevDir, err := os.Getwd()
+					require.NoError(t, err)
+					defer os.Chdir(prevDir)
+					os.Chdir(*testCase.Cwd)
+				}
+
+				assert.Equal(t, testCase.Expected, FindFile(testCase.FileName))
+			})
+		}
+	})
+}