Extracting i18n functionality to i18n core library (#16914)

* extracting i18n functionality to i18n core library

* Removing utils.T

* Adding documentation and changing one function name for better explanation

* Changing other missing utils.T

* Adding license string

* Renaming corelibs to pkg

* Renaming corelibs to pkg (moving directory)

* Renaming from pkg to shared

* Fixing bodyPage.Html casing

* Fixing merges

* Fixing merge problem

* Fixing tests

utils/api.go

@@ -16,6 +16,7 @@ import (
 	"unicode/utf8"
 
 	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/shared/i18n"
 )
 
 func CheckOrigin(r *http.Request, allowedOrigins string) bool {
@@ -121,9 +122,9 @@ func RenderMobileAuthComplete(w http.ResponseWriter, redirectURL string) {
 		<div class="icon text-success" style="font-size: 4em">
 			<i class="fa fa-check-circle" title="Success Icon"></i>
 		</div>
-		<h2> `+T("api.oauth.auth_complete")+` </h2>
-		<p id="redirecting-message"> `+T("api.oauth.redirecting_back")+` </p>
-		<p id="close-tab-message" style="display: none"> `+T("api.oauth.close_browser")+` </p>
+		<h2> `+i18n.T("api.oauth.auth_complete")+` </h2>
+		<p id="redirecting-message"> `+i18n.T("api.oauth.redirecting_back")+` </p>
+		<p id="close-tab-message" style="display: none"> `+i18n.T("api.oauth.close_browser")+` </p>
 		<noscript><meta http-equiv="refresh" content="2; url=`+template.HTMLEscapeString(redirectURL)+`"></noscript>
 		<script>
 			window.onload = function() {
@@ -142,10 +143,10 @@ func RenderMobileError(config *model.Config, w http.ResponseWriter, err *model.A
 		<div class="icon" style="color: #ccc; font-size: 4em">
 			<span class="fa fa-warning"></span>
 		</div>
-		<h2> `+T("error")+` </h2>
+		<h2> `+i18n.T("error")+` </h2>
 		<p> `+err.Message+` </p>
 		<a href="`+redirectURL+`">
-			`+T("api.back_to_app", map[string]interface{}{"SiteName": config.TeamSettings.SiteName})+`
+			`+i18n.T("api.back_to_app", map[string]interface{}{"SiteName": config.TeamSettings.SiteName})+`
 		</a>
 	`)
 }

utils/html.go

@@ -9,12 +9,9 @@ import (
 	"html/template"
 	"io"
 	"path/filepath"
-	"reflect"
-	"strings"
 	"sync/atomic"
 
 	"github.com/fsnotify/fsnotify"
-	"github.com/mattermost/go-i18n/i18n"
 
 	"github.com/mattermost/mattermost-server/v5/mlog"
 	"github.com/mattermost/mattermost-server/v5/utils/fileutils"
@@ -119,32 +116,3 @@ func (t *HTMLTemplate) RenderToWriter(w io.Writer) error {
 
 	return nil
 }
-
-func TranslateAsHTML(t i18n.TranslateFunc, translationID string, args map[string]interface{}) template.HTML {
-	message := t(translationID, escapeForHTML(args))
-	message = strings.Replace(message, "[[", "<strong>", -1)
-	message = strings.Replace(message, "]]", "</strong>", -1)
-	return template.HTML(message)
-}
-
-func escapeForHTML(arg interface{}) interface{} {
-	switch typedArg := arg.(type) {
-	case string:
-		return template.HTMLEscapeString(typedArg)
-	case *string:
-		return template.HTMLEscapeString(*typedArg)
-	case map[string]interface{}:
-		safeArg := make(map[string]interface{}, len(typedArg))
-		for key, value := range typedArg {
-			safeArg[key] = escapeForHTML(value)
-		}
-		return safeArg
-	default:
-		mlog.Warn(
-			"Unable to escape value for HTML template",
-			mlog.Any("html_template", arg),
-			mlog.String("template_type", reflect.ValueOf(arg).Type().String()),
-		)
-		return ""
-	}
-}

utils/html_test.go

@@ -12,27 +12,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/mattermost/go-i18n/i18n"
-	"github.com/mattermost/go-i18n/i18n/bundle"
-	"github.com/mattermost/go-i18n/i18n/language"
-	"github.com/mattermost/go-i18n/i18n/translation"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-
-	"github.com/mattermost/mattermost-server/v5/model"
 )
 
-var htmlTestTranslationBundle *bundle.Bundle
-
-func init() {
-	htmlTestTranslationBundle = bundle.New()
-	fooBold, _ := translation.NewTranslation(map[string]interface{}{
-		"id":          "foo.bold",
-		"translation": "<p>[[{{ .Foo }}]]</p>",
-	})
-	htmlTestTranslationBundle.AddTranslation(&language.Language{Tag: "en"}, fooBold)
-}
-
 func TestHTMLTemplateWatcher(t *testing.T) {
 	TranslationsPreInit()
 
@@ -101,47 +84,3 @@ func TestHTMLTemplate_RenderError(t *testing.T) {
 	assert.Error(t, htmlTemplate.RenderToWriter(buf))
 	assert.Equal(t, "foo", buf.String())
 }
-
-func TestTranslateAsHtml(t *testing.T) {
-	assert.EqualValues(t, "<p><strong>&lt;i&gt;foo&lt;/i&gt;</strong></p>", TranslateAsHTML(i18n.TranslateFunc(htmlTestTranslationBundle.MustTfunc("en")), "foo.bold", map[string]interface{}{
-		"Foo": "<i>foo</i>",
-	}))
-}
-
-func TestEscapeForHtml(t *testing.T) {
-	for name, tc := range map[string]struct {
-		In       interface{}
-		Expected interface{}
-	}{
-		"NoHTML": {
-			In:       "abc",
-			Expected: "abc",
-		},
-		"String": {
-			In:       "<b>abc</b>",
-			Expected: "&lt;b&gt;abc&lt;/b&gt;",
-		},
-		"StringPointer": {
-			In:       model.NewString("<b>abc</b>"),
-			Expected: "&lt;b&gt;abc&lt;/b&gt;",
-		},
-		"Map": {
-			In: map[string]interface{}{
-				"abc": "abc",
-				"123": "<b>123</b>",
-			},
-			Expected: map[string]interface{}{
-				"abc": "abc",
-				"123": "&lt;b&gt;123&lt;/b&gt;",
-			},
-		},
-		"Unsupported": {
-			In:       struct{ string }{"<b>abc</b>"},
-			Expected: "",
-		},
-	} {
-		t.Run(name, func(t *testing.T) {
-			assert.Equal(t, tc.Expected, escapeForHTML(tc.In))
-		})
-	}
-}

utils/i18n.go

@@ -5,135 +5,25 @@ package utils
 
 import (
 	"fmt"
-	"io/ioutil"
-	"net/http"
 	"os"
 	"path/filepath"
-	"strings"
 
-	"github.com/mattermost/go-i18n/i18n"
-
-	"github.com/mattermost/mattermost-server/v5/mlog"
-	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/shared/i18n"
 	"github.com/mattermost/mattermost-server/v5/utils/fileutils"
 )
 
-var T i18n.TranslateFunc
-var TDefault i18n.TranslateFunc
-var locales map[string]string = make(map[string]string)
-var settings model.LocalizationSettings
-
 // this functions loads translations from filesystem if they are not
 // loaded already and assigns english while loading server config
 func TranslationsPreInit() error {
-	if T != nil {
-		return nil
-	}
-
-	// Set T even if we fail to load the translations. Lots of shutdown handling code will
-	// segfault trying to handle the error, and the untranslated IDs are strictly better.
-	T = TfuncWithFallback("en")
-	TDefault = TfuncWithFallback("en")
-
 	translationsDir := "i18n"
 	if mattermostPath := os.Getenv("MM_SERVER_PATH"); mattermostPath != "" {
 		translationsDir = filepath.Join(mattermostPath, "i18n")
 	}
 
-	return InitTranslationsWithDir(translationsDir)
-}
-
-func InitTranslations(localizationSettings model.LocalizationSettings) error {
-	settings = localizationSettings
-
-	var err error
-	T, err = GetTranslationsBySystemLocale()
-	return err
-}
-
-func InitTranslationsWithDir(dir string) error {
-	i18nDirectory, found := fileutils.FindDirRelBinary(dir)
+	i18nDirectory, found := fileutils.FindDirRelBinary(translationsDir)
 	if !found {
-		return fmt.Errorf("unable to find i18n directory at %q", dir)
+		return fmt.Errorf("unable to find i18n directory at %q", translationsDir)
 	}
 
-	files, _ := ioutil.ReadDir(i18nDirectory)
-	for _, f := range files {
-		if filepath.Ext(f.Name()) == ".json" {
-			filename := f.Name()
-			locales[strings.Split(filename, ".")[0]] = filepath.Join(i18nDirectory, filename)
-
-			if err := i18n.LoadTranslationFile(filepath.Join(i18nDirectory, filename)); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func GetTranslationsBySystemLocale() (i18n.TranslateFunc, error) {
-	locale := *settings.DefaultServerLocale
-	if _, ok := locales[locale]; !ok {
-		mlog.Warn("Failed to load system translations for", mlog.String("locale", locale), mlog.String("attempting to fall back to default locale", model.DEFAULT_LOCALE))
-		locale = model.DEFAULT_LOCALE
-	}
-
-	if locales[locale] == "" {
-		return nil, fmt.Errorf("failed to load system translations for '%v'", model.DEFAULT_LOCALE)
-	}
-
-	translations := TfuncWithFallback(locale)
-	if translations == nil {
-		return nil, fmt.Errorf("failed to load system translations")
-	}
-
-	mlog.Info("Loaded system translations", mlog.String("for locale", locale), mlog.String("from locale", locales[locale]))
-	return translations, nil
-}
-
-func GetUserTranslations(locale string) i18n.TranslateFunc {
-	if _, ok := locales[locale]; !ok {
-		locale = model.DEFAULT_LOCALE
-	}
-
-	translations := TfuncWithFallback(locale)
-	return translations
-}
-
-func GetTranslationsAndLocale(r *http.Request) (i18n.TranslateFunc, string) {
-	// This is for checking against locales like pt_BR or zn_CN
-	headerLocaleFull := strings.Split(r.Header.Get("Accept-Language"), ",")[0]
-	// This is for checking against locales like en, es
-	headerLocale := strings.Split(strings.Split(r.Header.Get("Accept-Language"), ",")[0], "-")[0]
-	defaultLocale := *settings.DefaultClientLocale
-	if locales[headerLocaleFull] != "" {
-		translations := TfuncWithFallback(headerLocaleFull)
-		return translations, headerLocaleFull
-	} else if locales[headerLocale] != "" {
-		translations := TfuncWithFallback(headerLocale)
-		return translations, headerLocale
-	} else if locales[defaultLocale] != "" {
-		translations := TfuncWithFallback(defaultLocale)
-		return translations, headerLocale
-	}
-
-	translations := TfuncWithFallback(model.DEFAULT_LOCALE)
-	return translations, model.DEFAULT_LOCALE
-}
-
-func GetSupportedLocales() map[string]string {
-	return locales
-}
-
-func TfuncWithFallback(pref string) i18n.TranslateFunc {
-	t, _ := i18n.Tfunc(pref)
-	return func(translationID string, args ...interface{}) string {
-		if translated := t(translationID, args...); translated != translationID {
-			return translated
-		}
-
-		t, _ := i18n.Tfunc(model.DEFAULT_LOCALE)
-		return t(translationID, args...)
-	}
+	return i18n.TranslationsPreInit(i18nDirectory)
 }