// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. // See License.txt for license information. package commands import ( "bytes" "encoding/json" "errors" "fmt" "go/ast" "go/parser" "go/token" "log" "os" "path" "path/filepath" "sort" "strconv" "strings" "github.com/spf13/cobra" ) const enterpriseKeyPrefix = "ent." const untranslatedKey = "" type Translation struct { Id string `json:"id"` Translation interface{} `json:"translation"` } type Item struct { ID string `json:"id"` Translation json.RawMessage `json:"translation"` } var I18nCmd = &cobra.Command{ Use: "i18n", Short: "Management of Mattermost translations", } var ExtractCmd = &cobra.Command{ Use: "extract", Short: "Extract translations", Long: "Extract translations from the source code and put them into the i18n/en.json file", Example: " i18n extract", RunE: extractCmdF, } var CheckCmd = &cobra.Command{ Use: "check", Short: "Check translations", Long: "Check translations existing in the source code and compare it to the i18n/en.json file", Example: " i18n check", RunE: checkCmdF, } var CheckEmptySrcCmd = &cobra.Command{ Use: "check-empty-src", Short: "Check for empty translation source strings", Long: "Check the en.json file for empty translation source strings", Example: " i18n check-empty-src", RunE: checkEmptySrcCmdF, } var CleanEmptyCmd = &cobra.Command{ Use: "clean-empty", Short: "Clean empty translations", Long: "Clean empty translations in translation files other than i18n/en.json base file", Example: " i18n clean-empty", RunE: cleanEmptyCmdF, } func init() { ExtractCmd.Flags().Bool("skip-dynamic", false, "Whether to skip dynamically added translations") ExtractCmd.Flags().String("portal-dir", "../customer-web-server", "Path to folder with the Mattermost Customer Portal source code") ExtractCmd.Flags().String("enterprise-dir", "../../enterprise", "Path to folder with the Mattermost enterprise source code") ExtractCmd.Flags().String("server-dir", "./", "Path to folder with the Mattermost server source code") ExtractCmd.Flags().String("model-dir", "../model", "Path to folder with the Mattermost model package source code") ExtractCmd.Flags().String("plugin-dir", "../plugin", "Path to folder with the Mattermost plugin package source code") ExtractCmd.Flags().Bool("contributor", false, "Allows contributors safely extract translations from source code without removing enterprise messages keys") CheckCmd.Flags().Bool("skip-dynamic", false, "Whether to skip dynamically added translations") CheckCmd.Flags().String("portal-dir", "../customer-web-server", "Path to folder with the Mattermost Customer Portal source code") CheckCmd.Flags().String("enterprise-dir", "../../enterprise", "Path to folder with the Mattermost enterprise source code") CheckCmd.Flags().String("server-dir", "./", "Path to folder with the Mattermost server source code") CheckCmd.Flags().String("model-dir", "../model", "Path to folder with the Mattermost model package source code") CheckCmd.Flags().String("plugin-dir", "../plugin", "Path to folder with the Mattermost plugin package source code") CheckEmptySrcCmd.Flags().String("portal-dir", "../customer-web-server", "Path to folder with the Mattermost Customer Portal source code") CheckEmptySrcCmd.Flags().String("enterprise-dir", "../../enterprise", "Path to folder with the Mattermost enterprise source code") CheckEmptySrcCmd.Flags().String("server-dir", "./", "Path to folder with the Mattermost server source code") CleanEmptyCmd.Flags().Bool("dry-run", false, "Run without applying changes") CleanEmptyCmd.Flags().Bool("check", false, "Throw exit code on empty translation strings") CleanEmptyCmd.Flags().String("portal-dir", "../customer-web-server", "Path to folder with the Mattermost Customer Portal source code") CleanEmptyCmd.Flags().String("enterprise-dir", "../../enterprise", "Path to folder with the Mattermost enterprise source code") CleanEmptyCmd.Flags().String("server-dir", "./", "Path to folder with the Mattermost server source code") I18nCmd.AddCommand( ExtractCmd, CheckCmd, CheckEmptySrcCmd, CleanEmptyCmd, ) RootCmd.AddCommand(I18nCmd) } func getBaseFileSrcStrings(mattermostDir string) ([]Translation, error) { jsonFile, err := os.ReadFile(path.Join(mattermostDir, "i18n", "en.json")) if err != nil { return nil, err } var translations []Translation err = json.Unmarshal(jsonFile, &translations) return translations, err } func extractSrcStrings(enterpriseDir, mattermostDir, modelDir, pluginDir, portalDir string) map[string]bool { i18nStrings := map[string]bool{} walkFunc := func(p string, info os.FileInfo, err error) error { if strings.HasPrefix(p, path.Join(mattermostDir, "vendor")) { return nil } return extractFromPath(p, info, err, i18nStrings) } if portalDir != "" { _ = filepath.Walk(portalDir, walkFunc) } else { _ = filepath.Walk(mattermostDir, walkFunc) _ = filepath.Walk(enterpriseDir, walkFunc) _ = filepath.Walk(modelDir, walkFunc) _ = filepath.Walk(pluginDir, walkFunc) } return i18nStrings } func extractCmdF(command *cobra.Command, args []string) error { skipDynamic, err := command.Flags().GetBool("skip-dynamic") if err != nil { return errors.New("invalid skip-dynamic parameter") } enterpriseDir, err := command.Flags().GetString("enterprise-dir") if err != nil { return errors.New("invalid enterprise-dir parameter") } mattermostDir, err := command.Flags().GetString("server-dir") if err != nil { return errors.New("invalid server-dir parameter") } contributorMode, err := command.Flags().GetBool("contributor") if err != nil { return errors.New("invalid contributor parameter") } portalDir, err := command.Flags().GetString("portal-dir") if err != nil { return errors.New("invalid portal-dir parameter") } modelDir, err := command.Flags().GetString("model-dir") if err != nil { return errors.New("invalid model-dir parameter") } pluginDir, err := command.Flags().GetString("plugin-dir") if err != nil { return errors.New("invalid plugin-dir parameter") } translationDir := mattermostDir if portalDir != "" { if enterpriseDir != "" || mattermostDir != "" { return errors.New("please specify EITHER portal-dir or enterprise-dir/server-dir") } skipDynamic = true // dynamics are not needed for portal translationDir = portalDir } i18nStrings := extractSrcStrings(enterpriseDir, mattermostDir, modelDir, pluginDir, portalDir) if !skipDynamic { addDynamicallyGeneratedStrings(i18nStrings) } // Delete any untranslated keys delete(i18nStrings, untranslatedKey) var i18nStringsList []string for id := range i18nStrings { i18nStringsList = append(i18nStringsList, id) } sort.Strings(i18nStringsList) sourceStrings, err := getBaseFileSrcStrings(translationDir) if err != nil { return err } var baseFileList []string idx := map[string]bool{} resultMap := map[string]Translation{} for _, t := range sourceStrings { idx[t.Id] = true baseFileList = append(baseFileList, t.Id) resultMap[t.Id] = t } sort.Strings(baseFileList) for _, translationKey := range i18nStringsList { if _, hasKey := idx[translationKey]; !hasKey { resultMap[translationKey] = Translation{Id: translationKey, Translation: ""} } } for _, translationKey := range baseFileList { if _, hasKey := i18nStrings[translationKey]; !hasKey { if contributorMode && strings.HasPrefix(translationKey, enterpriseKeyPrefix) { continue } delete(resultMap, translationKey) } } var result []Translation for _, t := range resultMap { result = append(result, t) } sort.Slice(result, func(i, j int) bool { return result[i].Id < result[j].Id }) f, err := os.Create(path.Join(mattermostDir, "i18n", "en.json")) if err != nil { return err } defer f.Close() encoder := json.NewEncoder(f) encoder.SetIndent("", " ") encoder.SetEscapeHTML(false) err = encoder.Encode(result) if err != nil { return err } return nil } func checkCmdF(command *cobra.Command, args []string) error { skipDynamic, err := command.Flags().GetBool("skip-dynamic") if err != nil { return errors.New("invalid skip-dynamic parameter") } enterpriseDir, err := command.Flags().GetString("enterprise-dir") if err != nil { return errors.New("invalid enterprise-dir parameter") } mattermostDir, err := command.Flags().GetString("server-dir") if err != nil { return errors.New("invalid server-dir parameter") } portalDir, err := command.Flags().GetString("portal-dir") if err != nil { return errors.New("invalid portal-dir parameter") } modelDir, err := command.Flags().GetString("model-dir") if err != nil { return errors.New("invalid model-dir parameter") } pluginDir, err := command.Flags().GetString("plugin-dir") if err != nil { return errors.New("invalid plugin-dir parameter") } translationDir := mattermostDir if portalDir != "" { if enterpriseDir != "" || mattermostDir != "" { return errors.New("please specify EITHER portal-dir or enterprise-dir/server-dir") } translationDir = portalDir skipDynamic = true // dynamics are not needed for portal } extractedSrcStrings := extractSrcStrings(enterpriseDir, mattermostDir, modelDir, pluginDir, portalDir) if !skipDynamic { addDynamicallyGeneratedStrings(extractedSrcStrings) } // Delete any untranslated keys delete(extractedSrcStrings, untranslatedKey) var extractedList []string for id := range extractedSrcStrings { extractedList = append(extractedList, id) } sort.Strings(extractedList) srcStrings, err := getBaseFileSrcStrings(translationDir) if err != nil { return err } var baseFileList []string idx := map[string]bool{} for _, t := range srcStrings { idx[t.Id] = true baseFileList = append(baseFileList, t.Id) } sort.Strings(baseFileList) changed := false for _, translationKey := range extractedList { if _, hasKey := idx[translationKey]; !hasKey { fmt.Println("Added:", translationKey) changed = true } } for _, translationKey := range baseFileList { if _, hasKey := extractedSrcStrings[translationKey]; !hasKey { fmt.Println("Removed:", translationKey) changed = true } } if changed { command.SilenceUsage = true return errors.New("translation source strings file out of date") } return nil } func addDynamicallyGeneratedStrings(i18nStrings map[string]bool) { i18nStrings["model.user.is_valid.pwd_min_length.app_error"] = true i18nStrings["model.user.is_valid.pwd_max_length.app_error"] = true i18nStrings["model.user.is_valid.pwd_lowercase.app_error"] = true i18nStrings["model.user.is_valid.pwd_lowercase_number.app_error"] = true i18nStrings["model.user.is_valid.pwd_lowercase_number_symbol.app_error"] = true i18nStrings["model.user.is_valid.pwd_lowercase_symbol.app_error"] = true i18nStrings["model.user.is_valid.pwd_lowercase_uppercase.app_error"] = true i18nStrings["model.user.is_valid.pwd_lowercase_uppercase_number.app_error"] = true i18nStrings["model.user.is_valid.pwd_lowercase_uppercase_number_symbol.app_error"] = true i18nStrings["model.user.is_valid.pwd_lowercase_uppercase_symbol.app_error"] = true i18nStrings["model.user.is_valid.pwd_number.app_error"] = true i18nStrings["model.user.is_valid.pwd_number_symbol.app_error"] = true i18nStrings["model.user.is_valid.pwd_symbol.app_error"] = true i18nStrings["model.user.is_valid.pwd_uppercase.app_error"] = true i18nStrings["model.user.is_valid.pwd_uppercase_number.app_error"] = true i18nStrings["model.user.is_valid.pwd_uppercase_number_symbol.app_error"] = true i18nStrings["model.user.is_valid.pwd_uppercase_symbol.app_error"] = true i18nStrings["model.user.is_valid.id.app_error"] = true i18nStrings["model.user.is_valid.create_at.app_error"] = true i18nStrings["model.user.is_valid.update_at.app_error"] = true i18nStrings["model.user.is_valid.username.app_error"] = true i18nStrings["model.user.is_valid.email.app_error"] = true i18nStrings["model.user.is_valid.nickname.app_error"] = true i18nStrings["model.user.is_valid.position.app_error"] = true i18nStrings["model.user.is_valid.first_name.app_error"] = true i18nStrings["model.user.is_valid.last_name.app_error"] = true i18nStrings["model.user.is_valid.auth_data.app_error"] = true i18nStrings["model.user.is_valid.auth_data_type.app_error"] = true i18nStrings["model.user.is_valid.auth_data_pwd.app_error"] = true i18nStrings["model.user.is_valid.password_limit.app_error"] = true i18nStrings["model.user.is_valid.locale.app_error"] = true i18nStrings["January"] = true i18nStrings["February"] = true i18nStrings["March"] = true i18nStrings["April"] = true i18nStrings["May"] = true i18nStrings["June"] = true i18nStrings["July"] = true i18nStrings["August"] = true i18nStrings["September"] = true i18nStrings["October"] = true i18nStrings["November"] = true i18nStrings["December"] = true } func extractByFuncName(name string, args []ast.Expr) *string { if name == "T" { if len(args) == 0 { return nil } key, ok := args[0].(*ast.BasicLit) if !ok { return nil } return &key.Value } else if name == "NewAppError" { if len(args) < 2 { return nil } key, ok := args[1].(*ast.BasicLit) if !ok { return nil } return &key.Value } else if name == "newAppError" { if len(args) < 1 { return nil } key, ok := args[0].(*ast.BasicLit) if !ok { return nil } return &key.Value } else if name == "NewUserFacingError" { if len(args) < 1 { return nil } key, ok := args[0].(*ast.BasicLit) if !ok { return nil } return &key.Value } else if name == "translateFunc" { if len(args) < 1 { return nil } key, ok := args[0].(*ast.BasicLit) if !ok { return nil } return &key.Value } else if name == "TranslateAsHTML" || name == "TranslateAsHtml" { if len(args) < 2 { return nil } key, ok := args[1].(*ast.BasicLit) if !ok { return nil } return &key.Value } else if name == "userLocale" { if len(args) < 1 { return nil } key, ok := args[0].(*ast.BasicLit) if !ok { return nil } return &key.Value } else if name == "localT" { if len(args) < 1 { return nil } key, ok := args[0].(*ast.BasicLit) if !ok { return nil } return &key.Value } return nil } func extractForConstants(name string, valueNode ast.Expr) *string { validConstants := map[string]bool{ "MISSING_CHANNEL_ERROR": true, "MISSING_CHANNEL_MEMBER_ERROR": true, "CHANNEL_EXISTS_ERROR": true, "MISSING_STATUS_ERROR": true, "TEAM_MEMBER_EXISTS_ERROR": true, "MISSING_AUTH_ACCOUNT_ERROR": true, "MISSING_ACCOUNT_ERROR": true, "EXPIRED_LICENSE_ERROR": true, "INVALID_LICENSE_ERROR": true, "MissingChannelError": true, "MissingChannelMemberError": true, "ChannelExistsError": true, "MissingStatusError": true, "TeamMemberExistsError": true, "MissingAuthAccountError": true, "MissingAccountError": true, "ExpiredLicenseError": true, "InvalidLicenseError": true, "NoTranslation": true, "PayloadParseError": true, } if _, ok := validConstants[name]; !ok { return nil } value, ok := valueNode.(*ast.BasicLit) if !ok { return nil } return &value.Value } func extractFromPath(path string, info os.FileInfo, err error, i18nStrings map[string]bool) error { if strings.HasSuffix(path, "model/client4.go") { return nil } if strings.HasSuffix(path, "_test.go") { return nil } if !strings.HasSuffix(path, ".go") { return nil } if strings.Contains(path, ".git/") || strings.HasPrefix(path, ".git/") { return nil } src, err := os.ReadFile(path) if err != nil { panic(err) } fset := token.NewFileSet() f, err := parser.ParseFile(fset, "", src, 0) if err != nil { fmt.Printf("error parsing source: %s\n", path) panic(err) } ast.Inspect(f, func(n ast.Node) bool { var id *string = nil switch expr := n.(type) { case *ast.CallExpr: switch fun := expr.Fun.(type) { case *ast.SelectorExpr: id = extractByFuncName(fun.Sel.Name, expr.Args) if id == nil { return true } break case *ast.Ident: id = extractByFuncName(fun.Name, expr.Args) break default: return true } break case *ast.GenDecl: if expr.Tok == token.CONST { for _, spec := range expr.Specs { valueSpec, ok := spec.(*ast.ValueSpec) if !ok { continue } if len(valueSpec.Names) == 0 { continue } if len(valueSpec.Values) == 0 { continue } id = extractForConstants(valueSpec.Names[0].Name, valueSpec.Values[0]) if id == nil { continue } i18nStrings[strings.Trim(*id, "\"")] = true } } return true default: return true } if id != nil { i18nStrings[strings.Trim(*id, "\"")] = true } return true }) return nil } func checkEmptySrcCmdF(command *cobra.Command, args []string) error { enterpriseDir, err := command.Flags().GetString("enterprise-dir") if err != nil { return errors.New("invalid enterprise-dir parameter") } mattermostDir, err := command.Flags().GetString("server-dir") if err != nil { return errors.New("invalid server-dir parameter") } portalDir, err := command.Flags().GetString("portal-dir") if err != nil { return errors.New("invalid portal-dir parameter") } translationDir := path.Join(mattermostDir, "i18n") if portalDir != "" { if enterpriseDir != "" || mattermostDir != "" { return errors.New("please specify EITHER portal-dir or enterprise-dir/server-dir") } translationDir = portalDir } srcJSON, err := os.ReadFile(path.Join(translationDir, "en.json")) if err != nil { return err } var items []Item if err = json.Unmarshal(srcJSON, &items); err != nil { return err } err = countEmptyItems(items) if err != nil { return err } return nil } func countEmptyItems(items []Item) error { hasError := false for _, t := range items { str := string(t.Translation) if !strings.HasPrefix(str, "\"") { continue } unquoted, err := strconv.Unquote(str) if err != nil { return fmt.Errorf("error unquoting translation for %s, %v", t.ID, err) } if strings.TrimSpace(unquoted) == "" { log.Printf("Empty translation for %s. Please fix it.\n", t.ID) hasError = true } } if hasError { return errors.New("empty translations found") } return nil } func cleanEmptyCmdF(command *cobra.Command, args []string) error { dryRun, err := command.Flags().GetBool("dry-run") if err != nil { return errors.New("invalid dry-run parameter") } check, err := command.Flags().GetBool("check") if err != nil { return errors.New("invalid check parameter") } enterpriseDir, err := command.Flags().GetString("enterprise-dir") if err != nil { return errors.New("invalid enterprise-dir parameter") } mattermostDir, err := command.Flags().GetString("server-dir") if err != nil { return errors.New("invalid server-dir parameter") } portalDir, err := command.Flags().GetString("portal-dir") if err != nil { return errors.New("invalid portal-dir parameter") } translationDir := path.Join(mattermostDir, "i18n") if portalDir != "" { if enterpriseDir != "" || mattermostDir != "" { return errors.New("please specify EITHER portal-dir or enterprise-dir/server-dir") } translationDir = portalDir } var shippedFiles []string dirEntries, err := os.ReadDir(translationDir) if err != nil { return err } for _, dirEntry := range dirEntries { if !dirEntry.IsDir() && filepath.Ext(dirEntry.Name()) == ".json" && dirEntry.Name() != "en.json" { shippedFiles = append(shippedFiles, dirEntry.Name()) } } results := "" for _, file := range shippedFiles { result, err2 := clean(translationDir, file, dryRun, check) if err2 != nil { return err2 } results += *result } if results == "" { return nil } fmt.Print("\n" + results) if check { os.Exit(1) } return nil } func clean(translationDir string, file string, dryRun bool, check bool) (*string, error) { oldJSON, err := os.ReadFile(path.Join(translationDir, file)) if err != nil { return nil, err } if strings.TrimSpace(string(oldJSON)) == "{}" { noIssue := "" return &noIssue, err } var oldList []Item if err = json.Unmarshal(oldJSON, &oldList); err != nil { return nil, err } newList, count := removeEmptyTranslations(oldList) result := "" if count == 0 { return &result, nil } result = fmt.Sprintf("%v has %v empty translations\n", file, count) if dryRun || check { return &result, nil } newJSON, err := JSONMarshal(newList) if err != nil { return nil, err } filename := path.Join(translationDir, file) fileInfo, err := os.Lstat(filename) if err != nil { return nil, err } if err = os.WriteFile(filename, newJSON, fileInfo.Mode().Perm()); err != nil { return nil, err } return &result, nil } func removeEmptyTranslations(oldList []Item) ([]Item, int) { var count int var newList []Item for i, t := range oldList { if string(t.Translation) != "\"\"" { newList = append(newList, oldList[i]) } else { count++ } } return newList, count } func JSONMarshal(t interface{}) ([]byte, error) { buffer := &bytes.Buffer{} encoder := json.NewEncoder(buffer) encoder.SetEscapeHTML(false) encoder.SetIndent("", " ") err := encoder.Encode(t) return buffer.Bytes(), err }