* Implementing structured logging

* Changes to en.json to allow refactor to run.

* Fixing global logger

* Structured logger initalization.

* Add caller.

* Do some log redirection.

* Auto refactor

* Cleaning up l4g reference and removing dependancy.

* Removing junk.

* Copyright headers.

* Fixing tests

* Revert "Changes to en.json to allow refactor to run."

This reverts commit fd8249e99bcad0231e6ea65cd77c32aae9a54026.

* Fixing some auto refactor strangeness and typo.

* Making keys more human readable.
Этот коммит содержится в:
Christopher Speller
2018-04-27 12:49:45 -07:00
коммит произвёл GitHub
родитель 2acbc77d78
Коммит 686c2fbab7
205 изменённых файлов: 9480 добавлений и 2488 удалений

Просмотреть файл

@@ -15,7 +15,6 @@ import (
"strconv"
"strings"
l4g "github.com/alecthomas/log4go"
"github.com/fsnotify/fsnotify"
"github.com/pkg/errors"
"github.com/spf13/viper"
@@ -23,6 +22,7 @@ import (
"net/http"
"github.com/mattermost/mattermost-server/einterfaces"
"github.com/mattermost/mattermost-server/mlog"
"github.com/mattermost/mattermost-server/model"
"github.com/mattermost/mattermost-server/utils/jsonutils"
)
@@ -32,8 +32,6 @@ const (
LOG_FILENAME = "mattermost.log"
)
var originalDisableDebugLvl l4g.Level = l4g.DEBUG
// FindConfigFile attempts to find an existing configuration file. fileName can be an absolute or
// relative path or name such as "/opt/mattermost/config.json" or simply "config.json". An empty
// string is returned if no configuration is found.
@@ -66,71 +64,26 @@ func FindDir(dir string) (string, bool) {
return "./", false
}
func MloggerConfigFromLoggerConfig(s *model.LogSettings) *mlog.LoggerConfiguration {
return &mlog.LoggerConfiguration{
EnableConsole: s.EnableConsole,
ConsoleJson: *s.ConsoleJson,
ConsoleLevel: strings.ToLower(s.ConsoleLevel),
EnableFile: s.EnableFile,
FileJson: *s.FileJson,
FileLevel: strings.ToLower(s.FileLevel),
FileLocation: GetLogFileLocation(s.FileLocation),
}
}
// DON'T USE THIS Modify the level on the app logger
func DisableDebugLogForTest() {
if l4g.Global["stdout"] != nil {
originalDisableDebugLvl = l4g.Global["stdout"].Level
l4g.Global["stdout"].Level = l4g.ERROR
}
mlog.GloballyDisableDebugLogForTest()
}
// DON'T USE THIS Modify the level on the app logger
func EnableDebugLogForTest() {
if l4g.Global["stdout"] != nil {
l4g.Global["stdout"].Level = originalDisableDebugLvl
}
}
func ConfigureCmdLineLog() {
ls := model.LogSettings{}
ls.EnableConsole = true
ls.ConsoleLevel = "WARN"
ConfigureLog(&ls)
}
// ConfigureLog enables and configures logging.
//
// Note that it is not currently possible to disable filters nor to modify previously enabled
// filters, given the lack of concurrency guarantees from the underlying l4g library.
//
// TODO: this code initializes console and file logging. It will eventually be replaced by JSON logging in logger/logger.go
// See PLT-3893 for more information
func ConfigureLog(s *model.LogSettings) {
if _, alreadySet := l4g.Global["stdout"]; !alreadySet && s.EnableConsole {
level := l4g.DEBUG
if s.ConsoleLevel == "INFO" {
level = l4g.INFO
} else if s.ConsoleLevel == "WARN" {
level = l4g.WARNING
} else if s.ConsoleLevel == "ERROR" {
level = l4g.ERROR
}
lw := l4g.NewConsoleLogWriter()
lw.SetFormat("[%D %T] [%L] %M")
l4g.AddFilter("stdout", level, lw)
}
if _, alreadySet := l4g.Global["file"]; !alreadySet && s.EnableFile {
var fileFormat = s.FileFormat
if fileFormat == "" {
fileFormat = "[%D %T] [%L] %M"
}
level := l4g.DEBUG
if s.FileLevel == "INFO" {
level = l4g.INFO
} else if s.FileLevel == "WARN" {
level = l4g.WARNING
} else if s.FileLevel == "ERROR" {
level = l4g.ERROR
}
flw := l4g.NewFileLogWriter(GetLogFileLocation(s.FileLocation), false)
flw.SetFormat(fileFormat)
flw.SetRotate(true)
flw.SetRotateLines(LOG_ROTATE_SIZE)
l4g.AddFilter("file", level, flw)
}
mlog.GloballyEnableDebugLogForTest()
}
func GetLogFileLocation(fileLocation string) string {
@@ -189,17 +142,17 @@ func NewConfigWatcher(cfgFileName string, f func()) (*ConfigWatcher, error) {
// we only care about the config file
if filepath.Clean(event.Name) == configFile {
if event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Create == fsnotify.Create {
l4g.Info(fmt.Sprintf("Config file watcher detected a change reloading %v", cfgFileName))
mlog.Info(fmt.Sprintf("Config file watcher detected a change reloading %v", cfgFileName))
if _, _, configReadErr := ReadConfigFile(cfgFileName, true); configReadErr == nil {
f()
} else {
l4g.Error(fmt.Sprintf("Failed to read while watching config file at %v with err=%v", cfgFileName, configReadErr.Error()))
mlog.Error(fmt.Sprintf("Failed to read while watching config file at %v with err=%v", cfgFileName, configReadErr.Error()))
}
}
}
case err := <-watcher.Errors:
l4g.Error(fmt.Sprintf("Failed while watching config file at %v with err=%v", cfgFileName, err.Error()))
mlog.Error(fmt.Sprintf("Failed while watching config file at %v with err=%v", cfgFileName, err.Error()))
case <-ret.close:
return
}
@@ -278,7 +231,7 @@ func newViper(allowEnvironmentOverrides bool) *viper.Viper {
func structToMap(t reflect.Type) (out map[string]interface{}) {
defer func() {
if r := recover(); r != nil {
l4g.Error("Panicked in structToMap. This should never happen. %v", r)
mlog.Error(fmt.Sprintf("Panicked in structToMap. This should never happen. %v", r))
}
}()
@@ -345,7 +298,7 @@ func flattenStructToMap(in map[string]interface{}) map[string]interface{} {
func fixEnvSettingsCase(in map[string]interface{}) (out map[string]interface{}, err error) {
defer func() {
if r := recover(); r != nil {
l4g.Error("Panicked in fixEnvSettingsCase. This should never happen. %v", r)
mlog.Error(fmt.Sprintf("Panicked in fixEnvSettingsCase. This should never happen. %v", r))
out = in
}
}()
@@ -450,13 +403,13 @@ func LoadConfig(fileName string) (*model.Config, string, map[string]interface{},
if needSave {
if err := SaveConfig(configPath, config); err != nil {
l4g.Warn(err.Error())
mlog.Warn(err.Error())
}
}
if err := ValidateLocales(config); err != nil {
if err := SaveConfig(configPath, config); err != nil {
l4g.Warn(err.Error())
mlog.Warn(err.Error())
}
}

Просмотреть файл

@@ -19,9 +19,8 @@ import (
func TestConfig(t *testing.T) {
TranslationsPreInit()
cfg, _, _, err := LoadConfig("config.json")
_, _, _, err := LoadConfig("config.json")
require.Nil(t, err)
InitTranslations(cfg.LocalizationSettings)
}
func TestReadConfig(t *testing.T) {

Просмотреть файл

@@ -9,8 +9,7 @@ import (
"os"
"path/filepath"
l4g "github.com/alecthomas/log4go"
"github.com/mattermost/mattermost-server/mlog"
"github.com/mattermost/mattermost-server/model"
)
@@ -28,7 +27,7 @@ func (b *LocalFileBackend) TestConnection() *model.AppError {
return model.NewAppError("TestFileConnection", "Don't have permissions to write to local path specified or other error.", nil, err.Error(), http.StatusInternalServerError)
}
os.Remove(filepath.Join(b.directory, TEST_FILE_PATH))
l4g.Info("Able to write files to local storage.")
mlog.Info("Able to write files to local storage.")
return nil
}

Просмотреть файл

@@ -11,10 +11,10 @@ import (
"path/filepath"
"strings"
l4g "github.com/alecthomas/log4go"
s3 "github.com/minio/minio-go"
"github.com/minio/minio-go/pkg/credentials"
"github.com/mattermost/mattermost-server/mlog"
"github.com/mattermost/mattermost-server/model"
)
@@ -70,14 +70,14 @@ func (b *S3FileBackend) TestConnection() *model.AppError {
}
if !exists {
l4g.Warn("Bucket specified does not exist. Attempting to create...")
mlog.Warn("Bucket specified does not exist. Attempting to create...")
err := s3Clnt.MakeBucket(b.bucket, b.region)
if err != nil {
l4g.Error("Unable to create bucket.")
mlog.Error("Unable to create bucket.")
return model.NewAppError("TestFileConnection", "Unable to create bucket", nil, err.Error(), http.StatusInternalServerError)
}
}
l4g.Info("Connection to S3 or minio is good. Bucket exists.")
mlog.Info("Connection to S3 or minio is good. Bucket exists.")
return nil
}

Просмотреть файл

@@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/mattermost/mattermost-server/mlog"
"github.com/mattermost/mattermost-server/model"
)
@@ -23,6 +24,15 @@ type FileBackendTestSuite struct {
}
func TestLocalFileBackendTestSuite(t *testing.T) {
// Setup a global logger to catch tests logging outside of app context
// The global logger will be stomped by apps initalizing but that's fine for testing. Ideally this won't happen.
mlog.InitGlobalLogger(mlog.NewLogger(&mlog.LoggerConfiguration{
EnableConsole: true,
ConsoleJson: true,
ConsoleLevel: "error",
EnableFile: false,
}))
dir, err := ioutil.TempDir("", "")
require.NoError(t, err)
defer os.RemoveAll(dir)

Просмотреть файл

@@ -6,14 +6,15 @@ package utils
import (
"bytes"
"errors"
"fmt"
"html/template"
"io"
"path/filepath"
"reflect"
"sync/atomic"
l4g "github.com/alecthomas/log4go"
"github.com/fsnotify/fsnotify"
"github.com/mattermost/mattermost-server/mlog"
"github.com/nicksnyder/go-i18n/i18n"
)
@@ -25,7 +26,7 @@ type HTMLTemplateWatcher struct {
func NewHTMLTemplateWatcher(directory string) (*HTMLTemplateWatcher, error) {
templatesDir, _ := FindDir(directory)
l4g.Debug("Parsing server templates at %v", templatesDir)
mlog.Debug(fmt.Sprintf("Parsing server templates at %v", templatesDir))
ret := &HTMLTemplateWatcher{
stop: make(chan struct{}),
@@ -57,15 +58,15 @@ func NewHTMLTemplateWatcher(directory string) (*HTMLTemplateWatcher, error) {
return
case event := <-watcher.Events:
if event.Op&fsnotify.Write == fsnotify.Write {
l4g.Info("Re-parsing templates because of modified file %v", event.Name)
mlog.Info(fmt.Sprintf("Re-parsing templates because of modified file %v", event.Name))
if htmlTemplates, err := template.ParseGlob(filepath.Join(templatesDir, "*.html")); err != nil {
l4g.Error("Failed to parse templates %v", err)
mlog.Error(fmt.Sprintf("Failed to parse templates %v", err))
} else {
ret.templates.Store(htmlTemplates)
}
}
case err := <-watcher.Errors:
l4g.Error("Failed in directory watcher %s", err)
mlog.Error(fmt.Sprintf("Failed in directory watcher %s", err))
}
}
}()
@@ -110,7 +111,7 @@ func (t *HTMLTemplate) RenderToWriter(w io.Writer) error {
}
if err := t.Templates.ExecuteTemplate(w, t.TemplateName, t); err != nil {
l4g.Error(T("api.api.render.error"), t.TemplateName, err)
mlog.Error(fmt.Sprintf("Error rendering template %v err=%v", t.TemplateName, err))
return err
}
@@ -134,7 +135,7 @@ func escapeForHtml(arg interface{}) interface{} {
}
return safeArg
default:
l4g.Warn("Unable to escape value for HTML template %v of type %v", arg, reflect.ValueOf(arg).Type())
mlog.Warn(fmt.Sprintf("Unable to escape value for HTML template %v of type %v", arg, reflect.ValueOf(arg).Type()))
return ""
}
}

Просмотреть файл

@@ -10,7 +10,7 @@ import (
"path/filepath"
"strings"
l4g "github.com/alecthomas/log4go"
"github.com/mattermost/mattermost-server/mlog"
"github.com/mattermost/mattermost-server/model"
"github.com/nicksnyder/go-i18n/i18n"
)
@@ -67,7 +67,7 @@ func InitTranslationsWithDir(dir string) error {
func GetTranslationsBySystemLocale() (i18n.TranslateFunc, error) {
locale := *settings.DefaultServerLocale
if _, ok := locales[locale]; !ok {
l4g.Error("Failed to load system translations for '%v' attempting to fall back to '%v'", locale, model.DEFAULT_LOCALE)
mlog.Error(fmt.Sprintf("Failed to load system translations for '%v' attempting to fall back to '%v'", locale, model.DEFAULT_LOCALE))
locale = model.DEFAULT_LOCALE
}
@@ -80,7 +80,7 @@ func GetTranslationsBySystemLocale() (i18n.TranslateFunc, error) {
return nil, fmt.Errorf("Failed to load system translations")
}
l4g.Info(translations("utils.i18n.loaded"), locale, locales[locale])
mlog.Info(fmt.Sprintf("Loaded system translations for '%v' from '%v'", locale, locales[locale]))
return translations, nil
}

Просмотреть файл

@@ -10,14 +10,14 @@ import (
"crypto/x509"
"encoding/base64"
"encoding/pem"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
l4g "github.com/alecthomas/log4go"
"github.com/mattermost/mattermost-server/mlog"
"github.com/mattermost/mattermost-server/model"
)
@@ -36,12 +36,12 @@ func ValidateLicense(signed []byte) (bool, string) {
_, err := base64.StdEncoding.Decode(decoded, signed)
if err != nil {
l4g.Error(T("utils.license.validate_license.decode.error"), err.Error())
mlog.Error(fmt.Sprintf("Encountered error decoding license, err=%v", err.Error()))
return false, ""
}
if len(decoded) <= 256 {
l4g.Error(T("utils.license.validate_license.not_long.error"))
mlog.Error("Signed license not long enough")
return false, ""
}
@@ -57,7 +57,7 @@ func ValidateLicense(signed []byte) (bool, string) {
public, err := x509.ParsePKIXPublicKey(block.Bytes)
if err != nil {
l4g.Error(T("utils.license.validate_license.signing.error"), err.Error())
mlog.Error(fmt.Sprintf("Encountered error signing license, err=%v", err.Error()))
return false, ""
}
@@ -69,7 +69,7 @@ func ValidateLicense(signed []byte) (bool, string) {
err = rsa.VerifyPKCS1v15(rsaPublic, crypto.SHA512, d, signature)
if err != nil {
l4g.Error(T("utils.license.validate_license.invalid.error"), err.Error())
mlog.Error(fmt.Sprintf("Invalid signature, err=%v", err.Error()))
return false, ""
}
@@ -80,15 +80,15 @@ func GetAndValidateLicenseFileFromDisk(location string) (*model.License, []byte)
fileName := GetLicenseFileLocation(location)
if _, err := os.Stat(fileName); err != nil {
l4g.Debug("We could not find the license key in the database or on disk at %v", fileName)
mlog.Debug(fmt.Sprintf("We could not find the license key in the database or on disk at %v", fileName))
return nil, nil
}
l4g.Info("License key has not been uploaded. Loading license key from disk at %v", fileName)
mlog.Info(fmt.Sprintf("License key has not been uploaded. Loading license key from disk at %v", fileName))
licenseBytes := GetLicenseFileFromDisk(fileName)
if success, licenseStr := ValidateLicense(licenseBytes); !success {
l4g.Error("Found license key at %v but it appears to be invalid.", fileName)
mlog.Error(fmt.Sprintf("Found license key at %v but it appears to be invalid.", fileName))
return nil, nil
} else {
return model.LicenseFromJson(strings.NewReader(licenseStr)), licenseBytes
@@ -98,14 +98,14 @@ func GetAndValidateLicenseFileFromDisk(location string) (*model.License, []byte)
func GetLicenseFileFromDisk(fileName string) []byte {
file, err := os.Open(fileName)
if err != nil {
l4g.Error("Failed to open license key from disk at %v err=%v", fileName, err.Error())
mlog.Error(fmt.Sprintf("Failed to open license key from disk at %v err=%v", fileName, err.Error()))
return nil
}
defer file.Close()
licenseBytes, err := ioutil.ReadAll(file)
if err != nil {
l4g.Error("Failed to read license key from disk at %v err=%v", fileName, err.Error())
mlog.Error(fmt.Sprintf("Failed to read license key from disk at %v err=%v", fileName, err.Error()))
return nil
}

Просмотреть файл

@@ -6,6 +6,7 @@ package utils
import (
"crypto/tls"
"errors"
"fmt"
"io"
"mime"
"net"
@@ -17,8 +18,8 @@ import (
"net/http"
l4g "github.com/alecthomas/log4go"
"github.com/mattermost/html2text"
"github.com/mattermost/mattermost-server/mlog"
"github.com/mattermost/mattermost-server/model"
)
@@ -128,14 +129,14 @@ func ConnectToSMTPServer(config *model.Config) (net.Conn, *model.AppError) {
func NewSMTPClientAdvanced(conn net.Conn, hostname string, connectionInfo *SmtpConnectionInfo) (*smtp.Client, *model.AppError) {
c, err := smtp.NewClient(conn, connectionInfo.SmtpServerName+":"+connectionInfo.SmtpPort)
if err != nil {
l4g.Error(T("utils.mail.new_client.open.error"), err)
mlog.Error(fmt.Sprintf("Failed to open a connection to SMTP server %v", err))
return nil, model.NewAppError("SendMail", "utils.mail.connect_smtp.open_tls.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if hostname != "" {
err := c.Hello(hostname)
if err != nil {
l4g.Error(T("utils.mail.new_client.helo.error"), err)
mlog.Error(fmt.Sprintf("Failed to to set the HELO to SMTP server %v", err))
return nil, model.NewAppError("SendMail", "utils.mail.connect_smtp.helo.app_error", nil, err.Error(), http.StatusInternalServerError)
}
}
@@ -180,14 +181,14 @@ func TestConnection(config *model.Config) {
conn, err1 := ConnectToSMTPServer(config)
if err1 != nil {
l4g.Error(T("utils.mail.test.configured.error"), T(err1.Message), err1.DetailedError)
mlog.Error(fmt.Sprintf("SMTP server settings do not appear to be configured properly err=%v details=%v", T(err1.Message), err1.DetailedError))
return
}
defer conn.Close()
c, err2 := NewSMTPClient(conn, config)
if err2 != nil {
l4g.Error(T("utils.mail.test.configured.error"), T(err2.Message), err2.DetailedError)
mlog.Error(fmt.Sprintf("SMTP server settings do not appear to be configured properly err=%v details=%v", T(err2.Message), err2.DetailedError))
return
}
defer c.Quit()
@@ -228,13 +229,13 @@ func SendMailUsingConfigAdvanced(mimeTo, smtpTo string, from mail.Address, subje
}
func SendMail(c *smtp.Client, mimeTo, smtpTo string, from mail.Address, subject, htmlBody string, attachments []*model.FileInfo, mimeHeaders map[string]string, fileBackend FileBackend, date time.Time) *model.AppError {
l4g.Debug(T("utils.mail.send_mail.sending.debug"), mimeTo, subject)
mlog.Debug(fmt.Sprintf("sending mail to %v with subject of '%v'", mimeTo, subject))
htmlMessage := "\r\n<html><body>" + htmlBody + "</body></html>"
txtBody, err := html2text.FromString(htmlBody)
if err != nil {
l4g.Warn(err)
mlog.Warn(fmt.Sprint(err))
txtBody = ""
}

Просмотреть файл

@@ -1,65 +0,0 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package utils
import (
"bufio"
"log"
"os"
"strings"
l4g "github.com/alecthomas/log4go"
)
type RedirectStdLog struct {
reader *os.File
writer *os.File
system string
ignoreDebug bool
}
func NewRedirectStdLog(system string, ignoreDebug bool) *log.Logger {
r, w, _ := os.Pipe()
logger := &RedirectStdLog{
reader: r,
writer: w,
system: system,
ignoreDebug: ignoreDebug,
}
go func(l *RedirectStdLog) {
scanner := bufio.NewScanner(l.reader)
for scanner.Scan() {
line := scanner.Text()
if strings.Index(line, "[DEBUG]") == 0 {
if !ignoreDebug {
l4g.Debug("%v%v", system, line[7:])
}
} else if strings.Index(line, "[DEBG]") == 0 {
if !ignoreDebug {
l4g.Debug("%v%v", system, line[6:])
}
} else if strings.Index(line, "[WARN]") == 0 {
l4g.Info("%v%v", system, line[6:])
} else if strings.Index(line, "[ERROR]") == 0 {
l4g.Error("%v%v", system, line[7:])
} else if strings.Index(line, "[EROR]") == 0 {
l4g.Error("%v%v", system, line[6:])
} else if strings.Index(line, "[ERR]") == 0 {
l4g.Error("%v%v", system, line[5:])
} else if strings.Index(line, "[INFO]") == 0 {
l4g.Info("%v%v", system, line[6:])
} else {
l4g.Info("%v %v", system, line)
}
}
}(logger)
return log.New(logger.writer, "", 0)
}
func (l *RedirectStdLog) Write(p []byte) (n int, err error) {
return l.writer.Write(p)
}

Просмотреть файл

@@ -1,24 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package utils
import (
"testing"
"time"
)
func TestRedirectStdLog(t *testing.T) {
log := NewRedirectStdLog("test", false)
log.Println("[DEBUG] this is a message")
log.Println("[DEBG] this is a message")
log.Println("[WARN] this is a message")
log.Println("[ERROR] this is a message")
log.Println("[EROR] this is a message")
log.Println("[ERR] this is a message")
log.Println("[INFO] this is a message")
log.Println("this is a message")
time.Sleep(time.Second * 1)
}