XYZ-35: Added Support for GlobalRelay Compliance Export Format

* Added username to ChannelMemberHistory struct in anticipation of supporting GlobalRelay in Compliance Export * Removed translation from debug output - this makes it complicated to use utils functions from tests in the enterprise repo * Added an advanced email function that allows for greater control over message details. Updated MessageExport config to support GlobalRelay. Added attachment support to InBucket unit tests * Moving templates in from enterprise to solve test issues * Added export format to diagnostics * Changed email attachment code to use FileBackend so that S3 storage is properly supported
2018-02-07 09:02:46 -05:00
--- a/utils/html.go
+++ b/utils/html.go
@@ -23,7 +23,7 @@ type HTMLTemplateWatcher struct {

 func NewHTMLTemplateWatcher(directory string) (*HTMLTemplateWatcher, error) {
 	templatesDir, _ := FindDir(directory)
-	l4g.Debug(T("api.api.init.parsing_templates.debug"), templatesDir)
+	l4g.Debug("Parsing server templates at %v", templatesDir)

 	ret := &HTMLTemplateWatcher{
 		stop:    make(chan struct{}),
@@ -55,15 +55,15 @@ func NewHTMLTemplateWatcher(directory string) (*HTMLTemplateWatcher, error) {
 				return
 			case event := <-watcher.Events:
 				if event.Op&fsnotify.Write == fsnotify.Write {
-					l4g.Info(T("web.reparse_templates.info"), event.Name)
+					l4g.Info("Re-parsing templates because of modified file %v", event.Name)
 					if htmlTemplates, err := template.ParseGlob(templatesDir + "*.html"); err != nil {
-						l4g.Error(T("web.parsing_templates.error"), err)
+						l4g.Error("Failed to parse templates %v", err)
 					} else {
 						ret.templates.Store(htmlTemplates)
 					}
 				}
 			case err := <-watcher.Errors:
-				l4g.Error(T("web.dir_fail.error"), err)
+				l4g.Error("Failed in directory watcher %s", err)
 			}
 		}
 	}()
--- a/utils/inbucket.go
+++ b/utils/inbucket.go
@@ -4,6 +4,7 @@
 package utils

 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -37,6 +38,12 @@ type JSONMessageInbucket struct {
 		Text string
 		HTML string `json:"Html"`
 	}
+	Attachments []struct {
+		Filename     string
+		ContentType  string `json:"content-type"`
+		DownloadLink string `json:"download-link"`
+		Bytes        []byte `json:"-"`
+	}
 }

 func ParseEmail(email string) string {
@@ -89,21 +96,54 @@ func GetMessageFromMailbox(email, id string) (results JSONMessageInbucket, err e
 	var record JSONMessageInbucket

 	url := fmt.Sprintf("%s%s%s/%s", getInbucketHost(), INBUCKET_API, parsedEmail, id)
-	req, err := http.NewRequest("GET", url, nil)
+	emailResponse, err := get(url)
 	if err != nil {
 		return record, err
 	}
+	defer emailResponse.Body.Close()
+
+	err = json.NewDecoder(emailResponse.Body).Decode(&record)
+
+	// download attachments
+	if record.Attachments != nil && len(record.Attachments) > 0 {
+		for i := range record.Attachments {
+			if bytes, err := downloadAttachment(record.Attachments[i].DownloadLink); err != nil {
+				return record, err
+			} else {
+				record.Attachments[i].Bytes = make([]byte, len(bytes))
+				copy(record.Attachments[i].Bytes, bytes)
+			}
+		}
+	}
+
+	return record, err
+}
+
+func downloadAttachment(url string) ([]byte, error) {
+	attachmentResponse, err := get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer attachmentResponse.Body.Close()
+
+	buf := new(bytes.Buffer)
+	io.Copy(buf, attachmentResponse.Body)
+	return buf.Bytes(), nil
+}
+
+func get(url string) (*http.Response, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}

 	client := &http.Client{}
-
 	resp, err := client.Do(req)
 	if err != nil {
-		return record, err
+		return nil, err
 	}
-	defer resp.Body.Close()

-	err = json.NewDecoder(resp.Body).Decode(&record)
-	return record, err
+	return resp, nil
 }

 func DeleteMailBox(email string) (err error) {
--- a/utils/mail.go
+++ b/utils/mail.go
@@ -15,6 +15,8 @@ import (

 	"net/http"

+	"io"
+
 	l4g "github.com/alecthomas/log4go"
 	"github.com/mattermost/html2text"
 	"github.com/mattermost/mattermost-server/model"
@@ -104,36 +106,72 @@ func TestConnection(config *model.Config) {
 }

 func SendMailUsingConfig(to, subject, htmlBody string, config *model.Config) *model.AppError {
+	fromMail := mail.Address{Name: config.EmailSettings.FeedbackName, Address: config.EmailSettings.FeedbackEmail}
+	return sendMail(to, to, fromMail, subject, htmlBody, nil, nil, config)
+}
+
+// allows for sending an email with attachments and differing MIME/SMTP recipients
+func SendMailUsingConfigAdvanced(mimeTo, smtpTo string, from mail.Address, subject, htmlBody string, attachments []*model.FileInfo, mimeHeaders map[string]string, config *model.Config) *model.AppError {
+	return sendMail(mimeTo, smtpTo, from, subject, htmlBody, attachments, mimeHeaders, config)
+}
+
+func sendMail(mimeTo, smtpTo string, from mail.Address, subject, htmlBody string, attachments []*model.FileInfo, mimeHeaders map[string]string, config *model.Config) *model.AppError {
 	if !config.EmailSettings.SendEmailNotifications || len(config.EmailSettings.SMTPServer) == 0 {
 		return nil
 	}

-	l4g.Debug(T("utils.mail.send_mail.sending.debug"), to, subject)
+	l4g.Debug(T("utils.mail.send_mail.sending.debug"), mimeTo, subject)

 	htmlMessage := "\r\n<html><body>" + htmlBody + "</body></html>"

-	fromMail := mail.Address{Name: config.EmailSettings.FeedbackName, Address: config.EmailSettings.FeedbackEmail}
-
 	txtBody, err := html2text.FromString(htmlBody)
 	if err != nil {
 		l4g.Warn(err)
 		txtBody = ""
 	}

-	m := gomail.NewMessage(gomail.SetCharset("UTF-8"))
-	m.SetHeaders(map[string][]string{
-		"From":                      {fromMail.String()},
-		"To":                        {to},
+	headers := map[string][]string{
+		"From":                      {from.String()},
+		"To":                        {mimeTo},
 		"Subject":                   {encodeRFC2047Word(subject)},
 		"Content-Transfer-Encoding": {"8bit"},
 		"Auto-Submitted":            {"auto-generated"},
 		"Precedence":                {"bulk"},
-	})
-	m.SetDateHeader("Date", time.Now())
+	}
+	if mimeHeaders != nil {
+		for k, v := range mimeHeaders {
+			headers[k] = []string{encodeRFC2047Word(v)}
+		}
+	}

+	m := gomail.NewMessage(gomail.SetCharset("UTF-8"))
+	m.SetHeaders(headers)
+	m.SetDateHeader("Date", time.Now())
 	m.SetBody("text/plain", txtBody)
 	m.AddAlternative("text/html", htmlMessage)

+	if attachments != nil {
+		fileBackend, err := NewFileBackend(&config.FileSettings)
+		if err != nil {
+			return err
+		}
+
+		for _, fileInfo := range attachments {
+			m.Attach(fileInfo.Name, gomail.SetCopyFunc(func(writer io.Writer) error {
+				bytes, err := fileBackend.ReadFile(fileInfo.Path)
+				if err != nil {
+					return err
+				}
+				if _, err := writer.Write(bytes); err != nil {
+					return model.NewAppError("SendMail", "utils.mail.sendMail.attachments.write_error", nil, err.Error(), http.StatusInternalServerError)
+				}
+				return nil
+			}))
+
+		}
+
+	}
+
 	conn, err1 := connectToSMTPServer(config)
 	if err1 != nil {
 		return err1
@@ -147,11 +185,11 @@ func SendMailUsingConfig(to, subject, htmlBody string, config *model.Config) *mo
 	defer c.Quit()
 	defer c.Close()

-	if err := c.Mail(fromMail.Address); err != nil {
+	if err := c.Mail(from.Address); err != nil {
 		return model.NewAppError("SendMail", "utils.mail.send_mail.from_address.app_error", nil, err.Error(), http.StatusInternalServerError)
 	}

-	if err := c.Rcpt(to); err != nil {
+	if err := c.Rcpt(smtpTo); err != nil {
 		return model.NewAppError("SendMail", "utils.mail.send_mail.to_address.app_error", nil, err.Error(), http.StatusInternalServerError)
 	}

--- a/utils/mail_test.go
+++ b/utils/mail_test.go
@@ -7,6 +7,10 @@ import (
 	"strings"
 	"testing"

+	"net/mail"
+
+	"github.com/mattermost/mattermost-server/model"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

@@ -39,9 +43,9 @@ func TestSendMailUsingConfig(t *testing.T) {
 	require.Nil(t, err)
 	T = GetUserTranslations("en")

-	var emailTo string = "test@example.com"
-	var emailSubject string = "Testing this email"
-	var emailBody string = "This is a test from autobot"
+	var emailTo = "test@example.com"
+	var emailSubject = "Testing this email"
+	var emailBody = "This is a test from autobot"

 	//Delete all the messages before check the sample email
 	DeleteMailBox(emailTo)
@@ -50,7 +54,7 @@ func TestSendMailUsingConfig(t *testing.T) {
 		t.Log(err)
 		t.Fatal("Should connect to the STMP Server")
 	} else {
-		//Check if the email was send to the rigth email address
+		//Check if the email was send to the right email address
 		var resultsMailbox JSONMessageHeaderInbucket
 		err := RetryInbucket(5, func() error {
 			var err error
@@ -75,3 +79,78 @@ func TestSendMailUsingConfig(t *testing.T) {
 		}
 	}
 }
+
+func TestSendMailUsingConfigAdvanced(t *testing.T) {
+	cfg, _, err := LoadConfig("config.json")
+	require.Nil(t, err)
+	T = GetUserTranslations("en")
+
+	var mimeTo = "test@example.com"
+	var smtpTo = "test2@example.com"
+	var from = mail.Address{Name: "Nobody", Address: "nobody@mattermost.com"}
+	var emailSubject = "Testing this email"
+	var emailBody = "This is a test from autobot"
+
+	//Delete all the messages before check the sample email
+	DeleteMailBox(smtpTo)
+
+	// create a file that will be attached to the email
+	fileBackend, err := NewFileBackend(&cfg.FileSettings)
+	assert.Nil(t, err)
+	fileContents := []byte("hello world")
+	fileName := "file.txt"
+	assert.Nil(t, fileBackend.WriteFile(fileContents, fileName))
+	defer fileBackend.RemoveFile(fileName)
+
+	attachments := make([]*model.FileInfo, 1)
+	attachments[0] = &model.FileInfo{
+		Name: fileName,
+		Path: fileName,
+	}
+
+	headers := make(map[string]string)
+	headers["TestHeader"] = "TestValue"
+
+	if err := SendMailUsingConfigAdvanced(mimeTo, smtpTo, from, emailSubject, emailBody, attachments, headers, cfg); err != nil {
+		t.Log(err)
+		t.Fatal("Should connect to the STMP Server")
+	} else {
+		//Check if the email was send to the right email address
+		var resultsMailbox JSONMessageHeaderInbucket
+		err := RetryInbucket(5, func() error {
+			var err error
+			resultsMailbox, err = GetMailBox(smtpTo)
+			return err
+		})
+		if err != nil {
+			t.Log(err)
+			t.Fatal("No emails found for address " + smtpTo)
+		}
+		if err == nil && len(resultsMailbox) > 0 {
+			if !strings.ContainsAny(resultsMailbox[0].To[0], smtpTo) {
+				t.Fatal("Wrong To recipient")
+			} else {
+				if resultsEmail, err := GetMessageFromMailbox(smtpTo, resultsMailbox[0].ID); err == nil {
+					if !strings.Contains(resultsEmail.Body.Text, emailBody) {
+						t.Log(resultsEmail.Body.Text)
+						t.Fatal("Received message")
+					}
+
+					// verify that the To header of the email message is set to the MIME recipient, even though we got it out of the SMTP recipient's email inbox
+					assert.Equal(t, mimeTo, resultsEmail.Header["To"][0])
+
+					// verify that the MIME from address is correct - unfortunately, we can't verify the SMTP from address
+					assert.Equal(t, from.String(), resultsEmail.Header["From"][0])
+
+					// check that the custom mime headers came through - header case seems to get mutated
+					assert.Equal(t, "TestValue", resultsEmail.Header["Testheader"][0])
+
+					// ensure that the attachment was successfully sent
+					assert.Len(t, resultsEmail.Attachments, 1)
+					assert.Equal(t, fileName, resultsEmail.Attachments[0].Filename)
+					assert.Equal(t, fileContents, resultsEmail.Attachments[0].Bytes)
+				}
+			}
+		}
+	}
+}