XYZ-35: Added Support for GlobalRelay Compliance Export Format

* Added username to ChannelMemberHistory struct in anticipation of supporting GlobalRelay in Compliance Export
* Removed translation from debug output - this makes it complicated to use utils functions from tests in the enterprise repo
* Added an advanced email function that allows for greater control over message details. Updated MessageExport config to support GlobalRelay. Added attachment support to InBucket unit tests
* Moving templates in from enterprise to solve test issues
* Added export format to diagnostics
* Changed email attachment code to use FileBackend so that S3 storage is properly supported

app/diagnostics.go

@@ -501,6 +501,7 @@ func (a *App) trackConfig() {
 
 	a.SendDiagnostic(TRACK_CONFIG_MESSAGE_EXPORT, map[string]interface{}{
 		"enable_message_export":         *cfg.MessageExportSettings.EnableExport,
+		"export_format":                 *cfg.MessageExportSettings.ExportFormat,
 		"daily_run_time":                *cfg.MessageExportSettings.DailyRunTime,
 		"default_export_from_timestamp": *cfg.MessageExportSettings.ExportFromTimestamp,
 		"batch_size":                    *cfg.MessageExportSettings.BatchSize,

i18n/en.json

@@ -131,10 +131,6 @@
     "id": "api.admin.upload_brand_image.too_large.app_error",
     "translation": "Unable to upload file. File is too large."
   },
-  {
-    "id": "api.api.init.parsing_templates.debug",
-    "translation": "Parsing server templates at %v"
-  },
   {
     "id": "api.api.init.parsing_templates.error",
     "translation": "Failed to parse server templates %v"
@@ -4858,6 +4854,14 @@
     "id": "model.config.is_valid.message_export.batch_size.app_error",
     "translation": "Message export job BatchSize must be a positive integer"
   },
+  {
+    "id": "model.config.is_valid.message_export.export_type.app_error",
+    "translation": "Message export job ExportFormat must be one of either 'actiance' or 'globalrelay'"
+  },
+  {
+    "id": "model.config.is_valid.message_export.global_relay_email_address.app_error",
+    "translation": "Message export job GlobalRelayEmailAddress must be set to a valid email address"
+  },
   {
     "id": "model.config.is_valid.message_export.daily_runtime.app_error",
     "translation": "Message export job DailyRuntime must be a 24-hour time stamp in the form HH:MM."
@@ -7082,6 +7086,10 @@
     "id": "utils.mail.new_client.auth.app_error",
     "translation": "Failed to authenticate on SMTP server"
   },
+  {
+    "id": "utils.mail.sendMail.attachments.write_error",
+    "translation": "Failed to write attachment to email"
+  },
   {
     "id": "utils.mail.new_client.helo.error",
     "translation": "Failed to to set the HELO to SMTP server %v"
@@ -7158,10 +7166,6 @@
     "id": "web.create_dir.error",
     "translation": "Failed to create directory watcher %v"
   },
-  {
-    "id": "web.dir_fail.error",
-    "translation": "Failed in directory watcher %v"
-  },
   {
     "id": "web.do_load_channel.error",
     "translation": "Error in getting users profile for id=%v forcing logout"
@@ -7266,18 +7270,10 @@
     "id": "web.parsing_templates.debug",
     "translation": "Parsing templates at %v"
   },
-  {
-    "id": "web.parsing_templates.error",
-    "translation": "Failed to parse templates %v"
-  },
   {
     "id": "web.post_permalink.app_error",
     "translation": "Invalid Post ID"
   },
-  {
-    "id": "web.reparse_templates.info",
-    "translation": "Re-parsing templates because of modified file %v"
-  },
   {
     "id": "web.reset_password.expired_link.app_error",
     "translation": "The password reset link has expired"

model/channel_member_history.go

@@ -6,7 +6,10 @@ package model
 type ChannelMemberHistory struct {
 	ChannelId string
 	UserId    string
-	UserEmail string `db:"Email"`
 	JoinTime  int64
 	LeaveTime *int64
+
+	// these two fields are never set in the database - when we SELECT, we join on Users to get them
+	UserEmail string `db:"Email"`
+	Username  string
 }

model/config.go

@@ -158,6 +158,9 @@ const (
 
 	PLUGIN_SETTINGS_DEFAULT_DIRECTORY        = "./plugins"
 	PLUGIN_SETTINGS_DEFAULT_CLIENT_DIRECTORY = "./client/plugins"
+
+	COMPLIANCE_EXPORT_TYPE_ACTIANCE    = "actiance"
+	COMPLIANCE_EXPORT_TYPE_GLOBALRELAY = "globalrelay"
 )
 
 type ServiceSettings struct {
@@ -1623,9 +1626,13 @@ func (s *PluginSettings) SetDefaults() {
 
 type MessageExportSettings struct {
 	EnableExport        *bool
+	ExportFormat        *string
 	DailyRunTime        *string
 	ExportFromTimestamp *int64
 	BatchSize           *int
+
+	// formatter-specific settings - these are only expected to be non-nil if ExportFormat is set to the associated format
+	GlobalRelayEmailAddress *string
 }
 
 func (s *MessageExportSettings) SetDefaults() {
@@ -1633,6 +1640,10 @@ func (s *MessageExportSettings) SetDefaults() {
 		s.EnableExport = NewBool(false)
 	}
 
+	if s.ExportFormat == nil {
+		s.ExportFormat = NewString(COMPLIANCE_EXPORT_TYPE_ACTIANCE)
+	}
+
 	if s.DailyRunTime == nil {
 		s.DailyRunTime = NewString("01:00")
 	}
@@ -2170,6 +2181,16 @@ func (mes *MessageExportSettings) isValid(fs FileSettings) *AppError {
 			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.daily_runtime.app_error", nil, err.Error(), http.StatusBadRequest)
 		} else if mes.BatchSize == nil || *mes.BatchSize < 0 {
 			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.batch_size.app_error", nil, "", http.StatusBadRequest)
+		} else if mes.ExportFormat == nil || (*mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_ACTIANCE && *mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_GLOBALRELAY) {
+			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.export_type.app_error", nil, "", http.StatusBadRequest)
+		}
+
+		if *mes.ExportFormat == COMPLIANCE_EXPORT_TYPE_GLOBALRELAY {
+			// validating email addresses is hard - just make sure it contains an '@' sign
+			// see https://stackoverflow.com/questions/201323/using-a-regular-expression-to-validate-an-email-address
+			if mes.GlobalRelayEmailAddress == nil || !strings.Contains(*mes.GlobalRelayEmailAddress, "@") {
+				return NewAppError("Config.IsValid", "model.config.is_valid.message_export.global_relay_email_address.app_error", nil, "", http.StatusBadRequest)
+			}
 		}
 	}
 	return nil

model/config_test.go

@@ -136,7 +136,7 @@ func TestMessageExportSettingsIsValidBatchSizeInvalid(t *testing.T) {
 	require.Error(t, mes.isValid(*fs))
 }
 
-func TestMessageExportSettingsIsValid(t *testing.T) {
+func TestMessageExportSettingsIsValidExportFormatInvalid(t *testing.T) {
 	fs := &FileSettings{
 		DriverName: NewString("foo"), // bypass file location check
 	}
@@ -147,6 +147,55 @@ func TestMessageExportSettingsIsValid(t *testing.T) {
 		BatchSize:           NewInt(100),
 	}
 
+	// should fail fast because export format isn't set
+	require.Error(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValidGlobalRelayEmailAddressInvalid(t *testing.T) {
+	fs := &FileSettings{
+		DriverName: NewString("foo"), // bypass file location check
+	}
+	mes := &MessageExportSettings{
+		EnableExport:        NewBool(true),
+		ExportFormat:        NewString(COMPLIANCE_EXPORT_TYPE_GLOBALRELAY),
+		ExportFromTimestamp: NewInt64(0),
+		DailyRunTime:        NewString("15:04"),
+		BatchSize:           NewInt(100),
+	}
+
+	// should fail fast because global relay email address isn't set
+	require.Error(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValidActiance(t *testing.T) {
+	fs := &FileSettings{
+		DriverName: NewString("foo"), // bypass file location check
+	}
+	mes := &MessageExportSettings{
+		EnableExport:        NewBool(true),
+		ExportFormat:        NewString(COMPLIANCE_EXPORT_TYPE_ACTIANCE),
+		ExportFromTimestamp: NewInt64(0),
+		DailyRunTime:        NewString("15:04"),
+		BatchSize:           NewInt(100),
+	}
+
+	// should pass because everything is valid
+	require.Nil(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValidGlobalRelay(t *testing.T) {
+	fs := &FileSettings{
+		DriverName: NewString("foo"), // bypass file location check
+	}
+	mes := &MessageExportSettings{
+		EnableExport:            NewBool(true),
+		ExportFormat:            NewString(COMPLIANCE_EXPORT_TYPE_GLOBALRELAY),
+		ExportFromTimestamp:     NewInt64(0),
+		DailyRunTime:            NewString("15:04"),
+		BatchSize:               NewInt(100),
+		GlobalRelayEmailAddress: NewString("test@mattermost.com"),
+	}
+
 	// should pass because everything is valid
 	require.Nil(t, mes.isValid(*fs))
 }
@@ -159,6 +208,7 @@ func TestMessageExportSetDefaults(t *testing.T) {
 	require.Equal(t, "01:00", *mes.DailyRunTime)
 	require.Equal(t, int64(0), *mes.ExportFromTimestamp)
 	require.Equal(t, 10000, *mes.BatchSize)
+	require.Equal(t, COMPLIANCE_EXPORT_TYPE_ACTIANCE, *mes.ExportFormat)
 }
 
 func TestMessageExportSetDefaultsExportEnabledExportFromTimestampNil(t *testing.T) {

model/message_export.go

@@ -9,6 +9,7 @@ type MessageExport struct {
 
 	UserId    *string
 	UserEmail *string
+	Username  *string
 
 	PostId       *string
 	PostCreateAt *int64

store/sqlstore/channel_member_history_store.go

@@ -110,7 +110,8 @@ func (s SqlChannelMemberHistoryStore) getFromChannelMemberHistoryTable(startTime
 	query := `
 			SELECT
 				cmh.*,
-				u.Email
+				u.Email,
+				u.Username
 			FROM ChannelMemberHistory cmh
 			INNER JOIN Users u ON cmh.UserId = u.Id
 			WHERE cmh.ChannelId = :ChannelId
@@ -130,9 +131,10 @@ func (s SqlChannelMemberHistoryStore) getFromChannelMemberHistoryTable(startTime
 func (s SqlChannelMemberHistoryStore) getFromChannelMembersTable(startTime int64, endTime int64, channelId string) ([]*model.ChannelMemberHistory, error) {
 	query := `
 		SELECT DISTINCT
-  			ch.ChannelId,
+			ch.ChannelId,
-  			ch.UserId,
+			ch.UserId,
-  			u.email
+			u.Email,
+			u.Username
 		FROM ChannelMembers AS ch
 		INNER JOIN Users AS u ON ch.UserId = u.id
 		WHERE ch.ChannelId = :ChannelId`
@@ -158,7 +160,7 @@ func (s SqlChannelMemberHistoryStore) PermanentDeleteBatch(endTime int64, limit
 			query =
 				`DELETE FROM ChannelMemberHistory
 				 WHERE ctid IN (
-				 	SELECT ctid FROM ChannelMemberHistory
+					SELECT ctid FROM ChannelMemberHistory
 					WHERE LeaveTime IS NOT NULL
 					AND LeaveTime <= :EndTime
 					LIMIT :Limit

store/sqlstore/compliance_store.go

@@ -225,7 +225,8 @@ func (s SqlComplianceStore) MessageExport(after int64, limit int) store.StoreCha
 				Channels.Id AS ChannelId,
 				Channels.DisplayName AS ChannelDisplayName,
 				Users.Id AS UserId,
-				Users.Email AS UserEmail
+				Users.Email AS UserEmail,
+				Users.Username
 			FROM
 				Posts
 				LEFT OUTER JOIN Channels ON Posts.ChannelId = Channels.Id

store/storetest/channel_member_history_store.go

@@ -35,6 +35,7 @@ func testLogJoinEvent(t *testing.T, ss store.Store) {
 	user := model.User{
 		Email:    model.NewId() + "@mattermost.com",
 		Nickname: model.NewId(),
+		Username: model.NewId(),
 	}
 	user = *store.Must(ss.User().Save(&user)).(*model.User)
 
@@ -57,6 +58,7 @@ func testLogLeaveEvent(t *testing.T, ss store.Store) {
 	user := model.User{
 		Email:    model.NewId() + "@mattermost.com",
 		Nickname: model.NewId(),
+		Username: model.NewId(),
 	}
 	user = *store.Must(ss.User().Save(&user)).(*model.User)
 
@@ -82,6 +84,7 @@ func testGetUsersInChannelAtChannelMemberHistory(t *testing.T, ss store.Store) {
 	user := model.User{
 		Email:    model.NewId() + "@mattermost.com",
 		Nickname: model.NewId(),
+		Username: model.NewId(),
 	}
 	user = *store.Must(ss.User().Save(&user)).(*model.User)
 
@@ -108,6 +111,7 @@ func testGetUsersInChannelAtChannelMemberHistory(t *testing.T, ss store.Store) {
 	assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
 	assert.Equal(t, user.Id, channelMembers[0].UserId)
 	assert.Equal(t, user.Email, channelMembers[0].UserEmail)
+	assert.Equal(t, user.Username, channelMembers[0].Username)
 	assert.Equal(t, joinTime, channelMembers[0].JoinTime)
 	assert.Nil(t, channelMembers[0].LeaveTime)
 
@@ -117,6 +121,7 @@ func testGetUsersInChannelAtChannelMemberHistory(t *testing.T, ss store.Store) {
 	assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
 	assert.Equal(t, user.Id, channelMembers[0].UserId)
 	assert.Equal(t, user.Email, channelMembers[0].UserEmail)
+	assert.Equal(t, user.Username, channelMembers[0].Username)
 	assert.Equal(t, joinTime, channelMembers[0].JoinTime)
 	assert.Nil(t, channelMembers[0].LeaveTime)
 
@@ -129,6 +134,7 @@ func testGetUsersInChannelAtChannelMemberHistory(t *testing.T, ss store.Store) {
 	assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
 	assert.Equal(t, user.Id, channelMembers[0].UserId)
 	assert.Equal(t, user.Email, channelMembers[0].UserEmail)
+	assert.Equal(t, user.Username, channelMembers[0].Username)
 	assert.Equal(t, joinTime, channelMembers[0].JoinTime)
 	assert.Equal(t, leaveTime, *channelMembers[0].LeaveTime)
 
@@ -138,6 +144,7 @@ func testGetUsersInChannelAtChannelMemberHistory(t *testing.T, ss store.Store) {
 	assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
 	assert.Equal(t, user.Id, channelMembers[0].UserId)
 	assert.Equal(t, user.Email, channelMembers[0].UserEmail)
+	assert.Equal(t, user.Username, channelMembers[0].Username)
 	assert.Equal(t, joinTime, channelMembers[0].JoinTime)
 	assert.Equal(t, leaveTime, *channelMembers[0].LeaveTime)
 
@@ -160,6 +167,7 @@ func testGetUsersInChannelAtChannelMembers(t *testing.T, ss store.Store) {
 	user := model.User{
 		Email:    model.NewId() + "@mattermost.com",
 		Nickname: model.NewId(),
+		Username: model.NewId(),
 	}
 	user = *store.Must(ss.User().Save(&user)).(*model.User)
 
@@ -192,6 +200,7 @@ func testGetUsersInChannelAtChannelMembers(t *testing.T, ss store.Store) {
 	assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
 	assert.Equal(t, user.Id, channelMembers[0].UserId)
 	assert.Equal(t, user.Email, channelMembers[0].UserEmail)
+	assert.Equal(t, user.Username, channelMembers[0].Username)
 	assert.Equal(t, joinTime-500, channelMembers[0].JoinTime)
 	assert.Equal(t, joinTime-100, *channelMembers[0].LeaveTime)
 
@@ -201,6 +210,7 @@ func testGetUsersInChannelAtChannelMembers(t *testing.T, ss store.Store) {
 	assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
 	assert.Equal(t, user.Id, channelMembers[0].UserId)
 	assert.Equal(t, user.Email, channelMembers[0].UserEmail)
+	assert.Equal(t, user.Username, channelMembers[0].Username)
 	assert.Equal(t, joinTime-100, channelMembers[0].JoinTime)
 	assert.Equal(t, joinTime+500, *channelMembers[0].LeaveTime)
 
@@ -210,6 +220,7 @@ func testGetUsersInChannelAtChannelMembers(t *testing.T, ss store.Store) {
 	assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
 	assert.Equal(t, user.Id, channelMembers[0].UserId)
 	assert.Equal(t, user.Email, channelMembers[0].UserEmail)
+	assert.Equal(t, user.Username, channelMembers[0].Username)
 	assert.Equal(t, joinTime+100, channelMembers[0].JoinTime)
 	assert.Equal(t, joinTime+500, *channelMembers[0].LeaveTime)
 
@@ -219,6 +230,7 @@ func testGetUsersInChannelAtChannelMembers(t *testing.T, ss store.Store) {
 	assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
 	assert.Equal(t, user.Id, channelMembers[0].UserId)
 	assert.Equal(t, user.Email, channelMembers[0].UserEmail)
+	assert.Equal(t, user.Username, channelMembers[0].Username)
 	assert.Equal(t, joinTime+100, channelMembers[0].JoinTime)
 	assert.Equal(t, leaveTime-100, *channelMembers[0].LeaveTime)
 
@@ -228,6 +240,7 @@ func testGetUsersInChannelAtChannelMembers(t *testing.T, ss store.Store) {
 	assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
 	assert.Equal(t, user.Id, channelMembers[0].UserId)
 	assert.Equal(t, user.Email, channelMembers[0].UserEmail)
+	assert.Equal(t, user.Username, channelMembers[0].Username)
 	assert.Equal(t, joinTime-100, channelMembers[0].JoinTime)
 	assert.Equal(t, leaveTime+100, *channelMembers[0].LeaveTime)
 
@@ -237,6 +250,7 @@ func testGetUsersInChannelAtChannelMembers(t *testing.T, ss store.Store) {
 	assert.Equal(t, channel.Id, channelMembers[0].ChannelId)
 	assert.Equal(t, user.Id, channelMembers[0].UserId)
 	assert.Equal(t, user.Email, channelMembers[0].UserEmail)
+	assert.Equal(t, user.Username, channelMembers[0].Username)
 	assert.Equal(t, leaveTime+100, channelMembers[0].JoinTime)
 	assert.Equal(t, leaveTime+200, *channelMembers[0].LeaveTime)
 }
@@ -255,12 +269,14 @@ func testPermanentDeleteBatch(t *testing.T, ss store.Store) {
 	user := model.User{
 		Email:    model.NewId() + "@mattermost.com",
 		Nickname: model.NewId(),
+		Username: model.NewId(),
 	}
 	user = *store.Must(ss.User().Save(&user)).(*model.User)
 
 	user2 := model.User{
 		Email:    model.NewId() + "@mattermost.com",
 		Nickname: model.NewId(),
+		Username: model.NewId(),
 	}
 	user2 = *store.Must(ss.User().Save(&user2)).(*model.User)
 

store/storetest/compliance_store.go

@@ -341,7 +341,8 @@ func testComplianceMessageExport(t *testing.T, ss store.Store) {
 
 	// and two users that are a part of that team
 	user1 := &model.User{
-		Email: model.NewId(),
+		Email:    model.NewId(),
+		Username: model.NewId(),
 	}
 	user1 = store.Must(ss.User().Save(user1)).(*model.User)
 	store.Must(ss.Team().SaveMember(&model.TeamMember{
@@ -350,7 +351,8 @@ func testComplianceMessageExport(t *testing.T, ss store.Store) {
 	}, -1))
 
 	user2 := &model.User{
-		Email: model.NewId(),
+		Email:    model.NewId(),
+		Username: model.NewId(),
 	}
 	user2 = store.Must(ss.User().Save(user2)).(*model.User)
 	store.Must(ss.Team().SaveMember(&model.TeamMember{
@@ -415,6 +417,7 @@ func testComplianceMessageExport(t *testing.T, ss store.Store) {
 	assert.Equal(t, channel.DisplayName, *messageExportMap[post1.Id].ChannelDisplayName)
 	assert.Equal(t, user1.Id, *messageExportMap[post1.Id].UserId)
 	assert.Equal(t, user1.Email, *messageExportMap[post1.Id].UserEmail)
+	assert.Equal(t, user1.Username, *messageExportMap[post1.Id].Username)
 
 	// post2 was made by user1 in channel1 and team1
 	assert.Equal(t, post2.Id, *messageExportMap[post2.Id].PostId)
@@ -424,6 +427,7 @@ func testComplianceMessageExport(t *testing.T, ss store.Store) {
 	assert.Equal(t, channel.DisplayName, *messageExportMap[post2.Id].ChannelDisplayName)
 	assert.Equal(t, user1.Id, *messageExportMap[post2.Id].UserId)
 	assert.Equal(t, user1.Email, *messageExportMap[post2.Id].UserEmail)
+	assert.Equal(t, user1.Username, *messageExportMap[post2.Id].Username)
 
 	// post3 is a DM between user1 and user2
 	assert.Equal(t, post3.Id, *messageExportMap[post3.Id].PostId)
@@ -432,4 +436,5 @@ func testComplianceMessageExport(t *testing.T, ss store.Store) {
 	assert.Equal(t, directMessageChannel.Id, *messageExportMap[post3.Id].ChannelId)
 	assert.Equal(t, user1.Id, *messageExportMap[post3.Id].UserId)
 	assert.Equal(t, user1.Email, *messageExportMap[post3.Id].UserEmail)
+	assert.Equal(t, user1.Username, *messageExportMap[post3.Id].Username)
 }

templates/globalrelay_compliance_export.html

@@ -0,0 +1,91 @@
+{{define "globalrelay_compliance_export"}}
+<style type="text/css">
+    body {
+        font-family:Arial, sans-serif;
+        font-size:14px;
+        font-weight:normal;
+    }
+
+    .summary-list ul {
+        padding: 0px;
+        list-style:none;
+    }
+    .summary-list li {
+        display: inline;
+        padding: 0 1em 0 0;
+    }
+    .summary-list .bold {
+        font-weight: bold;
+    }
+
+    .participants {
+        border-collapse:collapse;
+        border-spacing:0;
+    }
+    .participants td {
+        padding:10px 5px;
+        border:1px solid black;
+        overflow:hidden;
+        text-align: center;
+        word-break:normal;
+    }
+    .participants th {
+        padding:10px 5px;
+        border:1px solid black;
+        overflow:hidden;
+        word-break:normal;
+    }
+    .participants th,td {
+        vertical-align:top
+    }
+
+    .message-list ul {
+        list-style:none;
+        padding: 0;
+    }
+    .message-list li {
+        padding: 0 0 1em 0;
+    }
+    .message .sent_time {
+        font-weight:bold;
+    }
+    .message .username {
+        font-weight:bold;
+    }
+    .message .email {
+        font-weight: bold;
+    }
+</style>
+
+<h1>Mattermost Compliance Export</h1>
+
+<h2>Conversation Summary</h2>
+<div class="summary-list">
+    <ul>
+        <li><span class="bold">Channel:&nbsp;</span>{{.Props.ChannelName}}</li>
+        <li><span class="bold">Started:&nbsp;</span>{{.Props.Started}}</li>
+        <li><span class="bold">Ended:&nbsp;</span>{{.Props.Ended}}</li>
+        <li><span class="bold">Duration:&nbsp;</span>{{.Props.Duration}}&nbsp;Minutes</li>
+    </ul>
+</div>
+<table class="participants">
+    <tr>
+        <th class="username">Username<br></th>
+        <th class="email">Email</th>
+        <th class="joined">Joined</th>
+        <th class="left">Left</th>
+        <th class="duration">Duration</th>
+        <th class="messages">Messages</th>
+    </tr>
+    {{.Props.ParticipantRows}}
+</table>
+
+<h2>Messages</h2>
+<div class="message-list">
+    <ul>
+        {{.Props.Messages}}
+    </ul>
+</div>
+
+<p>Exported on {{.Props.ExportDate}}</p>
+{{end}}

templates/globalrelay_compliance_export_message.html

@@ -0,0 +1,8 @@
+{{define "globalrelay_compliance_export_message"}}
+<li class="message">
+    <span class="sent_time">{{.Props.SentTime}}</span>
+    <span class="username">@{{.Props.Username}}</span>
+    <span class="email">({{.Props.Email}}):</span>
+    <span class="message">{{.Props.Message}}</span>
+</li>
+{{end}}

templates/globalrelay_compliance_export_participant_row.html

@@ -0,0 +1,10 @@
+{{define "globalrelay_compliance_export_participant_row"}}
+<tr>
+    <td class="username">@{{.Props.Username}}</td>
+    <td class="email">{{.Props.Email}}</td>
+    <td class="joined">{{.Props.Joined}}</td>
+    <td class="left">{{.Props.Left}}</td>
+    <td class="duration">{{.Props.DurationMinutes}} Minutes</td>
+    <td class="messages">{{.Props.NumMessages}}</td>
+</tr>
+{{end}}

utils/html.go

@@ -23,7 +23,7 @@ type HTMLTemplateWatcher struct {
 
 func NewHTMLTemplateWatcher(directory string) (*HTMLTemplateWatcher, error) {
 	templatesDir, _ := FindDir(directory)
-	l4g.Debug(T("api.api.init.parsing_templates.debug"), templatesDir)
+	l4g.Debug("Parsing server templates at %v", templatesDir)
 
 	ret := &HTMLTemplateWatcher{
 		stop:    make(chan struct{}),
@@ -55,15 +55,15 @@ func NewHTMLTemplateWatcher(directory string) (*HTMLTemplateWatcher, error) {
 				return
 			case event := <-watcher.Events:
 				if event.Op&fsnotify.Write == fsnotify.Write {
-					l4g.Info(T("web.reparse_templates.info"), event.Name)
+					l4g.Info("Re-parsing templates because of modified file %v", event.Name)
 					if htmlTemplates, err := template.ParseGlob(templatesDir + "*.html"); err != nil {
-						l4g.Error(T("web.parsing_templates.error"), err)
+						l4g.Error("Failed to parse templates %v", err)
 					} else {
 						ret.templates.Store(htmlTemplates)
 					}
 				}
 			case err := <-watcher.Errors:
-				l4g.Error(T("web.dir_fail.error"), err)
+				l4g.Error("Failed in directory watcher %s", err)
 			}
 		}
 	}()

utils/inbucket.go

@@ -4,6 +4,7 @@
 package utils
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -37,6 +38,12 @@ type JSONMessageInbucket struct {
 		Text string
 		HTML string `json:"Html"`
 	}
+	Attachments []struct {
+		Filename     string
+		ContentType  string `json:"content-type"`
+		DownloadLink string `json:"download-link"`
+		Bytes        []byte `json:"-"`
+	}
 }
 
 func ParseEmail(email string) string {
@@ -89,21 +96,54 @@ func GetMessageFromMailbox(email, id string) (results JSONMessageInbucket, err e
 	var record JSONMessageInbucket
 
 	url := fmt.Sprintf("%s%s%s/%s", getInbucketHost(), INBUCKET_API, parsedEmail, id)
-	req, err := http.NewRequest("GET", url, nil)
+	emailResponse, err := get(url)
 	if err != nil {
 		return record, err
 	}
+	defer emailResponse.Body.Close()
+
+	err = json.NewDecoder(emailResponse.Body).Decode(&record)
+
+	// download attachments
+	if record.Attachments != nil && len(record.Attachments) > 0 {
+		for i := range record.Attachments {
+			if bytes, err := downloadAttachment(record.Attachments[i].DownloadLink); err != nil {
+				return record, err
+			} else {
+				record.Attachments[i].Bytes = make([]byte, len(bytes))
+				copy(record.Attachments[i].Bytes, bytes)
+			}
+		}
+	}
+
+	return record, err
+}
+
+func downloadAttachment(url string) ([]byte, error) {
+	attachmentResponse, err := get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer attachmentResponse.Body.Close()
+
+	buf := new(bytes.Buffer)
+	io.Copy(buf, attachmentResponse.Body)
+	return buf.Bytes(), nil
+}
+
+func get(url string) (*http.Response, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
 
 	client := &http.Client{}
-
 	resp, err := client.Do(req)
 	if err != nil {
-		return record, err
+		return nil, err
 	}
-	defer resp.Body.Close()
 
-	err = json.NewDecoder(resp.Body).Decode(&record)
+	return resp, nil
-	return record, err
 }
 
 func DeleteMailBox(email string) (err error) {

utils/mail.go

@@ -15,6 +15,8 @@ import (
 
 	"net/http"
 
+	"io"
+
 	l4g "github.com/alecthomas/log4go"
 	"github.com/mattermost/html2text"
 	"github.com/mattermost/mattermost-server/model"
@@ -104,36 +106,72 @@ func TestConnection(config *model.Config) {
 }
 
 func SendMailUsingConfig(to, subject, htmlBody string, config *model.Config) *model.AppError {
+	fromMail := mail.Address{Name: config.EmailSettings.FeedbackName, Address: config.EmailSettings.FeedbackEmail}
+	return sendMail(to, to, fromMail, subject, htmlBody, nil, nil, config)
+}
+
+// allows for sending an email with attachments and differing MIME/SMTP recipients
+func SendMailUsingConfigAdvanced(mimeTo, smtpTo string, from mail.Address, subject, htmlBody string, attachments []*model.FileInfo, mimeHeaders map[string]string, config *model.Config) *model.AppError {
+	return sendMail(mimeTo, smtpTo, from, subject, htmlBody, attachments, mimeHeaders, config)
+}
+
+func sendMail(mimeTo, smtpTo string, from mail.Address, subject, htmlBody string, attachments []*model.FileInfo, mimeHeaders map[string]string, config *model.Config) *model.AppError {
 	if !config.EmailSettings.SendEmailNotifications || len(config.EmailSettings.SMTPServer) == 0 {
 		return nil
 	}
 
-	l4g.Debug(T("utils.mail.send_mail.sending.debug"), to, subject)
+	l4g.Debug(T("utils.mail.send_mail.sending.debug"), mimeTo, subject)
 
 	htmlMessage := "\r\n<html><body>" + htmlBody + "</body></html>"
 
-	fromMail := mail.Address{Name: config.EmailSettings.FeedbackName, Address: config.EmailSettings.FeedbackEmail}
-
 	txtBody, err := html2text.FromString(htmlBody)
 	if err != nil {
 		l4g.Warn(err)
 		txtBody = ""
 	}
 
-	m := gomail.NewMessage(gomail.SetCharset("UTF-8"))
+	headers := map[string][]string{
-	m.SetHeaders(map[string][]string{
+		"From":                      {from.String()},
-		"From":                      {fromMail.String()},
+		"To":                        {mimeTo},
-		"To":                        {to},
 		"Subject":                   {encodeRFC2047Word(subject)},
 		"Content-Transfer-Encoding": {"8bit"},
 		"Auto-Submitted":            {"auto-generated"},
 		"Precedence":                {"bulk"},
-	})
+	}
-	m.SetDateHeader("Date", time.Now())
+	if mimeHeaders != nil {
+		for k, v := range mimeHeaders {
+			headers[k] = []string{encodeRFC2047Word(v)}
+		}
+	}
 
+	m := gomail.NewMessage(gomail.SetCharset("UTF-8"))
+	m.SetHeaders(headers)
+	m.SetDateHeader("Date", time.Now())
 	m.SetBody("text/plain", txtBody)
 	m.AddAlternative("text/html", htmlMessage)
 
+	if attachments != nil {
+		fileBackend, err := NewFileBackend(&config.FileSettings)
+		if err != nil {
+			return err
+		}
+
+		for _, fileInfo := range attachments {
+			m.Attach(fileInfo.Name, gomail.SetCopyFunc(func(writer io.Writer) error {
+				bytes, err := fileBackend.ReadFile(fileInfo.Path)
+				if err != nil {
+					return err
+				}
+				if _, err := writer.Write(bytes); err != nil {
+					return model.NewAppError("SendMail", "utils.mail.sendMail.attachments.write_error", nil, err.Error(), http.StatusInternalServerError)
+				}
+				return nil
+			}))
+
+		}
+
+	}
+
 	conn, err1 := connectToSMTPServer(config)
 	if err1 != nil {
 		return err1
@@ -147,11 +185,11 @@ func SendMailUsingConfig(to, subject, htmlBody string, config *model.Config) *mo
 	defer c.Quit()
 	defer c.Close()
 
-	if err := c.Mail(fromMail.Address); err != nil {
+	if err := c.Mail(from.Address); err != nil {
 		return model.NewAppError("SendMail", "utils.mail.send_mail.from_address.app_error", nil, err.Error(), http.StatusInternalServerError)
 	}
 
-	if err := c.Rcpt(to); err != nil {
+	if err := c.Rcpt(smtpTo); err != nil {
 		return model.NewAppError("SendMail", "utils.mail.send_mail.to_address.app_error", nil, err.Error(), http.StatusInternalServerError)
 	}
 

utils/mail_test.go

@@ -7,6 +7,10 @@ import (
 	"strings"
 	"testing"
 
+	"net/mail"
+
+	"github.com/mattermost/mattermost-server/model"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -39,9 +43,9 @@ func TestSendMailUsingConfig(t *testing.T) {
 	require.Nil(t, err)
 	T = GetUserTranslations("en")
 
-	var emailTo string = "test@example.com"
+	var emailTo = "test@example.com"
-	var emailSubject string = "Testing this email"
+	var emailSubject = "Testing this email"
-	var emailBody string = "This is a test from autobot"
+	var emailBody = "This is a test from autobot"
 
 	//Delete all the messages before check the sample email
 	DeleteMailBox(emailTo)
@@ -50,7 +54,7 @@ func TestSendMailUsingConfig(t *testing.T) {
 		t.Log(err)
 		t.Fatal("Should connect to the STMP Server")
 	} else {
-		//Check if the email was send to the rigth email address
+		//Check if the email was send to the right email address
 		var resultsMailbox JSONMessageHeaderInbucket
 		err := RetryInbucket(5, func() error {
 			var err error
@@ -75,3 +79,78 @@ func TestSendMailUsingConfig(t *testing.T) {
 		}
 	}
 }
+
+func TestSendMailUsingConfigAdvanced(t *testing.T) {
+	cfg, _, err := LoadConfig("config.json")
+	require.Nil(t, err)
+	T = GetUserTranslations("en")
+
+	var mimeTo = "test@example.com"
+	var smtpTo = "test2@example.com"
+	var from = mail.Address{Name: "Nobody", Address: "nobody@mattermost.com"}
+	var emailSubject = "Testing this email"
+	var emailBody = "This is a test from autobot"
+
+	//Delete all the messages before check the sample email
+	DeleteMailBox(smtpTo)
+
+	// create a file that will be attached to the email
+	fileBackend, err := NewFileBackend(&cfg.FileSettings)
+	assert.Nil(t, err)
+	fileContents := []byte("hello world")
+	fileName := "file.txt"
+	assert.Nil(t, fileBackend.WriteFile(fileContents, fileName))
+	defer fileBackend.RemoveFile(fileName)
+
+	attachments := make([]*model.FileInfo, 1)
+	attachments[0] = &model.FileInfo{
+		Name: fileName,
+		Path: fileName,
+	}
+
+	headers := make(map[string]string)
+	headers["TestHeader"] = "TestValue"
+
+	if err := SendMailUsingConfigAdvanced(mimeTo, smtpTo, from, emailSubject, emailBody, attachments, headers, cfg); err != nil {
+		t.Log(err)
+		t.Fatal("Should connect to the STMP Server")
+	} else {
+		//Check if the email was send to the right email address
+		var resultsMailbox JSONMessageHeaderInbucket
+		err := RetryInbucket(5, func() error {
+			var err error
+			resultsMailbox, err = GetMailBox(smtpTo)
+			return err
+		})
+		if err != nil {
+			t.Log(err)
+			t.Fatal("No emails found for address " + smtpTo)
+		}
+		if err == nil && len(resultsMailbox) > 0 {
+			if !strings.ContainsAny(resultsMailbox[0].To[0], smtpTo) {
+				t.Fatal("Wrong To recipient")
+			} else {
+				if resultsEmail, err := GetMessageFromMailbox(smtpTo, resultsMailbox[0].ID); err == nil {
+					if !strings.Contains(resultsEmail.Body.Text, emailBody) {
+						t.Log(resultsEmail.Body.Text)
+						t.Fatal("Received message")
+					}
+
+					// verify that the To header of the email message is set to the MIME recipient, even though we got it out of the SMTP recipient's email inbox
+					assert.Equal(t, mimeTo, resultsEmail.Header["To"][0])
+
+					// verify that the MIME from address is correct - unfortunately, we can't verify the SMTP from address
+					assert.Equal(t, from.String(), resultsEmail.Header["From"][0])
+
+					// check that the custom mime headers came through - header case seems to get mutated
+					assert.Equal(t, "TestValue", resultsEmail.Header["Testheader"][0])
+
+					// ensure that the attachment was successfully sent
+					assert.Len(t, resultsEmail.Attachments, 1)
+					assert.Equal(t, fileName, resultsEmail.Attachments[0].Filename)
+					assert.Equal(t, fileContents, resultsEmail.Attachments[0].Bytes)
+				}
+			}
+		}
+	}
+}