Decouple emailservice from app package (#17827)

* decouple emailservice from app package

* fix some escaped errors

* move email package under app directory

* fix i18n

* reflect review comments
Этот коммит содержится в:
Ibrahim Serdar Acikgoz
2021-07-19 18:26:06 +03:00
коммит произвёл GitHub
родитель a78a7c7b54
Коммит 41dc05a6bd
35 изменённых файлов: 1156 добавлений и 555 удалений

1075
app/email/email.go Обычный файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

311
app/email/email_batching.go Обычный файл
Просмотреть файл

@@ -0,0 +1,311 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package email
import (
"bytes"
"fmt"
"html/template"
"io"
"net/http"
"strconv"
"sync"
"time"
"github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/shared/i18n"
"github.com/mattermost/mattermost-server/v5/shared/mlog"
)
const (
EmailBatchingTaskName = "Email Batching"
)
type postData struct {
SenderName string
ChannelName string
Message template.HTML
MessageURL string
SenderPhoto string
PostPhoto string
Time string
}
func (es *Service) InitEmailBatching() {
if *es.config().EmailSettings.EnableEmailBatching {
if es.EmailBatching == nil {
es.EmailBatching = NewEmailBatchingJob(es, *es.config().EmailSettings.EmailBatchingBufferSize)
}
// note that we don't support changing EmailBatchingBufferSize without restarting the server
es.EmailBatching.Start()
}
}
func (es *Service) AddNotificationEmailToBatch(user *model.User, post *model.Post, team *model.Team) *model.AppError {
if !*es.config().EmailSettings.EnableEmailBatching {
return model.NewAppError("AddNotificationEmailToBatch", "api.email_batching.add_notification_email_to_batch.disabled.app_error", nil, "", http.StatusNotImplemented)
}
if !es.EmailBatching.Add(user, post, team) {
mlog.Error("Email batching job's receiving channel was full. Please increase the EmailBatchingBufferSize.")
return model.NewAppError("AddNotificationEmailToBatch", "api.email_batching.add_notification_email_to_batch.channel_full.app_error", nil, "", http.StatusInternalServerError)
}
return nil
}
type batchedNotification struct {
userID string
post *model.Post
teamName string
}
type EmailBatchingJob struct {
config func() *model.Config
service *Service
newNotifications chan *batchedNotification
pendingNotifications map[string][]*batchedNotification
task *model.ScheduledTask
taskMutex sync.Mutex
}
func NewEmailBatchingJob(es *Service, bufferSize int) *EmailBatchingJob {
return &EmailBatchingJob{
config: es.config,
service: es,
newNotifications: make(chan *batchedNotification, bufferSize),
pendingNotifications: make(map[string][]*batchedNotification),
}
}
func (job *EmailBatchingJob) Start() {
mlog.Debug("Email batching job starting. Checking for pending emails periodically.", mlog.Int("interval_in_seconds", *job.config().EmailSettings.EmailBatchingInterval))
newTask := model.CreateRecurringTask(EmailBatchingTaskName, job.CheckPendingEmails, time.Duration(*job.config().EmailSettings.EmailBatchingInterval)*time.Second)
job.taskMutex.Lock()
oldTask := job.task
job.task = newTask
job.taskMutex.Unlock()
if oldTask != nil {
oldTask.Cancel()
}
}
func (job *EmailBatchingJob) Add(user *model.User, post *model.Post, team *model.Team) bool {
notification := &batchedNotification{
userID: user.Id,
post: post,
teamName: team.Name,
}
select {
case job.newNotifications <- notification:
return true
default:
// return false if we couldn't queue the email notification so that we can send an immediate email
return false
}
}
func (job *EmailBatchingJob) CheckPendingEmails() {
job.handleNewNotifications()
// it's a bit weird to pass the send email function through here, but it makes it so that we can test
// without actually sending emails
job.checkPendingNotifications(time.Now(), job.service.sendBatchedEmailNotification)
mlog.Debug("Email batching job ran. Some users still have notifications pending.", mlog.Int("number_of_users", len(job.pendingNotifications)))
}
func (job *EmailBatchingJob) handleNewNotifications() {
receiving := true
// read in new notifications to send
for receiving {
select {
case notification := <-job.newNotifications:
userID := notification.userID
if _, ok := job.pendingNotifications[userID]; !ok {
job.pendingNotifications[userID] = []*batchedNotification{notification}
} else {
job.pendingNotifications[userID] = append(job.pendingNotifications[userID], notification)
}
default:
receiving = false
}
}
}
func (job *EmailBatchingJob) checkPendingNotifications(now time.Time, handler func(string, []*batchedNotification)) {
for userID, notifications := range job.pendingNotifications {
batchStartTime := notifications[0].post.CreateAt
inspectedTeamNames := make(map[string]string)
for _, notification := range notifications {
// at most, we'll do one check for each team that notifications were sent for
if inspectedTeamNames[notification.teamName] != "" {
continue
}
team, nErr := job.service.store.Team().GetByName(notifications[0].teamName)
if nErr != nil {
mlog.Error("Unable to find Team id for notification", mlog.Err(nErr))
continue
}
if team != nil {
inspectedTeamNames[notification.teamName] = team.Id
}
// if the user has viewed any channels in this team since the notification was queued, delete
// all queued notifications
channelMembers, err := job.service.store.Channel().GetMembersForUser(inspectedTeamNames[notification.teamName], userID)
if err != nil {
mlog.Error("Unable to find ChannelMembers for user", mlog.Err(err))
continue
}
for _, channelMember := range *channelMembers {
if channelMember.LastViewedAt >= batchStartTime {
mlog.Debug("Deleted notifications for user", mlog.String("user_id", userID))
delete(job.pendingNotifications, userID)
break
}
}
}
// get how long we need to wait to send notifications to the user
var interval int64
preference, err := job.service.store.Preference().Get(userID, model.PREFERENCE_CATEGORY_NOTIFICATIONS, model.PREFERENCE_NAME_EMAIL_INTERVAL)
if err != nil {
// use the default batching interval if an error ocurrs while fetching user preferences
interval, _ = strconv.ParseInt(model.PREFERENCE_EMAIL_INTERVAL_BATCHING_SECONDS, 10, 64)
} else {
if value, err := strconv.ParseInt(preference.Value, 10, 64); err != nil {
// // use the default batching interval if an error ocurrs while deserializing user preferences
interval, _ = strconv.ParseInt(model.PREFERENCE_EMAIL_INTERVAL_BATCHING_SECONDS, 10, 64)
} else {
interval = value
}
}
// send the email notification if there are notifications to send AND it's been long enough
if len(job.pendingNotifications[userID]) > 0 && now.Sub(time.Unix(batchStartTime/1000, 0)) > time.Duration(interval)*time.Second {
job.service.goFn(func(userID string, notifications []*batchedNotification) func() {
return func() {
handler(userID, notifications)
}
}(userID, job.pendingNotifications[userID]))
delete(job.pendingNotifications, userID)
}
}
}
func (es *Service) sendBatchedEmailNotification(userID string, notifications []*batchedNotification) {
user, err := es.userService.GetUser(userID)
if err != nil {
mlog.Warn("Unable to find recipient for batched email notification")
return
}
translateFunc := i18n.GetUserTranslations(user.Locale)
displayNameFormat := *es.config().TeamSettings.TeammateNameDisplay
siteURL := *es.config().ServiceSettings.SiteURL
postsData := make([]*postData, 0 /* len */, len(notifications) /* cap */)
embeddedFiles := make(map[string]io.Reader)
emailNotificationContentsType := model.EMAIL_NOTIFICATION_CONTENTS_FULL
if license := es.license(); license != nil && *license.Features.EmailNotificationContents {
emailNotificationContentsType = *es.config().EmailSettings.EmailNotificationContentsType
}
if emailNotificationContentsType == model.EMAIL_NOTIFICATION_CONTENTS_FULL {
for i, notification := range notifications {
sender, errSender := es.userService.GetUser(notification.post.UserId)
if errSender != nil {
mlog.Warn("Unable to find sender of post for batched email notification")
}
channel, errCh := es.store.Channel().Get(notification.post.ChannelId, true)
if errCh != nil {
mlog.Warn("Unable to find channel of post for batched email notification")
}
senderProfileImage, _, errProfileImage := es.userService.GetProfileImage(sender)
if errProfileImage != nil {
mlog.Warn("Unable to get the sender user profile image.", mlog.String("user_id", sender.Id), mlog.Err(errProfileImage))
}
senderPhoto := fmt.Sprintf("user-avatar-%d.png", i)
if senderProfileImage != nil {
embeddedFiles[senderPhoto] = bytes.NewReader(senderProfileImage)
}
tm := time.Unix(notification.post.CreateAt/1000, 0)
timezone, _ := tm.Zone()
t := translateFunc("api.email_batching.send_batched_email_notification.time", map[string]interface{}{
"Hour": tm.Hour(),
"Minute": fmt.Sprintf("%02d", tm.Minute()),
"Month": translateFunc(tm.Month().String()),
"Day": tm.Day(),
"Year": tm.Year(),
"Timezone": timezone,
})
MessageURL := siteURL + "/" + notification.teamName + "/pl/" + notification.post.Id
postsData = append(postsData, &postData{
SenderPhoto: senderPhoto,
SenderName: sender.GetDisplayName(displayNameFormat),
Time: t,
ChannelName: channel.DisplayName,
Message: template.HTML(es.GetMessageForNotification(notification.post, translateFunc)),
MessageURL: MessageURL,
})
}
}
tm := time.Unix(notifications[0].post.CreateAt/1000, 0)
subject := translateFunc("api.email_batching.send_batched_email_notification.subject", len(notifications), map[string]interface{}{
"SiteName": es.config().TeamSettings.SiteName,
"Year": tm.Year(),
"Month": translateFunc(tm.Month().String()),
"Day": tm.Day(),
})
firstSender, err := es.userService.GetUser(notifications[0].post.UserId)
if err != nil {
mlog.Warn("Unable to find sender of post for batched email notification")
}
data := es.NewEmailTemplateData(user.Locale)
data.Props["SiteURL"] = siteURL
data.Props["Title"] = translateFunc("api.email_batching.send_batched_email_notification.title", len(notifications)-1, map[string]interface{}{
"SenderName": firstSender.GetDisplayName(displayNameFormat),
})
data.Props["SubTitle"] = translateFunc("api.email_batching.send_batched_email_notification.subTitle")
data.Props["Button"] = translateFunc("api.email_batching.send_batched_email_notification.button")
data.Props["ButtonURL"] = siteURL
data.Props["Posts"] = postsData
data.Props["MessageButton"] = translateFunc("api.email_batching.send_batched_email_notification.messageButton")
data.Props["NotificationFooterTitle"] = translateFunc("app.notification.footer.title")
data.Props["NotificationFooterInfoLogin"] = translateFunc("app.notification.footer.infoLogin")
data.Props["NotificationFooterInfo"] = translateFunc("app.notification.footer.info")
renderedPage, renderErr := es.templatesContainer.RenderToString("messages_notification", data)
if renderErr != nil {
mlog.Error("Unable to render email", mlog.Err(renderErr))
}
if nErr := es.SendNotificationMail(user.Email, subject, renderedPage); nErr != nil {
mlog.Warn("Unable to send batched email notification", mlog.String("email", user.Email), mlog.Err(nErr))
}
}

285
app/email/email_batching_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,285 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package email
import (
"context"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost-server/v5/model"
)
func TestHandleNewNotifications(t *testing.T) {
th := SetupWithStoreMock(t)
defer th.TearDown()
id1 := model.NewId()
id2 := model.NewId()
id3 := model.NewId()
// test queueing of received posts by user
job := NewEmailBatchingJob(th.service, 128)
job.handleNewNotifications()
require.Empty(t, job.pendingNotifications, "shouldn't have added any pending notifications")
job.Add(&model.User{Id: id1}, &model.Post{UserId: id1, Message: "test"}, &model.Team{Name: "team"})
require.Empty(t, job.pendingNotifications, "shouldn't have added any pending notifications")
job.handleNewNotifications()
require.Len(t, job.pendingNotifications, 1, "should have received posts for 1 user")
require.Len(t, job.pendingNotifications[id1], 1, "should have received 1 post for user")
job.Add(&model.User{Id: id1}, &model.Post{UserId: id1, Message: "test"}, &model.Team{Name: "team"})
job.handleNewNotifications()
require.Len(t, job.pendingNotifications, 1, "should have received posts for 1 user")
require.Len(t, job.pendingNotifications[id1], 2, "should have received 2 posts for user1")
job.Add(&model.User{Id: id2}, &model.Post{UserId: id1, Message: "test"}, &model.Team{Name: "team"})
job.handleNewNotifications()
require.Len(t, job.pendingNotifications, 2, "should have received posts for 2 users")
require.Len(t, job.pendingNotifications[id1], 2, "should have received 2 posts for user1")
require.Len(t, job.pendingNotifications[id2], 1, "should have received 1 post for user2")
job.Add(&model.User{Id: id2}, &model.Post{UserId: id2, Message: "test"}, &model.Team{Name: "team"})
job.Add(&model.User{Id: id1}, &model.Post{UserId: id3, Message: "test"}, &model.Team{Name: "team"})
job.Add(&model.User{Id: id3}, &model.Post{UserId: id3, Message: "test"}, &model.Team{Name: "team"})
job.Add(&model.User{Id: id2}, &model.Post{UserId: id2, Message: "test"}, &model.Team{Name: "team"})
job.handleNewNotifications()
require.Len(t, job.pendingNotifications, 3, "should have received posts for 3 users")
require.Len(t, job.pendingNotifications[id1], 3, "should have received 3 posts for user1")
require.Len(t, job.pendingNotifications[id2], 3, "should have received 3 posts for user2")
require.Len(t, job.pendingNotifications[id3], 1, "should have received 1 post for user3")
// test ordering of received posts
job = NewEmailBatchingJob(th.service, 128)
job.Add(&model.User{Id: id1}, &model.Post{UserId: id1, Message: "test1"}, &model.Team{Name: "team"})
job.Add(&model.User{Id: id1}, &model.Post{UserId: id1, Message: "test2"}, &model.Team{Name: "team"})
job.Add(&model.User{Id: id2}, &model.Post{UserId: id1, Message: "test3"}, &model.Team{Name: "team"})
job.Add(&model.User{Id: id1}, &model.Post{UserId: id1, Message: "test4"}, &model.Team{Name: "team"})
job.Add(&model.User{Id: id2}, &model.Post{UserId: id1, Message: "test5"}, &model.Team{Name: "team"})
job.handleNewNotifications()
assert.Equal(t, job.pendingNotifications[id1][0].post.Message, "test1", "incorrect order of received posts for user1")
assert.Equal(t, job.pendingNotifications[id1][1].post.Message, "test2", "incorrect order of received posts for user1")
assert.Equal(t, job.pendingNotifications[id1][2].post.Message, "test4", "incorrect order of received posts for user1")
assert.Equal(t, job.pendingNotifications[id2][0].post.Message, "test3", "incorrect order of received posts for user2")
assert.Equal(t, job.pendingNotifications[id2][1].post.Message, "test5", "incorrect order of received posts for user2")
}
func TestCheckPendingNotifications(t *testing.T) {
th := Setup(t).InitBasic()
defer th.TearDown()
job := NewEmailBatchingJob(th.service, 128)
job.pendingNotifications[th.BasicUser.Id] = []*batchedNotification{
{
post: &model.Post{
UserId: th.BasicUser.Id,
ChannelId: th.BasicChannel.Id,
CreateAt: 10000000,
},
teamName: th.BasicTeam.Name,
},
}
channelMember, err := th.store.Channel().GetMember(context.Background(), th.BasicChannel.Id, th.BasicUser.Id)
require.NoError(t, err)
channelMember.LastViewedAt = 9999999
_, err = th.store.Channel().UpdateMember(channelMember)
require.NoError(t, err)
nErr := th.store.Preference().Save(&model.Preferences{{
UserId: th.BasicUser.Id,
Category: model.PREFERENCE_CATEGORY_NOTIFICATIONS,
Name: model.PREFERENCE_NAME_EMAIL_INTERVAL,
Value: "60",
}})
require.NoError(t, nErr)
// test that notifications aren't sent before interval
job.checkPendingNotifications(time.Unix(10001, 0), func(string, []*batchedNotification) {})
require.NotNil(t, job.pendingNotifications[th.BasicUser.Id])
require.Len(t, job.pendingNotifications[th.BasicUser.Id], 1, "shouldn't have sent queued post")
// test that notifications are cleared if the user has acted
channelMember, err = th.store.Channel().GetMember(context.Background(), th.BasicChannel.Id, th.BasicUser.Id)
require.NoError(t, err)
channelMember.LastViewedAt = 10001000
_, err = th.store.Channel().UpdateMember(channelMember)
require.NoError(t, err)
// We reset the interval to something shorter
nErr = th.store.Preference().Save(&model.Preferences{{
UserId: th.BasicUser.Id,
Category: model.PREFERENCE_CATEGORY_NOTIFICATIONS,
Name: model.PREFERENCE_NAME_EMAIL_INTERVAL,
Value: "10",
}})
require.NoError(t, nErr)
var wasCalled int32
job.checkPendingNotifications(time.Unix(10050, 0), func(string, []*batchedNotification) {
atomic.StoreInt32(&wasCalled, int32(1))
})
// A hack to check whether the handler was called.
// It's not straightforward to just wait for it using a channel because the test should
// NOT call the handler, and it will be called only if the test fails.
time.Sleep(1 * time.Second)
// We do a check outside the email handler, because otherwise, failing from
// inside the handler doesn't let the .Go() function exit cleanly, and it gets
// stuck during server shutdown, trying to wait for the goroutine to exit
require.Equal(t, atomic.LoadInt32(&wasCalled), int32(0), "email handler should not have been called")
require.Nil(t, job.pendingNotifications[th.BasicUser.Id])
require.Empty(t, job.pendingNotifications[th.BasicUser.Id], "should've remove queued post since user acted")
// test that notifications are sent if enough time passes since the first message
job.pendingNotifications[th.BasicUser.Id] = []*batchedNotification{
{
post: &model.Post{
UserId: th.BasicUser.Id,
ChannelId: th.BasicChannel.Id,
CreateAt: 10060000,
Message: "post1",
},
teamName: th.BasicTeam.Name,
},
{
post: &model.Post{
UserId: th.BasicUser.Id,
ChannelId: th.BasicChannel.Id,
CreateAt: 10090000,
Message: "post2",
},
teamName: th.BasicTeam.Name,
},
}
received := make(chan *model.Post, 2)
job.checkPendingNotifications(time.Unix(10130, 0), func(s string, notifications []*batchedNotification) {
for _, notification := range notifications {
received <- notification.post
}
})
require.Nil(t, job.pendingNotifications[th.BasicUser.Id], "shouldn't have sent queued post")
select {
case post := <-received:
require.Equal(t, post.Message, "post1", "should've received post1 first")
case <-time.After(5 * time.Second):
require.Fail(t, "timed out waiting for first post notification")
}
select {
case post := <-received:
require.Equal(t, post.Message, "post2", "should've received post2 second")
case <-time.After(5 * time.Second):
require.Fail(t, "timed out waiting for second post notification")
}
}
/**
* Ensures that email batch interval defaults to 15 minutes for users that haven't explicitly set this preference
*/
func TestCheckPendingNotificationsDefaultInterval(t *testing.T) {
th := Setup(t).InitBasic()
defer th.TearDown()
job := NewEmailBatchingJob(th.service, 128)
// bypasses recent user activity check
require.NotNil(t, th.store)
require.NotNil(t, th.store.Channel())
require.NotNil(t, th.BasicUser)
require.NotNil(t, th.BasicChannel)
channelMember, err := th.store.Channel().GetMember(context.Background(), th.BasicChannel.Id, th.BasicUser.Id)
require.NoError(t, err)
channelMember.LastViewedAt = 9999000
_, err = th.store.Channel().UpdateMember(channelMember)
require.NoError(t, err)
job.pendingNotifications[th.BasicUser.Id] = []*batchedNotification{
{
post: &model.Post{
UserId: th.BasicUser.Id,
ChannelId: th.BasicChannel.Id,
CreateAt: 10000000,
},
teamName: th.BasicTeam.Name,
},
}
// notifications should not be sent 1s after post was created, because default batch interval is 15mins
job.checkPendingNotifications(time.Unix(10001, 0), func(string, []*batchedNotification) {})
require.NotNil(t, job.pendingNotifications[th.BasicUser.Id])
require.Len(t, job.pendingNotifications[th.BasicUser.Id], 1, "shouldn't have sent queued post")
// notifications should be sent 901s after post was created, because default batch interval is 15mins
job.checkPendingNotifications(time.Unix(10901, 0), func(string, []*batchedNotification) {})
require.Nil(t, job.pendingNotifications[th.BasicUser.Id])
require.Empty(t, job.pendingNotifications[th.BasicUser.Id], "should have sent queued post")
}
/**
* Ensures that email batch interval defaults to 15 minutes if user preference is invalid
*/
func TestCheckPendingNotificationsCantParseInterval(t *testing.T) {
th := Setup(t).InitBasic()
defer th.TearDown()
job := NewEmailBatchingJob(th.service, 128)
require.NotNil(t, th.store)
require.NotNil(t, th.store.Channel())
require.NotNil(t, th.BasicChannel)
require.NotNil(t, th.BasicUser)
// bypasses recent user activity check
channelMember, err := th.store.Channel().GetMember(context.Background(), th.BasicChannel.Id, th.BasicUser.Id)
require.NoError(t, err)
channelMember.LastViewedAt = 9999000
_, err = th.store.Channel().UpdateMember(channelMember)
require.NoError(t, err)
// preference value is not an integer, so we'll fall back to the default 15min value
nErr := th.store.Preference().Save(&model.Preferences{{
UserId: th.BasicUser.Id,
Category: model.PREFERENCE_CATEGORY_NOTIFICATIONS,
Name: model.PREFERENCE_NAME_EMAIL_INTERVAL,
Value: "notAnIntegerValue",
}})
require.NoError(t, nErr)
job.pendingNotifications[th.BasicUser.Id] = []*batchedNotification{
{
post: &model.Post{
UserId: th.BasicUser.Id,
ChannelId: th.BasicChannel.Id,
CreateAt: 10000000,
},
teamName: th.BasicTeam.Name,
},
}
// notifications should not be sent 1s after post was created, because default batch interval is 15mins
job.checkPendingNotifications(time.Unix(10001, 0), func(string, []*batchedNotification) {})
require.NotNil(t, job.pendingNotifications[th.BasicUser.Id])
require.Len(t, job.pendingNotifications[th.BasicUser.Id], 1, "shouldn't have sent queued post")
// notifications should be sent 901s after post was created, because default batch interval is 15mins
job.checkPendingNotifications(time.Unix(10901, 0), func(string, []*batchedNotification) {})
require.Nil(t, job.pendingNotifications[th.BasicUser.Id], "should have sent queued post")
}

73
app/email/email_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,73 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package email
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/shared/mail"
)
func TestCondenseSiteURL(t *testing.T) {
require.Equal(t, "", condenseSiteURL(""))
require.Equal(t, "mattermost.com", condenseSiteURL("mattermost.com"))
require.Equal(t, "mattermost.com", condenseSiteURL("mattermost.com/"))
require.Equal(t, "chat.mattermost.com", condenseSiteURL("chat.mattermost.com"))
require.Equal(t, "chat.mattermost.com", condenseSiteURL("chat.mattermost.com/"))
require.Equal(t, "mattermost.com/subpath", condenseSiteURL("mattermost.com/subpath"))
require.Equal(t, "mattermost.com/subpath", condenseSiteURL("mattermost.com/subpath/"))
require.Equal(t, "chat.mattermost.com/subpath", condenseSiteURL("chat.mattermost.com/subpath"))
require.Equal(t, "chat.mattermost.com/subpath", condenseSiteURL("chat.mattermost.com/subpath/"))
require.Equal(t, "mattermost.com:8080", condenseSiteURL("http://mattermost.com:8080"))
require.Equal(t, "mattermost.com:8080", condenseSiteURL("http://mattermost.com:8080/"))
require.Equal(t, "chat.mattermost.com:8080", condenseSiteURL("http://chat.mattermost.com:8080"))
require.Equal(t, "chat.mattermost.com:8080", condenseSiteURL("http://chat.mattermost.com:8080/"))
require.Equal(t, "mattermost.com:8080/subpath", condenseSiteURL("http://mattermost.com:8080/subpath"))
require.Equal(t, "mattermost.com:8080/subpath", condenseSiteURL("http://mattermost.com:8080/subpath/"))
require.Equal(t, "chat.mattermost.com:8080/subpath", condenseSiteURL("http://chat.mattermost.com:8080/subpath"))
require.Equal(t, "chat.mattermost.com:8080/subpath", condenseSiteURL("http://chat.mattermost.com:8080/subpath/"))
}
func TestSendInviteEmails(t *testing.T) {
th := Setup(t).InitBasic()
defer th.TearDown()
th.ConfigureInbucketMail()
th.UpdateConfig(func(cfg *model.Config) {
*cfg.ServiceSettings.EnableEmailInvitations = true
})
require.NotNil(t, th.BasicUser)
require.NotNil(t, th.BasicChannel)
emailTo := "test@example.com"
mail.DeleteMailBox(emailTo)
err := th.service.SendInviteEmails(th.BasicTeam, "test-user", th.BasicUser.Id, []string{emailTo}, "http://testserver")
require.NoError(t, err)
var resultsMailbox mail.JSONMessageHeaderInbucket
err2 := mail.RetryInbucket(5, func() error {
var err error
resultsMailbox, err = mail.GetMailBox(emailTo)
return err
})
if err2 != nil {
t.Log(err2)
t.Log("No email was received, maybe due load on the server. Skipping this verification")
} else if len(resultsMailbox) > 0 {
require.Len(t, resultsMailbox, 1)
require.Contains(t, resultsMailbox[0].To[0], emailTo, "Wrong To: recipient")
resultsEmail, err := mail.GetMessageFromMailbox(emailTo, resultsMailbox[0].ID)
require.NoError(t, err, "Could not get message from mailbox")
require.Contains(t, resultsEmail.Body.HTML, "http://testserver", "Wrong received message %s", resultsEmail.Body.Text)
require.Contains(t, resultsEmail.Body.HTML, "test-user", "Wrong received message %s", resultsEmail.Body.Text)
require.Contains(t, resultsEmail.Body.Text, "http://testserver", "Wrong received message %s", resultsEmail.Body.Text)
require.Contains(t, resultsEmail.Body.Text, "test-user", "Wrong received message %s", resultsEmail.Body.Text)
}
}

13
app/email/errors.go Обычный файл
Просмотреть файл

@@ -0,0 +1,13 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package email
import "github.com/pkg/errors"
var (
CreateEmailTokenError = errors.New("could not create token")
NoRateLimiterError = errors.New("the rate limit could not be found")
SetupRateLimiterError = errors.New("the rate limiter could not be set")
RateLimitExceededError = errors.New("the rate limit is exceeded")
)

307
app/email/helper_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,307 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package email
import (
"bytes"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/mattermost/mattermost-server/v5/config"
"github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/plugin/plugintest/mock"
"github.com/mattermost/mattermost-server/v5/services/users"
"github.com/mattermost/mattermost-server/v5/shared/mlog"
"github.com/mattermost/mattermost-server/v5/shared/templates"
"github.com/mattermost/mattermost-server/v5/store"
"github.com/mattermost/mattermost-server/v5/store/storetest/mocks"
"github.com/mattermost/mattermost-server/v5/testlib"
"github.com/mattermost/mattermost-server/v5/utils"
)
type TestHelper struct {
service *Service
configStore *config.Store
store store.Store
workspace string
BasicTeam *model.Team
BasicChannel *model.Channel
BasicUser *model.User
BasicUser2 *model.User
SystemAdminUser *model.User
LogBuffer *bytes.Buffer
}
func Setup(tb testing.TB) *TestHelper {
if testing.Short() {
tb.SkipNow()
}
dbStore := mainHelper.GetStore()
dbStore.DropAllTables()
dbStore.MarkSystemRanUnitTests()
mainHelper.PreloadMigrations()
return setupTestHelper(dbStore, tb)
}
func SetupWithStoreMock(tb testing.TB) *TestHelper {
mockStore := testlib.GetMockStoreForSetupFunctions()
th := setupTestHelper(mockStore, tb)
statusMock := mocks.StatusStore{}
statusMock.On("UpdateExpiredDNDStatuses").Return([]*model.Status{}, nil)
statusMock.On("Get", "user1").Return(&model.Status{UserId: "user1", Status: model.STATUS_ONLINE}, nil)
statusMock.On("UpdateLastActivityAt", "user1", mock.Anything).Return(nil)
statusMock.On("SaveOrUpdate", mock.AnythingOfType("*model.Status")).Return(nil)
emptyMockStore := mocks.Store{}
emptyMockStore.On("Close").Return(nil)
emptyMockStore.On("Status").Return(&statusMock)
th.service.store = &emptyMockStore
return th
}
func setupTestHelper(s store.Store, tb testing.TB) *TestHelper {
tempWorkspace, err := ioutil.TempDir("", "userservicetest")
if err != nil {
panic(err)
}
configStore := config.NewTestMemoryStore()
config := configStore.Get()
*config.PluginSettings.Directory = filepath.Join(tempWorkspace, "plugins")
*config.PluginSettings.ClientDirectory = filepath.Join(tempWorkspace, "webapp")
*config.PluginSettings.AutomaticPrepackagedPlugins = false
*config.LogSettings.EnableSentry = false // disable error reporting during tests
*config.AnnouncementSettings.AdminNoticesEnabled = false
*config.AnnouncementSettings.UserNoticesEnabled = false
*config.TeamSettings.MaxUsersPerTeam = 50
*config.RateLimitSettings.Enable = false
*config.TeamSettings.EnableOpenServer = true
// Disable strict password requirements for test
*config.PasswordSettings.MinimumLength = 5
*config.PasswordSettings.Lowercase = false
*config.PasswordSettings.Uppercase = false
*config.PasswordSettings.Symbol = false
*config.PasswordSettings.Number = false
configStore.Set(config)
licenseFn := func() *model.License { return model.NewTestLicense() }
us, err := users.New(users.ServiceConfig{
UserStore: s.User(),
SessionStore: s.Session(),
OAuthStore: s.OAuth(),
ConfigFn: configStore.Get,
LicenseFn: licenseFn,
})
if err != nil {
panic(err)
}
templatesDir, ok := templates.GetTemplateDirectory()
if !ok {
panic("failed find server templates")
}
htmlTemplateWatcher, errorsChan, err := templates.NewWithWatcher(templatesDir)
if err != nil {
panic(err)
}
go func() {
for err2 := range errorsChan {
mlog.Error("Server templates error", mlog.Err(err2))
}
}()
service := &Service{
store: s,
userService: us,
license: licenseFn,
config: configStore.Get,
templatesContainer: htmlTemplateWatcher,
goFn: func(f func()) { go f() },
}
if err := service.setUpRateLimiters(); err != nil {
panic(err)
}
return &TestHelper{
service: service,
configStore: configStore,
store: s,
LogBuffer: &bytes.Buffer{},
workspace: tempWorkspace,
}
}
func (th *TestHelper) InitBasic() *TestHelper {
th.BasicTeam = th.CreateTeam()
th.SystemAdminUser = th.CreateUser()
th.SystemAdminUser, _ = th.service.userService.GetUser(th.SystemAdminUser.Id)
th.addUserToTeam(th.BasicTeam, th.SystemAdminUser)
th.BasicUser = th.CreateUser()
th.BasicUser, _ = th.service.userService.GetUser(th.BasicUser.Id)
th.addUserToTeam(th.BasicTeam, th.BasicUser)
th.BasicUser2 = th.CreateUser()
th.BasicUser2, _ = th.service.userService.GetUser(th.BasicUser2.Id)
th.addUserToTeam(th.BasicTeam, th.BasicUser2)
th.BasicChannel = th.createChannel(th.BasicTeam, model.CHANNEL_OPEN)
th.addUserToChannel(th.BasicChannel, th.SystemAdminUser)
th.addUserToChannel(th.BasicChannel, th.BasicUser)
th.addUserToChannel(th.BasicChannel, th.BasicUser2)
return th
}
func (th *TestHelper) CreateTeam() *model.Team {
id := model.NewId()
team := &model.Team{
DisplayName: "dn_" + id,
Name: "name" + id,
Email: "success+" + id + "@simulator.amazonses.com",
Type: model.TEAM_OPEN,
}
utils.DisableDebugLogForTest()
var err error
if team, err = th.store.Team().Save(team); err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return team
}
func (th *TestHelper) createChannel(team *model.Team, channelType string) *model.Channel {
id := model.NewId()
channel := &model.Channel{
DisplayName: "dn_" + id,
Name: "name_" + id,
Type: channelType,
TeamId: team.Id,
CreatorId: th.BasicUser.Id,
}
utils.DisableDebugLogForTest()
var err error
if channel, err = th.store.Channel().Save(channel, *th.configStore.Get().TeamSettings.MaxChannelsPerTeam); err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return channel
}
func (th *TestHelper) addUserToChannel(channel *model.Channel, user *model.User) *model.ChannelMember {
newMember := &model.ChannelMember{
ChannelId: channel.Id,
UserId: user.Id,
NotifyProps: model.GetDefaultChannelNotifyProps(),
SchemeGuest: user.IsGuest(),
SchemeUser: !user.IsGuest(),
}
var err error
newMember, err = th.store.Channel().SaveMember(newMember)
if err != nil {
panic(err)
}
return newMember
}
func (th *TestHelper) addUserToTeam(team *model.Team, user *model.User) *model.TeamMember {
tm := &model.TeamMember{
TeamId: team.Id,
UserId: user.Id,
SchemeGuest: user.IsGuest(),
SchemeUser: !user.IsGuest(),
}
var err error
tm, err = th.store.Team().SaveMember(tm, *th.service.config().TeamSettings.MaxUsersPerTeam)
if err != nil {
panic(err)
}
return tm
}
func (th *TestHelper) CreateUser() *model.User {
return th.CreateUserOrGuest(false)
}
func (th *TestHelper) CreateGuest() *model.User {
return th.CreateUserOrGuest(true)
}
func (th *TestHelper) CreateUserOrGuest(guest bool) *model.User {
id := model.NewId()
user := &model.User{
Email: "success+" + id + "@simulator.amazonses.com",
Username: "un_" + id,
Nickname: "nn_" + id,
Password: "Password1",
EmailVerified: true,
}
var err error
if guest {
if user, err = th.service.userService.CreateUser(user, users.UserCreateOptions{Guest: true}); err != nil {
panic(err)
}
} else {
if user, err = th.service.userService.CreateUser(user, users.UserCreateOptions{}); err != nil {
panic(err)
}
}
return user
}
func (th *TestHelper) TearDown() {
th.configStore.Close()
th.store.Close()
if th.workspace != "" {
os.RemoveAll(th.workspace)
}
}
func (th *TestHelper) UpdateConfig(f func(*model.Config)) {
if th.configStore.IsReadOnly() {
return
}
old := th.configStore.Get()
updated := old.Clone()
f(updated)
if _, _, err := th.configStore.Set(updated); err != nil {
panic(err)
}
}
func (th *TestHelper) ConfigureInbucketMail() {
inbucket_host := os.Getenv("CI_INBUCKET_HOST")
if inbucket_host == "" {
inbucket_host = "localhost"
}
inbucket_port := os.Getenv("CI_INBUCKET_SMTP_PORT")
if inbucket_port == "" {
inbucket_port = "10025"
}
th.UpdateConfig(func(cfg *model.Config) {
*cfg.EmailSettings.SMTPServer = inbucket_host
*cfg.EmailSettings.SMTPPort = inbucket_port
})
}

35
app/email/main_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,35 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package email
import (
"flag"
"testing"
"github.com/mattermost/mattermost-server/v5/shared/mlog"
"github.com/mattermost/mattermost-server/v5/testlib"
)
var mainHelper *testlib.MainHelper
var replicaFlag bool
func TestMain(m *testing.M) {
if f := flag.Lookup("mysql-replica"); f == nil {
flag.BoolVar(&replicaFlag, "mysql-replica", false, "")
flag.Parse()
}
var options = testlib.HelperOptions{
EnableStore: true,
EnableResources: true,
WithReadReplica: replicaFlag,
}
mlog.DisableZap()
mainHelper = testlib.NewMainHelperWithOptions(&options)
defer mainHelper.Close()
mainHelper.Main(m)
}

46
app/email/notification_email.go Обычный файл
Просмотреть файл

@@ -0,0 +1,46 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package email
import (
"net/url"
"path/filepath"
"strings"
"github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/shared/i18n"
"github.com/mattermost/mattermost-server/v5/shared/mlog"
)
func (es *Service) GetMessageForNotification(post *model.Post, translateFunc i18n.TranslateFunc) string {
if strings.TrimSpace(post.Message) != "" || len(post.FileIds) == 0 {
return post.Message
}
// extract the filenames from their paths and determine what type of files are attached
infos, err := es.store.FileInfo().GetForPost(post.Id, true, false, true)
if err != nil {
mlog.Warn("Encountered error when getting files for notification message", mlog.String("post_id", post.Id), mlog.Err(err))
}
filenames := make([]string, len(infos))
onlyImages := true
for i, info := range infos {
if escaped, err := url.QueryUnescape(filepath.Base(info.Name)); err != nil {
// this should never error since filepath was escaped using url.QueryEscape
filenames[i] = escaped
} else {
filenames[i] = info.Name
}
onlyImages = onlyImages && info.IsImage()
}
props := map[string]interface{}{"Filenames": strings.Join(filenames, ", ")}
if onlyImages {
return translateFunc("api.post.get_message_for_notification.images_sent", len(filenames), props)
}
return translateFunc("api.post.get_message_for_notification.files_sent", len(filenames), props)
}

120
app/email/service.go Обычный файл
Просмотреть файл

@@ -0,0 +1,120 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package email
import (
"net/url"
"path"
"github.com/pkg/errors"
"github.com/throttled/throttled"
"github.com/throttled/throttled/store/memstore"
"github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/services/users"
"github.com/mattermost/mattermost-server/v5/shared/templates"
"github.com/mattermost/mattermost-server/v5/store"
)
const (
emailRateLimitingMemstoreSize = 65536
emailRateLimitingPerHour = 20
emailRateLimitingMaxBurst = 20
TokenTypePasswordRecovery = "password_recovery"
TokenTypeVerifyEmail = "verify_email"
TokenTypeTeamInvitation = "team_invitation"
TokenTypeGuestInvitation = "guest_invitation"
TokenTypeCWSAccess = "cws_access_token"
)
func condenseSiteURL(siteURL string) string {
parsedSiteURL, _ := url.Parse(siteURL)
if parsedSiteURL.Path == "" || parsedSiteURL.Path == "/" {
return parsedSiteURL.Host
}
return path.Join(parsedSiteURL.Host, parsedSiteURL.Path)
}
type Service struct {
config func() *model.Config
goFn func(f func())
license func() *model.License
userService *users.UserService
store store.Store
templatesContainer *templates.Container
PerHourEmailRateLimiter *throttled.GCRARateLimiter
PerDayEmailRateLimiter *throttled.GCRARateLimiter
EmailBatching *EmailBatchingJob
}
type ServiceConfig struct {
ConfigFn func() *model.Config
LicenseFn func() *model.License
GoFn func(f func())
TemplatesContainer *templates.Container
UserService *users.UserService
Store store.Store
}
func NewService(config ServiceConfig) (*Service, error) {
if err := config.validate(); err != nil {
return nil, err
}
service := &Service{
config: config.ConfigFn,
templatesContainer: config.TemplatesContainer,
license: config.LicenseFn,
goFn: config.GoFn,
store: config.Store,
userService: config.UserService,
}
if err := service.setUpRateLimiters(); err != nil {
return nil, err
}
service.InitEmailBatching()
return service, nil
}
func (c *ServiceConfig) validate() error {
if c.ConfigFn == nil || c.GoFn == nil || c.Store == nil || c.LicenseFn == nil || c.TemplatesContainer == nil {
return errors.New("invalid service config")
}
return nil
}
func (es *Service) setUpRateLimiters() error {
store, err := memstore.New(emailRateLimitingMemstoreSize)
if err != nil {
return errors.Wrap(err, "Unable to setup email rate limiting memstore.")
}
perHourQuota := throttled.RateQuota{
MaxRate: throttled.PerHour(emailRateLimitingPerHour),
MaxBurst: emailRateLimitingMaxBurst,
}
perDayQuota := throttled.RateQuota{
MaxRate: throttled.PerDay(1),
MaxBurst: 0,
}
perHourRateLimiter, err := throttled.NewGCRARateLimiter(store, perHourQuota)
if err != nil || perHourRateLimiter == nil {
return errors.Wrap(err, "Unable to setup email rate limiting GCRA rate limiter.")
}
perDayRateLimiter, err := throttled.NewGCRARateLimiter(store, perDayQuota)
if err != nil || perDayRateLimiter == nil {
return errors.Wrap(err, "Unable to setup per day email rate limiting GCRA rate limiter.")
}
es.PerHourEmailRateLimiter = perHourRateLimiter
es.PerDayEmailRateLimiter = perDayRateLimiter
return nil
}

31
app/email/utils.go Обычный файл
Просмотреть файл

@@ -0,0 +1,31 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package email
import (
"github.com/mattermost/mattermost-server/v5/shared/mail"
"github.com/mattermost/mattermost-server/v5/utils"
)
func (es *Service) mailServiceConfig() *mail.SMTPConfig {
emailSettings := es.config().EmailSettings
hostname := utils.GetHostnameFromSiteURL(*es.config().ServiceSettings.SiteURL)
cfg := mail.SMTPConfig{
Hostname: hostname,
ConnectionSecurity: *emailSettings.ConnectionSecurity,
SkipServerCertificateVerification: *emailSettings.SkipServerCertificateVerification,
ServerName: *emailSettings.SMTPServer,
Server: *emailSettings.SMTPServer,
Port: *emailSettings.SMTPPort,
ServerTimeout: *emailSettings.SMTPServerTimeout,
Username: *emailSettings.SMTPUsername,
Password: *emailSettings.SMTPPassword,
EnableSMTPAuth: *emailSettings.EnableSMTPAuth,
SendEmailNotifications: *emailSettings.SendEmailNotifications,
FeedbackName: *emailSettings.FeedbackName,
FeedbackEmail: *emailSettings.FeedbackEmail,
ReplyToAddress: *emailSettings.ReplyToAddress,
}
return &cfg
}