[MM-36872] add threading headers to notification emails (#18232)

* [MM-36872] Adding headers for mail threading

* MM-36872 Add message id and mail threading headers

* MM-36872 Add message id and mail threading headers

* MM-36872 Add msgId, inreplyto, references headers

* Fixing comment issues

* Applying gofmt lint fixes

* Applying gofmt lint fixes

* MM-36872: Updates required from merge.

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Co-authored-by: Martin Kraft <martin@upspin.org>

app/admin.go

@@ -231,7 +231,7 @@ func (a *App) TestEmail(userID string, cfg *model.Config) *model.AppError {
 	T := i18n.GetUserTranslations(user.Locale)
 	license := a.Srv().License()
 	mailConfig := a.Srv().MailServiceConfig()
-	if err := mail.SendMailUsingConfig(user.Email, T("api.admin.test_email.subject"), T("api.admin.test_email.body"), mailConfig, license != nil && *license.Features.Compliance, ""); err != nil {
+	if err := mail.SendMailUsingConfig(user.Email, T("api.admin.test_email.subject"), T("api.admin.test_email.body"), mailConfig, license != nil && *license.Features.Compliance, "", "", "", ""); err != nil {
 		return model.NewAppError("testEmail", "app.admin.test_email.failure", map[string]interface{}{"Error": err.Error()}, "", http.StatusInternalServerError)
 	}
 

app/admin_advisor.go

@@ -194,7 +194,7 @@ func (a *App) NotifyAndSetWarnMetricAck(warnMetricId string, sender *model.User,
 				return model.NewAppError("NotifyAndSetWarnMetricAck", "api.email.send_warn_metric_ack.failure.app_error", map[string]interface{}{"Error": err.Error()}, "", http.StatusInternalServerError)
 			}
 
-			if err := mail.SendMailUsingConfig(model.MmSupportAdvisorAddress, subject, body, mailConfig, false, sender.Email); err != nil {
+			if err := mail.SendMailUsingConfig(model.MmSupportAdvisorAddress, subject, body, mailConfig, false, "", "", "", sender.Email); err != nil {
 				return model.NewAppError("NotifyAndSetWarnMetricAck", "api.email.send_warn_metric_ack.failure.app_error", map[string]interface{}{"Error": err.Error()}, "", http.StatusInternalServerError)
 			}
 		}

app/email/email.go

@@ -625,7 +625,7 @@ func (es *Service) SendGuestInviteEmails(team *model.Team, channels []*model.Cha
 				mlog.Error("Failed to send invite email successfully", mlog.Err(err))
 			}
 
-			if nErr := es.SendMailWithEmbeddedFiles(invite, subject, body, embeddedFiles); nErr != nil {
+			if nErr := es.SendMailWithEmbeddedFiles(invite, subject, body, embeddedFiles, "", "", ""); nErr != nil {
 				mlog.Error("Failed to send invite email successfully", mlog.Err(nErr))
 				if errorWhenNotSent {
 					return SendMailError
@@ -769,7 +769,7 @@ func (es *Service) SendInviteEmailsToTeamAndChannels(
 			mlog.Error("Failed to send invite email successfully ", mlog.Err(err))
 		}
 
-		if nErr := es.SendMailWithEmbeddedFiles(invite, subject, body, embeddedFiles); nErr != nil {
+		if nErr := es.SendMailWithEmbeddedFiles(invite, subject, body, embeddedFiles, "", "", ""); nErr != nil {
 			mlog.Error("Failed to send invite email successfully", mlog.Err(nErr))
 			if errorWhenNotSent {
 				inviteWithError := &model.EmailInviteWithError{
@@ -832,7 +832,7 @@ func (es *Service) SendDeactivateAccountEmail(email string, locale, siteURL stri
 		return err
 	}
 
-	if err := es.sendMail(email, subject, body); err != nil {
+	if err := es.sendMail(email, subject, body); err != nil { // this needs to receive the header options
 		return err
 	}
 
@@ -854,21 +854,21 @@ func (es *Service) sendEmailWithCustomReplyTo(to, subject, htmlBody, replyToAddr
 	license := es.license()
 	mailConfig := es.mailServiceConfig(replyToAddress)
 
-	return mail.SendMailUsingConfig(to, subject, htmlBody, mailConfig, license != nil && *license.Features.Compliance, "")
+	return mail.SendMailUsingConfig(to, subject, htmlBody, mailConfig, license != nil && *license.Features.Compliance, "", "", "", "")
 }
 
 func (es *Service) sendMailWithCC(to, subject, htmlBody string, ccMail string) error {
 	license := es.license()
 	mailConfig := es.mailServiceConfig("")
 
-	return mail.SendMailUsingConfig(to, subject, htmlBody, mailConfig, license != nil && *license.Features.Compliance, ccMail)
+	return mail.SendMailUsingConfig(to, subject, htmlBody, mailConfig, license != nil && *license.Features.Compliance, "", "", "", ccMail)
 }
 
-func (es *Service) SendMailWithEmbeddedFiles(to, subject, htmlBody string, embeddedFiles map[string]io.Reader) error {
+func (es *Service) SendMailWithEmbeddedFiles(to, subject, htmlBody string, embeddedFiles map[string]io.Reader, messageID string, inReplyTo string, references string) error {
 	license := es.license()
 	mailConfig := es.mailServiceConfig("")
 
-	return mail.SendMailWithEmbeddedFilesUsingConfig(to, subject, htmlBody, embeddedFiles, mailConfig, license != nil && *license.Features.Compliance, "")
+	return mail.SendMailWithEmbeddedFilesUsingConfig(to, subject, htmlBody, embeddedFiles, mailConfig, license != nil && *license.Features.Compliance, messageID, inReplyTo, references, "")
 }
 
 func (es *Service) InvalidateVerifyEmailTokensForUser(userID string) *model.AppError {

app/email/email_batching.go

@@ -344,7 +344,7 @@ func (es *Service) sendBatchedEmailNotification(userID string, notifications []*
 		mlog.Error("Unable to render email", mlog.Err(renderErr))
 	}
 
-	if nErr := es.SendMailWithEmbeddedFiles(user.Email, subject, renderedPage, embeddedFiles); nErr != nil {
+	if nErr := es.SendMailWithEmbeddedFiles(user.Email, subject, renderedPage, embeddedFiles, "", "", ""); nErr != nil {
 		mlog.Warn("Unable to send batched email notification", mlog.String("email", user.Email), mlog.Err(nErr))
 	}
 }

app/email/mocks/ServiceInterface.go

@@ -302,13 +302,13 @@ func (_m *ServiceInterface) SendLicenseUpForRenewalEmail(_a0 string, name string
 	return r0
 }
 
-// SendMailWithEmbeddedFiles provides a mock function with given fields: to, subject, htmlBody, embeddedFiles
+// SendMailWithEmbeddedFiles provides a mock function with given fields: to, subject, htmlBody, embeddedFiles, messageID, inReplyTo, references
-func (_m *ServiceInterface) SendMailWithEmbeddedFiles(to string, subject string, htmlBody string, embeddedFiles map[string]io.Reader) error {
+func (_m *ServiceInterface) SendMailWithEmbeddedFiles(to string, subject string, htmlBody string, embeddedFiles map[string]io.Reader, messageID string, inReplyTo string, references string) error {
-	ret := _m.Called(to, subject, htmlBody, embeddedFiles)
+	ret := _m.Called(to, subject, htmlBody, embeddedFiles, messageID, inReplyTo, references)
 
 	var r0 error
-	if rf, ok := ret.Get(0).(func(string, string, string, map[string]io.Reader) error); ok {
+	if rf, ok := ret.Get(0).(func(string, string, string, map[string]io.Reader, string, string, string) error); ok {
-		r0 = rf(to, subject, htmlBody, embeddedFiles)
+		r0 = rf(to, subject, htmlBody, embeddedFiles, messageID, inReplyTo, references)
 	} else {
 		r0 = ret.Error(0)
 	}

app/email/service.go

@@ -142,7 +142,7 @@ type ServiceInterface interface {
 	SendInviteEmailsToTeamAndChannels(team *model.Team, channels []*model.Channel, senderName string, senderUserId string, senderProfileImage []byte, invites []string, siteURL string, reminderData *model.TeamInviteReminderData, message string, errorWhenNotSent bool) ([]*model.EmailInviteWithError, error)
 	SendDeactivateAccountEmail(email string, locale, siteURL string) error
 	SendNotificationMail(to, subject, htmlBody string) error
-	SendMailWithEmbeddedFiles(to, subject, htmlBody string, embeddedFiles map[string]io.Reader) error
+	SendMailWithEmbeddedFiles(to, subject, htmlBody string, embeddedFiles map[string]io.Reader, messageID string, inReplyTo string, references string) error
 	SendLicenseUpForRenewalEmail(email, name, locale, siteURL, renewalLink string, daysToExpiration int) error
 	SendPaymentFailedEmail(email string, locale string, failedPayment *model.FailedPayment, siteURL string) (bool, error)
 	SendNoCardPaymentFailedEmail(email string, locale string, siteURL string) error

app/notification_email.go

@@ -112,8 +112,23 @@ func (a *App) sendNotificationEmail(notification *PostNotification, user *model.
 		return errors.Wrap(err, "unable to render the email notification template")
 	}
 
+	templateString := "<%s@mattermost.com>"
+	messageID := ""
+	inReplyTo := ""
+	references := ""
+
+	if post.Id != "" {
+		messageID = fmt.Sprintf(templateString, post.Id)
+	}
+
+	if post.RootId != "" {
+		referencesVal := fmt.Sprintf(templateString, post.RootId)
+		inReplyTo = referencesVal
+		references = referencesVal
+	}
+
 	a.Srv().Go(func() {
-		if nErr := a.Srv().EmailService.SendMailWithEmbeddedFiles(user.Email, html.UnescapeString(subjectText), bodyText, embeddedFiles); nErr != nil {
+		if nErr := a.Srv().EmailService.SendMailWithEmbeddedFiles(user.Email, html.UnescapeString(subjectText), bodyText, embeddedFiles, messageID, inReplyTo, references); nErr != nil {
 			mlog.Error("Error while sending the email", mlog.String("user_email", user.Email), mlog.Err(nErr))
 		}
 	})

app/security_update_check.go

@@ -121,7 +121,7 @@ func (s *Server) DoSecurityUpdateCheck() {
 						mlog.Info("Sending security bulletin", mlog.String("bulletin_id", bulletin.Id), mlog.String("user_email", user.Email))
 						license := s.License()
 						mailConfig := s.MailServiceConfig()
-						mail.SendMailUsingConfig(user.Email, i18n.T("mattermost.bulletin.subject"), string(body), mailConfig, license != nil && *license.Features.Compliance, "")
+						mail.SendMailUsingConfig(user.Email, i18n.T("mattermost.bulletin.subject"), string(body), mailConfig, license != nil && *license.Features.Compliance, "", "", "", "")
 					}
 
 					bulletinSeen := &model.System{Name: "SecurityBulletin_" + bulletin.Id, Value: bulletin.Id}

shared/mail/mail.go

@@ -52,6 +52,9 @@ type mailData struct {
 	htmlBody      string
 	embeddedFiles map[string]io.Reader
 	mimeHeaders   map[string]string
+	messageID     string
+	inReplyTo     string
+	references    string
 }
 
 // smtpClient is implemented by an smtp.Client. See https://golang.org/pkg/net/smtp/#Client.
@@ -230,7 +233,7 @@ func TestConnection(config *SMTPConfig) error {
 	return nil
 }
 
-func SendMailWithEmbeddedFilesUsingConfig(to, subject, htmlBody string, embeddedFiles map[string]io.Reader, config *SMTPConfig, enableComplianceFeatures bool, ccMail string) error {
+func SendMailWithEmbeddedFilesUsingConfig(to, subject, htmlBody string, embeddedFiles map[string]io.Reader, config *SMTPConfig, enableComplianceFeatures bool, messageID string, inReplyTo string, references string, ccMail string) error {
 	fromMail := mail.Address{Name: config.FeedbackName, Address: config.FeedbackEmail}
 	replyTo := mail.Address{Name: config.FeedbackName, Address: config.ReplyToAddress}
 
@@ -243,13 +246,16 @@ func SendMailWithEmbeddedFilesUsingConfig(to, subject, htmlBody string, embedded
 		subject:       subject,
 		htmlBody:      htmlBody,
 		embeddedFiles: embeddedFiles,
+		messageID:     messageID,
+		inReplyTo:     inReplyTo,
+		references:    references,
 	}
 
 	return sendMailUsingConfigAdvanced(mail, config)
 }
 
-func SendMailUsingConfig(to, subject, htmlBody string, config *SMTPConfig, enableComplianceFeatures bool, ccMail string) error {
+func SendMailUsingConfig(to, subject, htmlBody string, config *SMTPConfig, enableComplianceFeatures bool, messageID string, inReplyTo string, references string, ccMail string) error {
-	return SendMailWithEmbeddedFilesUsingConfig(to, subject, htmlBody, nil, config, enableComplianceFeatures, ccMail)
+	return SendMailWithEmbeddedFilesUsingConfig(to, subject, htmlBody, nil, config, enableComplianceFeatures, messageID, inReplyTo, references, ccMail)
 }
 
 // allows for sending an email with differing MIME/SMTP recipients
@@ -308,6 +314,18 @@ func SendMail(c smtpClient, mail mailData, date time.Time) error {
 		headers["CC"] = []string{mail.cc}
 	}
 
+	if mail.messageID != "" {
+		headers["Message-ID"] = []string{mail.messageID}
+	}
+
+	if mail.inReplyTo != "" {
+		headers["In-Reply-To"] = []string{mail.inReplyTo}
+	}
+
+	if mail.references != "" {
+		headers["References"] = []string{mail.references}
+	}
+
 	for k, v := range mail.mimeHeaders {
 		headers[k] = []string{encodeRFC2047Word(v)}
 	}

shared/mail/mail_test.go

@@ -125,7 +125,7 @@ func TestSendMailUsingConfig(t *testing.T) {
 	//Delete all the messages before check the sample email
 	DeleteMailBox(emailTo)
 
-	err2 := SendMailUsingConfig(emailTo, emailSubject, emailBody, cfg, true, emailCC)
+	err2 := SendMailUsingConfig(emailTo, emailSubject, emailBody, cfg, true, "", "", "", emailCC)
 	require.NoError(t, err2, "Should connect to the SMTP Server")
 
 	//Check if the email was send to the right email address
@@ -163,7 +163,7 @@ func TestSendMailWithEmbeddedFilesUsingConfig(t *testing.T) {
 		"test1.png": bytes.NewReader([]byte("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")),
 		"test2.png": bytes.NewReader([]byte("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")),
 	}
-	err2 := SendMailWithEmbeddedFilesUsingConfig(emailTo, emailSubject, emailBody, embeddedFiles, cfg, true, emailCC)
+	err2 := SendMailWithEmbeddedFilesUsingConfig(emailTo, emailSubject, emailBody, embeddedFiles, cfg, true, "", "", "", emailCC)
 	require.NoError(t, err2, "Should connect to the SMTP Server")
 
 	//Check if the email was send to the right email address
@@ -333,24 +333,82 @@ func TestSendMail(t *testing.T) {
 
 	testCases := map[string]struct {
 		replyTo     mail.Address
+		messageID   string
+		inReplyTo   string
+		references  string
 		contains    string
 		notContains string
 	}{
 		"adds reply-to header": {
 			mail.Address{Address: "foo@test.com"},
+			"",
+			"",
+			"",
 			"\r\nReply-To: <foo@test.com>\r\n",
 			"",
 		},
 		"doesn't add reply-to header": {
 			mail.Address{},
 			"",
+			"",
+			"",
+			"",
 			"\r\nReply-To:",
 		},
+
+		"adds message-id header": {
+			mail.Address{},
+			"<abc123@mattermost.com>",
+			"",
+			"",
+			"\r\nMessage-ID: <abc123@mattermost.com>\r\n",
+			"",
+		},
+		"doesn't add message-id header": {
+			mail.Address{},
+			"",
+			"",
+			"",
+			"",
+			"\r\nMessage-ID:",
+		},
+		"adds in-reply-to header": {
+			mail.Address{},
+			"",
+			"<defg456@mattermost.com>",
+			"",
+			"\r\nIn-Reply-To: <defg456@mattermost.com>\r\n",
+			"",
+		},
+		"doesn't add in-reply-to header": {
+			mail.Address{},
+			"",
+			"",
+			"",
+			"",
+			"\r\nIn-Reply-To:",
+		},
+		"adds references header": {
+			mail.Address{},
+			"",
+			"",
+			"<ghi789@mattermost.com>",
+			"\r\nReferences: <ghi789@mattermost.com>\r\n",
+			"",
+		},
+		"doesn't add references header": {
+			mail.Address{},
+			"",
+			"",
+			"",
+			"",
+			"\r\nReferences:",
+		},
 	}
 
 	for testName, tc := range testCases {
 		t.Run(testName, func(t *testing.T) {
-			mail := mailData{"", "", mail.Address{}, "", tc.replyTo, "", "", nil, nil}
+			mail := mailData{"", "", mail.Address{}, "", tc.replyTo, "", "", nil, nil, tc.messageID, tc.inReplyTo, tc.references}
 			err = SendMail(mocm, mail, time.Now())
 			require.NoError(t, err)
 			if tc.contains != "" {