MM-60987: Move message_export code to monorepo (#28674)

* move message_export code to monorepo

* update license headers

* moving over `dedicated filestore` (#1772) from enterprise master

server/enterprise/external_imports.go

@@ -19,16 +19,8 @@ import (
 	// Needed to ensure the init() method in the EE gets run
 	_ "github.com/mattermost/enterprise/ldap"
 	// Needed to ensure the init() method in the EE gets run
-	_ "github.com/mattermost/enterprise/message_export"
-	// Needed to ensure the init() method in the EE gets run
 	_ "github.com/mattermost/enterprise/cloud"
 	// Needed to ensure the init() method in the EE gets run
-	_ "github.com/mattermost/enterprise/message_export/actiance_export"
-	// Needed to ensure the init() method in the EE gets run
-	_ "github.com/mattermost/enterprise/message_export/csv_export"
-	// Needed to ensure the init() method in the EE gets run
-	_ "github.com/mattermost/enterprise/message_export/global_relay_export"
-	// Needed to ensure the init() method in the EE gets run
 	_ "github.com/mattermost/enterprise/notification"
 	// Needed to ensure the init() method in the EE gets run
 	_ "github.com/mattermost/enterprise/oauth/google"

server/enterprise/local_imports.go

@@ -8,4 +8,12 @@ package enterprise
 import (
 	// Needed to ensure the init() method in the EE gets run
 	_ "github.com/mattermost/mattermost/server/v8/enterprise/metrics"
+	// Needed to ensure the init() method in the EE gets run
+	_ "github.com/mattermost/mattermost/server/v8/enterprise/message_export"
+	// Needed to ensure the init() method in the EE gets run
+	_ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/actiance_export"
+	// Needed to ensure the init() method in the EE gets run
+	_ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/csv_export"
+	// Needed to ensure the init() method in the EE gets run
+	_ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/global_relay_export"
 )

server/enterprise/message_export/actiance_export/actiance_export.go

@@ -0,0 +1,356 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package actiance_export
+
+import (
+	"bytes"
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+	"sort"
+	"strings"
+
+	"github.com/mattermost/mattermost/server/v8/enterprise/message_export/common_export"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+)
+
+const (
+	XMLNS                   = "http://www.w3.org/2001/XMLSchema-instance"
+	ActianceExportFilename  = "actiance_export.xml"
+	ActianceWarningFilename = "warning.txt"
+)
+
+// The root-level element of an actiance export
+type RootNode struct {
+	XMLName  xml.Name        `xml:"FileDump"`
+	XMLNS    string          `xml:"xmlns:xsi,attr"` // this should default to "http://www.w3.org/2001/XMLSchema-instance"
+	Channels []ChannelExport // one element per channel (open or invite-only), group message, or direct message
+}
+
+// The Conversation element indicates an ad hoc IM conversation or a group chat room.
+// The messages from a persistent chat room are exported once a day so that a Conversation entry contains the messages posted to a chat room from 12:00:00 AM to 11:59:59 PM
+type ChannelExport struct {
+	XMLName      xml.Name     `xml:"Conversation"`
+	Perspective  string       `xml:"Perspective,attr"` // the value of this attribute doesn't seem to matter. Using the channel name makes the export more human readable
+	ChannelId    string       `xml:"-"`                // the unique id of the channel
+	RoomId       string       `xml:"RoomID"`
+	StartTime    int64        `xml:"StartTimeUTC"` // utc timestamp (seconds), start of export period or create time of channel, whichever is greater. Example: 1366611728.
+	JoinEvents   []JoinExport // start with a list of all users who were present in the channel during the export period
+	Elements     []any
+	UploadStarts []*FileUploadStartExport
+	UploadStops  []*FileUploadStopExport
+	LeaveEvents  []LeaveExport // finish with a list of all users who were present in the channel during the export period
+	EndTime      int64         `xml:"EndTimeUTC"` // utc timestamp (seconds), end of export period or delete time of channel, whichever is lesser. Example: 1366611728.
+}
+
+// The ParticipantEntered element indicates each user who participates in a conversation.
+// For chat rooms, there must be one ParticipantEntered element for each user present in the chat room at the beginning of the reporting period
+type JoinExport struct {
+	XMLName          xml.Name `xml:"ParticipantEntered"`
+	UserEmail        string   `xml:"LoginName"`   // the email of the person that joined the channel
+	UserType         string   `xml:"UserType"`    // the type of the user that joined the channel
+	JoinTime         int64    `xml:"DateTimeUTC"` // utc timestamp (seconds), time at which the user joined. Example: 1366611728
+	CorporateEmailID string   `xml:"CorporateEmailID"`
+}
+
+// The ParticipantLeft element indicates the user who leaves an active IM or chat room conversation.
+// For chat rooms, there must be one ParticipantLeft element for each user present in the chat room at the end of the reporting period.
+type LeaveExport struct {
+	XMLName          xml.Name `xml:"ParticipantLeft"`
+	UserEmail        string   `xml:"LoginName"`   // the email of the person that left the channel
+	UserType         string   `xml:"UserType"`    // the type of the user that left the channel
+	LeaveTime        int64    `xml:"DateTimeUTC"` // utc timestamp (seconds), time at which the user left. Example: 1366611728
+	CorporateEmailID string   `xml:"CorporateEmailID"`
+}
+
+// The Message element indicates the message sent by a user
+type PostExport struct {
+	XMLName      xml.Name `xml:"Message"`
+	UserEmail    string   `xml:"LoginName"`    // the email of the person that sent the post
+	UserType     string   `xml:"UserType"`     // the type of the person that sent the post
+	PostTime     int64    `xml:"DateTimeUTC"`  // utc timestamp (seconds), time at which the user sent the post. Example: 1366611728
+	Message      string   `xml:"Content"`      // the text body of the post
+	PreviewsPost string   `xml:"PreviewsPost"` // the post id of the post that is previewed by the permalink preview feature
+}
+
+// The FileTransferStarted element indicates the beginning of a file transfer in a conversation
+type FileUploadStartExport struct {
+	XMLName         xml.Name `xml:"FileTransferStarted"`
+	UserEmail       string   `xml:"LoginName"`    // the email of the person that sent the file
+	UploadStartTime int64    `xml:"DateTimeUTC"`  // utc timestamp (seconds), time at which the user started the upload. Example: 1366611728
+	Filename        string   `xml:"UserFileName"` // the name of the file that was uploaded
+	FilePath        string   `xml:"FileName"`     // the path to the file, as stored on the server
+}
+
+// The FileTransferEnded element indicates the end of a file transfer in a conversation
+type FileUploadStopExport struct {
+	XMLName        xml.Name `xml:"FileTransferEnded"`
+	UserEmail      string   `xml:"LoginName"`    // the email of the person that sent the file
+	UploadStopTime int64    `xml:"DateTimeUTC"`  // utc timestamp (seconds), time at which the user finished the upload. Example: 1366611728
+	Filename       string   `xml:"UserFileName"` // the name of the file that was uploaded
+	FilePath       string   `xml:"FileName"`     // the path to the file, as stored on the server
+	Status         string   `xml:"Status"`       // set to either "Completed" or "Failed" depending on the outcome of the upload operation
+}
+
+func ActianceExport(rctx request.CTX, posts []*model.MessageExport, db store.Store, exportBackend filestore.FileBackend, fileAttachmentBackend filestore.FileBackend, exportDirectory string) (warningCount int64, appErr *model.AppError) {
+	// sort the posts into buckets based on the channel in which they appeared
+	membersByChannel := common_export.MembersByChannel{}
+	metadata := common_export.Metadata{
+		Channels:         map[string]common_export.MetadataChannel{},
+		MessagesCount:    0,
+		AttachmentsCount: 0,
+		StartTime:        0,
+		EndTime:          0,
+	}
+	elementsByChannel := map[string][]any{}
+	allUploadedFiles := []*model.FileInfo{}
+
+	for _, post := range posts {
+		if post == nil {
+			rctx.Logger().Warn("ignored a nil post reference in the list")
+			continue
+		}
+		elementsByChannel[*post.ChannelId] = append(elementsByChannel[*post.ChannelId], postToExportEntry(post, post.PostCreateAt, *post.PostMessage))
+
+		if post.PostDeleteAt != nil && *post.PostDeleteAt > 0 && post.PostProps != nil {
+			props := map[string]any{}
+			if json.Unmarshal([]byte(*post.PostProps), &props) == nil {
+				if _, ok := props[model.PostPropsDeleteBy]; ok {
+					elementsByChannel[*post.ChannelId] = append(elementsByChannel[*post.ChannelId], postToExportEntry(post,
+						post.PostDeleteAt, "delete "+*post.PostMessage))
+				}
+			}
+		}
+
+		startUploads, stopUploads, uploadedFiles, deleteFileMessages, err := postToAttachmentsEntries(post, db)
+		if err != nil {
+			return warningCount, err
+		}
+		elementsByChannel[*post.ChannelId] = append(elementsByChannel[*post.ChannelId], startUploads...)
+		elementsByChannel[*post.ChannelId] = append(elementsByChannel[*post.ChannelId], stopUploads...)
+		elementsByChannel[*post.ChannelId] = append(elementsByChannel[*post.ChannelId], deleteFileMessages...)
+
+		allUploadedFiles = append(allUploadedFiles, uploadedFiles...)
+
+		metadata.Update(post, len(uploadedFiles))
+
+		if _, ok := membersByChannel[*post.ChannelId]; !ok {
+			membersByChannel[*post.ChannelId] = common_export.ChannelMembers{}
+		}
+		membersByChannel[*post.ChannelId][*post.UserId] = common_export.ChannelMember{
+			Email:    *post.UserEmail,
+			UserId:   *post.UserId,
+			IsBot:    post.IsBot,
+			Username: *post.Username,
+		}
+	}
+
+	rctx.Logger().Info("Exported data for channels", mlog.Int("number_of_channels", len(metadata.Channels)))
+
+	channelExports := []ChannelExport{}
+	for _, channel := range metadata.Channels {
+		channelExport, err := buildChannelExport(
+			channel,
+			membersByChannel[channel.ChannelId],
+			elementsByChannel[channel.ChannelId],
+			db,
+		)
+		if err != nil {
+			return warningCount, err
+		}
+		channelExports = append(channelExports, *channelExport)
+	}
+
+	export := &RootNode{
+		XMLNS:    XMLNS,
+		Channels: channelExports,
+	}
+
+	return writeExport(rctx, export, allUploadedFiles, exportDirectory, exportBackend, fileAttachmentBackend)
+}
+
+func postToExportEntry(post *model.MessageExport, createTime *int64, message string) *PostExport {
+	userType := "user"
+	if post.IsBot {
+		userType = "bot"
+	}
+	return &PostExport{
+		PostTime:     *createTime,
+		Message:      message,
+		UserType:     userType,
+		UserEmail:    *post.UserEmail,
+		PreviewsPost: post.PreviewID(),
+	}
+}
+
+func postToAttachmentsEntries(post *model.MessageExport, db store.Store) ([]any, []any, []*model.FileInfo, []any, *model.AppError) {
+	// if the post included any files, we need to add special elements to the export.
+	if len(post.PostFileIds) == 0 {
+		return nil, nil, nil, nil, nil
+	}
+
+	fileInfos, err := db.FileInfo().GetForPost(*post.PostId, true, true, false)
+	if err != nil {
+		return nil, nil, nil, nil, model.NewAppError("postToAttachmentsEntries", "ent.message_export.actiance_export.get_attachment_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	startUploads := []any{}
+	stopUploads := []any{}
+	deleteFileMessages := []any{}
+
+	uploadedFiles := []*model.FileInfo{}
+	for _, fileInfo := range fileInfos {
+		// insert a record of the file upload into the export file
+		// path to exported file is relative to the xml file, so it's just the name of the exported file
+		startUploads = append(startUploads, &FileUploadStartExport{
+			UserEmail:       *post.UserEmail,
+			Filename:        fileInfo.Name,
+			FilePath:        fileInfo.Path,
+			UploadStartTime: *post.PostCreateAt,
+		})
+
+		stopUploads = append(stopUploads, &FileUploadStopExport{
+			UserEmail:      *post.UserEmail,
+			Filename:       fileInfo.Name,
+			FilePath:       fileInfo.Path,
+			UploadStopTime: *post.PostCreateAt,
+			Status:         "Completed",
+		})
+
+		if fileInfo.DeleteAt > 0 && post.PostDeleteAt != nil {
+			deleteFileMessages = append(deleteFileMessages, postToExportEntry(post, post.PostDeleteAt, "delete "+fileInfo.Path))
+		}
+
+		uploadedFiles = append(uploadedFiles, fileInfo)
+	}
+	return startUploads, stopUploads, uploadedFiles, deleteFileMessages, nil
+}
+
+func buildChannelExport(channel common_export.MetadataChannel, members common_export.ChannelMembers, elements []any, db store.Store) (*ChannelExport, *model.AppError) {
+	channelExport := ChannelExport{
+		ChannelId:   channel.ChannelId,
+		RoomId:      fmt.Sprintf("%v - %v - %v", common_export.ChannelTypeDisplayName(channel.ChannelType), channel.ChannelName, channel.ChannelId),
+		StartTime:   channel.StartTime,
+		EndTime:     channel.EndTime,
+		Perspective: channel.ChannelDisplayName,
+	}
+
+	channelMembersHistory, err := db.ChannelMemberHistory().GetUsersInChannelDuring(channel.StartTime, channel.EndTime, channel.ChannelId)
+	if err != nil {
+		return nil, model.NewAppError("buildChannelExport", "ent.get_users_in_channel_during", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	joins, leaves := common_export.GetJoinsAndLeavesForChannel(channel.StartTime, channel.EndTime, channelMembersHistory, members)
+	type StillJoinedInfo struct {
+		Time int64
+		Type string
+	}
+	stillJoined := map[string]StillJoinedInfo{}
+	for _, join := range joins {
+		userType := "user"
+		if join.IsBot {
+			userType = "bot"
+		}
+		channelExport.JoinEvents = append(channelExport.JoinEvents, JoinExport{
+			JoinTime:         join.Datetime,
+			UserEmail:        join.Email,
+			UserType:         userType,
+			CorporateEmailID: join.Email,
+		})
+		if value, ok := stillJoined[join.Email]; !ok {
+			stillJoined[join.Email] = StillJoinedInfo{Time: join.Datetime, Type: userType}
+		} else {
+			if join.Datetime > value.Time {
+				stillJoined[join.Email] = StillJoinedInfo{Time: join.Datetime, Type: userType}
+			}
+		}
+	}
+	for _, leave := range leaves {
+		userType := "user"
+		if leave.IsBot {
+			userType = "bot"
+		}
+		channelExport.LeaveEvents = append(channelExport.LeaveEvents, LeaveExport{
+			LeaveTime:        leave.Datetime,
+			UserEmail:        leave.Email,
+			UserType:         userType,
+			CorporateEmailID: leave.Email,
+		})
+		if leave.Datetime > stillJoined[leave.Email].Time {
+			delete(stillJoined, leave.Email)
+		}
+	}
+
+	for email := range stillJoined {
+		channelExport.LeaveEvents = append(channelExport.LeaveEvents, LeaveExport{
+			LeaveTime:        channel.EndTime,
+			UserEmail:        email,
+			UserType:         stillJoined[email].Type,
+			CorporateEmailID: email,
+		})
+	}
+
+	sort.Slice(channelExport.LeaveEvents, func(i, j int) bool {
+		if channelExport.LeaveEvents[i].LeaveTime == channelExport.LeaveEvents[j].LeaveTime {
+			return channelExport.LeaveEvents[i].UserEmail < channelExport.LeaveEvents[j].UserEmail
+		}
+		return channelExport.LeaveEvents[i].LeaveTime < channelExport.LeaveEvents[j].LeaveTime
+	})
+
+	channelExport.Elements = elements
+	return &channelExport, nil
+}
+
+func writeExport(rctx request.CTX, export *RootNode, uploadedFiles []*model.FileInfo, exportDirectory string, exportBackend filestore.FileBackend, fileAttachmentBackend filestore.FileBackend) (warningCount int64, appErr *model.AppError) {
+	// marshal the export object to xml
+	xmlData := &bytes.Buffer{}
+	xmlData.WriteString(xml.Header)
+
+	enc := xml.NewEncoder(xmlData)
+	enc.Indent("", "  ")
+	if err := enc.Encode(export); err != nil {
+		return warningCount, model.NewAppError("ActianceExport.AtianceExport", "ent.actiance.export.marshalToXml.appError", nil, "", 0).Wrap(err)
+	}
+	enc.Flush()
+
+	// Try to disable the write timeout if the backend supports it
+	if _, err := filestore.TryWriteFileContext(rctx.Context(), exportBackend, xmlData, path.Join(exportDirectory, ActianceExportFilename)); err != nil {
+		return warningCount, model.NewAppError("ActianceExport.AtianceExport", "ent.actiance.export.write_file.appError", nil, "", 0).Wrap(err)
+	}
+
+	var missingFiles []string
+	for _, fileInfo := range uploadedFiles {
+		var attachmentSrc io.ReadCloser
+		attachmentSrc, nErr := fileAttachmentBackend.Reader(fileInfo.Path)
+		if nErr != nil {
+			missingFiles = append(missingFiles, "Warning:"+common_export.MissingFileMessage+" - "+fileInfo.Path)
+			rctx.Logger().Warn(common_export.MissingFileMessage, mlog.String("FileName", fileInfo.Path))
+			continue
+		}
+		defer attachmentSrc.Close()
+
+		destPath := path.Join(exportDirectory, fileInfo.Path)
+
+		_, nErr = exportBackend.WriteFile(attachmentSrc, destPath)
+		if nErr != nil {
+			return warningCount, model.NewAppError("ActianceExport.AtianceExport", "ent.actiance.export.write_file.appError", nil, "", 0).Wrap(nErr)
+		}
+	}
+	warningCount = int64(len(missingFiles))
+	if warningCount > 0 {
+		_, err := filestore.TryWriteFileContext(rctx.Context(), exportBackend, strings.NewReader(strings.Join(missingFiles, "\n")), path.Join(exportDirectory, ActianceWarningFilename))
+		if err != nil {
+			appErr = model.NewAppError("ActianceExport.AtianceExport", "ent.actiance.export.write_file.appError", nil, "", 0).Wrap(err)
+		}
+	}
+	return warningCount, appErr
+}

server/enterprise/message_export/common_export/common_export.go

@@ -0,0 +1,155 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common_export
+
+import (
+	"fmt"
+
+	"github.com/mattermost/mattermost/server/public/model"
+)
+
+const MissingFileMessage = "File missing for post; cannot copy file to archive"
+
+type ChannelMemberJoin struct {
+	UserId   string
+	IsBot    bool
+	Email    string
+	Username string
+	Datetime int64
+}
+
+type ChannelMemberLeave struct {
+	UserId   string
+	IsBot    bool
+	Email    string
+	Username string
+	Datetime int64
+}
+
+type ChannelMember struct {
+	UserId   string
+	IsBot    bool
+	Email    string
+	Username string
+}
+type ChannelMembers map[string]ChannelMember
+type MembersByChannel map[string]ChannelMembers
+
+type MetadataChannel struct {
+	TeamId             *string
+	TeamName           *string
+	TeamDisplayName    *string
+	ChannelId          string
+	ChannelName        string
+	ChannelDisplayName string
+	ChannelType        model.ChannelType
+	RoomId             string
+	StartTime          int64
+	EndTime            int64
+	MessagesCount      int
+	AttachmentsCount   int
+}
+
+type Metadata struct {
+	Channels         map[string]MetadataChannel
+	MessagesCount    int
+	AttachmentsCount int
+	StartTime        int64
+	EndTime          int64
+}
+
+func (metadata *Metadata) Update(post *model.MessageExport, attachments int) {
+	channelMetadata, ok := metadata.Channels[*post.ChannelId]
+	if !ok {
+		channelMetadata = MetadataChannel{
+			TeamId:             post.TeamId,
+			TeamName:           post.TeamName,
+			TeamDisplayName:    post.TeamDisplayName,
+			ChannelId:          *post.ChannelId,
+			ChannelName:        *post.ChannelName,
+			ChannelDisplayName: *post.ChannelDisplayName,
+			ChannelType:        *post.ChannelType,
+			RoomId:             fmt.Sprintf("%v - %v", ChannelTypeDisplayName(*post.ChannelType), *post.ChannelId),
+			StartTime:          *post.PostCreateAt,
+			MessagesCount:      0,
+			AttachmentsCount:   0,
+		}
+	}
+
+	channelMetadata.EndTime = *post.PostCreateAt
+	channelMetadata.AttachmentsCount += attachments
+	metadata.AttachmentsCount += attachments
+	channelMetadata.MessagesCount += 1
+	metadata.MessagesCount += 1
+	if metadata.StartTime == 0 {
+		metadata.StartTime = *post.PostCreateAt
+	}
+	metadata.EndTime = *post.PostCreateAt
+	metadata.Channels[*post.ChannelId] = channelMetadata
+}
+
+func GetJoinsAndLeavesForChannel(startTime int64, endTime int64, channelMembersHistory []*model.ChannelMemberHistoryResult, channelMembers ChannelMembers) ([]ChannelMemberJoin, []ChannelMemberLeave) {
+	joins := []ChannelMemberJoin{}
+	leaves := []ChannelMemberLeave{}
+
+	alreadyJoined := map[string]bool{}
+	for _, cmh := range channelMembersHistory {
+		if cmh.UserDeleteAt > 0 && cmh.UserDeleteAt < startTime {
+			continue
+		}
+
+		if cmh.JoinTime > endTime {
+			continue
+		}
+
+		if cmh.LeaveTime != nil && *cmh.LeaveTime < startTime {
+			continue
+		}
+
+		if cmh.JoinTime <= endTime {
+			joins = append(joins, ChannelMemberJoin{
+				UserId:   cmh.UserId,
+				IsBot:    cmh.IsBot,
+				Email:    cmh.UserEmail,
+				Username: cmh.Username,
+				Datetime: cmh.JoinTime,
+			})
+			alreadyJoined[cmh.UserId] = true
+		}
+
+		if cmh.LeaveTime != nil && *cmh.LeaveTime <= endTime {
+			leaves = append(leaves, ChannelMemberLeave{
+				UserId:   cmh.UserId,
+				IsBot:    cmh.IsBot,
+				Email:    cmh.UserEmail,
+				Username: cmh.Username,
+				Datetime: *cmh.LeaveTime,
+			})
+		}
+	}
+
+	for _, member := range channelMembers {
+		if alreadyJoined[member.UserId] {
+			continue
+		}
+
+		joins = append(joins, ChannelMemberJoin{
+			UserId:   member.UserId,
+			IsBot:    member.IsBot,
+			Email:    member.Email,
+			Username: member.Username,
+			Datetime: startTime,
+		})
+	}
+	return joins, leaves
+}
+
+func ChannelTypeDisplayName(channelType model.ChannelType) string {
+	return map[model.ChannelType]string{
+		model.ChannelTypeOpen:    "public",
+		model.ChannelTypePrivate: "private",
+		model.ChannelTypeDirect:  "direct",
+		model.ChannelTypeGroup:   "group",
+	}[channelType]
+}

server/enterprise/message_export/common_export/common_export_test.go

@@ -0,0 +1,237 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common_export
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/mattermost/mattermost/server/public/model"
+)
+
+func TestUpdateMetadata(t *testing.T) {
+	metadata := Metadata{
+		Channels:         map[string]MetadataChannel{},
+		MessagesCount:    0,
+		AttachmentsCount: 0,
+		StartTime:        0,
+		EndTime:          0,
+	}
+
+	testString := "test"
+	chanTypeDirect := model.ChannelTypeDirect
+	createdAt := int64(12345)
+
+	post := model.MessageExport{
+		TeamId:          &testString,
+		TeamName:        &testString,
+		TeamDisplayName: &testString,
+
+		ChannelId:          &testString,
+		ChannelName:        &testString,
+		ChannelDisplayName: &testString,
+		ChannelType:        &chanTypeDirect,
+
+		UserId:    &testString,
+		UserEmail: &testString,
+		Username:  &testString,
+
+		PostId:         &testString,
+		PostCreateAt:   &createdAt,
+		PostMessage:    &testString,
+		PostType:       &testString,
+		PostOriginalId: &testString,
+		PostFileIds:    []string{},
+	}
+	metadata.Update(&post, 2)
+
+	assert.Len(t, metadata.Channels, 1)
+	assert.Equal(t, 1, metadata.Channels["test"].MessagesCount)
+	assert.Equal(t, 2, metadata.Channels["test"].AttachmentsCount)
+	assert.Equal(t, 1, metadata.MessagesCount)
+	assert.Equal(t, 2, metadata.AttachmentsCount)
+
+	metadata.Update(&post, 2)
+
+	assert.Len(t, metadata.Channels, 1)
+	assert.Equal(t, 2, metadata.Channels["test"].MessagesCount)
+	assert.Equal(t, 4, metadata.Channels["test"].AttachmentsCount)
+	assert.Equal(t, 2, metadata.MessagesCount)
+	assert.Equal(t, 4, metadata.AttachmentsCount)
+
+	testString2 := "test2"
+	post.ChannelId = &testString2
+
+	metadata.Update(&post, 2)
+
+	assert.Len(t, metadata.Channels, 2)
+	assert.Equal(t, 2, metadata.Channels["test"].MessagesCount)
+	assert.Equal(t, 4, metadata.Channels["test"].AttachmentsCount)
+	assert.Equal(t, 1, metadata.Channels["test2"].MessagesCount)
+	assert.Equal(t, 2, metadata.Channels["test2"].AttachmentsCount)
+	assert.Equal(t, 3, metadata.MessagesCount)
+	assert.Equal(t, 6, metadata.AttachmentsCount)
+}
+
+func TestGetJoinsAndLeavesForChannel(t *testing.T) {
+	channel := MetadataChannel{
+		StartTime:          100,
+		EndTime:            200,
+		ChannelId:          "good-request-1",
+		TeamId:             model.NewPointer("test"),
+		TeamName:           model.NewPointer("test"),
+		TeamDisplayName:    model.NewPointer("test"),
+		ChannelName:        "test",
+		ChannelDisplayName: "test",
+		ChannelType:        "O",
+	}
+
+	tt := []struct {
+		name           string
+		channel        MetadataChannel
+		membersHistory []*model.ChannelMemberHistoryResult
+		usersInPosts   ChannelMembers
+		expectedJoins  int
+		expectedLeaves int
+	}{
+		{
+			name:           "no-joins-no-leaves",
+			channel:        channel,
+			membersHistory: nil,
+			usersInPosts:   nil,
+			expectedJoins:  0,
+			expectedLeaves: 0,
+		},
+		{
+			name:    "joins-and-leaves-outside-the-range",
+			channel: channel,
+			membersHistory: []*model.ChannelMemberHistoryResult{
+				{JoinTime: 1, LeaveTime: model.NewPointer(int64(10)), UserId: "test", UserEmail: "test", Username: "test"},
+				{JoinTime: 250, LeaveTime: model.NewPointer(int64(260)), UserId: "test", UserEmail: "test", Username: "test"},
+				{JoinTime: 300, UserId: "test", UserEmail: "test", Username: "test"},
+			},
+			usersInPosts:   nil,
+			expectedJoins:  0,
+			expectedLeaves: 0,
+		},
+		{
+			name:    "join-and-leave-during-the-range",
+			channel: channel,
+			membersHistory: []*model.ChannelMemberHistoryResult{
+				{JoinTime: 100, LeaveTime: model.NewPointer(int64(150)), UserId: "test", UserEmail: "test", Username: "test"},
+			},
+			usersInPosts:   nil,
+			expectedJoins:  1,
+			expectedLeaves: 1,
+		},
+		{
+			name:    "join-during-and-leave-after-the-range",
+			channel: channel,
+			membersHistory: []*model.ChannelMemberHistoryResult{
+				{JoinTime: 150, LeaveTime: model.NewPointer(int64(300)), UserId: "test", UserEmail: "test", Username: "test"},
+			},
+			usersInPosts:   nil,
+			expectedJoins:  1,
+			expectedLeaves: 0,
+		},
+		{
+			name:    "join-before-and-leave-during-the-range",
+			channel: channel,
+			membersHistory: []*model.ChannelMemberHistoryResult{
+				{JoinTime: 99, LeaveTime: model.NewPointer(int64(150)), UserId: "test", UserEmail: "test", Username: "test"},
+			},
+			usersInPosts:   nil,
+			expectedJoins:  1,
+			expectedLeaves: 1,
+		},
+		{
+			name:    "join-before-and-leave-after-the-range",
+			channel: channel,
+			membersHistory: []*model.ChannelMemberHistoryResult{
+				{JoinTime: 99, LeaveTime: model.NewPointer(int64(350)), UserId: "test", UserEmail: "test", Username: "test"},
+			},
+			usersInPosts:   nil,
+			expectedJoins:  1,
+			expectedLeaves: 0,
+		},
+		{
+			name:           "implicit-joins",
+			channel:        channel,
+			membersHistory: nil,
+			usersInPosts: ChannelMembers{
+				"test1": ChannelMember{UserId: "test1", Email: "test1", Username: "test1"},
+				"test2": ChannelMember{UserId: "test2", Email: "test2", Username: "test2"},
+			},
+			expectedJoins:  2,
+			expectedLeaves: 0,
+		},
+		{
+			name:    "implicit-joins-with-explicit-joins",
+			channel: channel,
+			membersHistory: []*model.ChannelMemberHistoryResult{
+				{JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test1", UserEmail: "test1", Username: "test1"},
+				{JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test3", UserEmail: "test3", Username: "test3"},
+			},
+			usersInPosts: ChannelMembers{
+				"test1": ChannelMember{UserId: "test1", Email: "test1", Username: "test1"},
+				"test2": ChannelMember{UserId: "test2", Email: "test2", Username: "test2"},
+			},
+			expectedJoins:  3,
+			expectedLeaves: 2,
+		},
+		{
+			name:    "join-leave-and-join-again",
+			channel: channel,
+			membersHistory: []*model.ChannelMemberHistoryResult{
+				{JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test1", UserEmail: "test1", Username: "test1"},
+				{JoinTime: 160, LeaveTime: model.NewPointer(int64(180)), UserId: "test1", UserEmail: "test1", Username: "test1"},
+			},
+			usersInPosts:   nil,
+			expectedJoins:  2,
+			expectedLeaves: 2,
+		},
+		{
+			name:    "deactivated-members-dont-show",
+			channel: channel,
+			membersHistory: []*model.ChannelMemberHistoryResult{
+				{JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 50},
+				{JoinTime: 160, LeaveTime: model.NewPointer(int64(180)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 50},
+			},
+			usersInPosts:   nil,
+			expectedJoins:  0,
+			expectedLeaves: 0,
+		},
+		{
+			name:    "deactivated-members-show-if-deleted-after-latest-export",
+			channel: channel,
+			membersHistory: []*model.ChannelMemberHistoryResult{
+				{JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 150},
+				{JoinTime: 160, LeaveTime: model.NewPointer(int64(180)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 150},
+			},
+			usersInPosts:   nil,
+			expectedJoins:  2,
+			expectedLeaves: 2,
+		},
+		{
+			name:    "deactivated-members-show-and-dont-show",
+			channel: channel,
+			membersHistory: []*model.ChannelMemberHistoryResult{
+				{JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 50},
+				{JoinTime: 160, LeaveTime: model.NewPointer(int64(180)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 150},
+			},
+			usersInPosts:   nil,
+			expectedJoins:  1,
+			expectedLeaves: 1,
+		},
+	}
+
+	for _, tc := range tt {
+		t.Run(tc.name, func(t *testing.T) {
+			joins, leaves := GetJoinsAndLeavesForChannel(tc.channel.StartTime, tc.channel.EndTime, tc.membersHistory, tc.usersInPosts)
+			assert.Len(t, joins, tc.expectedJoins)
+			assert.Len(t, leaves, tc.expectedLeaves)
+		})
+	}
+}

server/enterprise/message_export/csv_export/csv_export.go

@@ -0,0 +1,410 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package csv_export
+
+import (
+	"archive/zip"
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"sort"
+	"strconv"
+
+	"github.com/mattermost/mattermost/server/v8/enterprise/message_export/common_export"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+)
+
+const (
+	EnterPostType            = "enter"
+	LeavePostType            = "leave"
+	PreviouslyJoinedPostType = "previously-joined"
+	CSVExportFilename        = "csv_export.zip"
+	CSVWarningFilename       = "warning.txt"
+)
+
+func CsvExport(rctx request.CTX, posts []*model.MessageExport, db store.Store, exportBackend filestore.FileBackend, fileAttachmentBackend filestore.FileBackend, exportDirectory string) (warningCount int64, appErr *model.AppError) {
+	dest, err := os.CreateTemp("", CSVExportFilename)
+	if err != nil {
+		return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.file.creation.appError", nil, "", 0).Wrap(err)
+	}
+	defer os.Remove(dest.Name())
+
+	zipFile := zip.NewWriter(dest)
+
+	csvFile, err := zipFile.Create("posts.csv")
+	if err != nil {
+		return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.zip.creation.appError", nil, "", 0).Wrap(err)
+	}
+	csvWriter := csv.NewWriter(csvFile)
+	err = csvWriter.Write([]string{
+		"Post Creation Time",
+		"Team Id",
+		"Team Name",
+		"Team Display Name",
+		"Channel Id",
+		"Channel Name",
+		"Channel Display Name",
+		"Channel Type",
+		"User Id",
+		"User Email",
+		"Username",
+		"Post Id",
+		"Edited By Post Id",
+		"Replied to Post Id",
+		"Post Message",
+		"Post Type",
+		"User Type",
+		"Previews Post Id",
+	})
+
+	if err != nil {
+		return warningCount, model.NewAppError("CsvExportPost", "ent.compliance.csv.header.export.appError", nil, "", 0).Wrap(err)
+	}
+
+	metadata := common_export.Metadata{
+		Channels:         map[string]common_export.MetadataChannel{},
+		MessagesCount:    0,
+		AttachmentsCount: 0,
+		StartTime:        0,
+		EndTime:          0,
+	}
+
+	membersByChannel := make(common_export.MembersByChannel)
+
+	for _, post := range posts {
+		attachments, err := getPostAttachments(db, post)
+		if err != nil {
+			return warningCount, err
+		}
+
+		if _, ok := membersByChannel[*post.ChannelId]; !ok {
+			membersByChannel[*post.ChannelId] = common_export.ChannelMembers{}
+		}
+
+		membersByChannel[*post.ChannelId][*post.UserId] = common_export.ChannelMember{
+			UserId:   *post.UserId,
+			Username: *post.Username,
+			IsBot:    post.IsBot,
+			Email:    *post.UserEmail,
+		}
+
+		metadata.Update(post, len(attachments))
+	}
+
+	joinLeavePosts, appErr2 := getJoinLeavePosts(metadata.Channels, membersByChannel, db)
+	if appErr2 != nil {
+		return warningCount, appErr2
+	}
+
+	postsGenerator := mergePosts(joinLeavePosts, posts)
+
+	for post := postsGenerator(); post != nil; post = postsGenerator() {
+		if err = csvWriter.Write(postToRow(post, post.PostCreateAt, *post.PostMessage)); err != nil {
+			return warningCount, model.NewAppError("CsvExportPost", "ent.compliance.csv.post.export.appError", nil, "", 0).Wrap(err)
+		}
+
+		if post.PostDeleteAt != nil && *post.PostDeleteAt > 0 && post.PostProps != nil {
+			props := map[string]any{}
+			if json.Unmarshal([]byte(*post.PostProps), &props) == nil {
+				if _, ok := props[model.PostPropsDeleteBy]; ok {
+					if err = csvWriter.Write(postToRow(post, post.PostDeleteAt, "delete "+*post.PostMessage)); err != nil {
+						return warningCount, model.NewAppError("CsvExportPost", "ent.compliance.csv.post.export.appError", nil, "", 0).Wrap(err)
+					}
+				}
+			}
+		}
+
+		var attachments []*model.FileInfo
+		attachments, appErr = getPostAttachments(db, post)
+		if appErr != nil {
+			return warningCount, appErr
+		}
+
+		for _, attachment := range attachments {
+			if err = csvWriter.Write(attachmentToRow(post, attachment)); err != nil {
+				return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.attachment.export.appError", nil, "", 0).Wrap(err)
+			}
+		}
+	}
+
+	csvWriter.Flush()
+
+	var missingFiles []string
+	for _, post := range posts {
+		attachments, err := getPostAttachments(db, post)
+		if err != nil {
+			return warningCount, err
+		}
+
+		for _, attachment := range attachments {
+			var attachmentSrc io.ReadCloser
+			attachmentSrc, nErr := fileAttachmentBackend.Reader(attachment.Path)
+			if nErr != nil {
+				missingFiles = append(missingFiles, "Warning:"+common_export.MissingFileMessage+" - Post: "+*post.PostId+" - "+attachment.Path)
+				rctx.Logger().Warn(common_export.MissingFileMessage, mlog.String("PostId", *post.PostId), mlog.String("FileName", attachment.Path))
+				continue
+			}
+			defer attachmentSrc.Close()
+
+			attachmentDst, err := zipFile.Create(path.Join("files", *post.PostId, fmt.Sprintf("%s-%s", attachment.Id, path.Base(attachment.Path))))
+			if err != nil {
+				return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.attachment.copy.appError", nil, "", 0).Wrap(err)
+			}
+
+			_, err = io.Copy(attachmentDst, attachmentSrc)
+			if err != nil {
+				return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.attachment.copy.appError", nil, "", 0).Wrap(err)
+			}
+		}
+	}
+
+	warningCount = int64(len(missingFiles))
+	if warningCount > 0 {
+		metadataFile, _ := zipFile.Create(CSVWarningFilename)
+		for _, value := range missingFiles {
+			_, err = metadataFile.Write([]byte(value + "\n"))
+			if err != nil {
+				appErr = model.NewAppError("CsvExport", "ent.compliance.csv.warning.appError", nil, "", 0).Wrap(err)
+				return warningCount, appErr
+			}
+		}
+	}
+
+	metadataFile, err := zipFile.Create("metadata.json")
+	if err != nil {
+		return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.metadata.json.zipfile.appError", nil, "", 0).Wrap(err)
+	}
+	data, err := json.MarshalIndent(metadata, "", "  ")
+	if err != nil {
+		return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.metadata.json.marshalling.appError", nil, "", 0).Wrap(err)
+	}
+	_, err = metadataFile.Write(data)
+	if err != nil {
+		return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.metadata.export.appError", nil, "", 0).Wrap(err)
+	}
+	err = zipFile.Close()
+	if err != nil {
+		return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.metadata.close.appError", nil, "", 0).Wrap(err)
+	}
+
+	_, err = dest.Seek(0, 0)
+	if err != nil {
+		return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.seek.appError", nil, "", 0).Wrap(err)
+	}
+	// Try to write the file without a timeout due to the potential size of the file.
+	_, err = filestore.TryWriteFileContext(rctx.Context(), exportBackend, dest, path.Join(exportDirectory, CSVExportFilename))
+	if err != nil {
+		return warningCount, model.NewAppError("CsvExport", "ent.compliance.csv.write_file.appError", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return warningCount, appErr
+}
+
+func mergePosts(left []*model.MessageExport, right []*model.MessageExport) func() *model.MessageExport {
+	leftCursor := 0
+	rightCursor := 0
+	return func() *model.MessageExport {
+		if leftCursor >= len(left) && rightCursor >= len(right) {
+			return nil
+		}
+
+		if leftCursor >= len(left) {
+			rightCursor += 1
+			return right[rightCursor-1]
+		}
+
+		if rightCursor >= len(right) {
+			leftCursor += 1
+			return left[leftCursor-1]
+		}
+
+		if *left[leftCursor].PostCreateAt <= *right[rightCursor].PostCreateAt {
+			leftCursor += 1
+			return left[leftCursor-1]
+		}
+
+		rightCursor += 1
+		return right[rightCursor-1]
+	}
+}
+
+func getJoinLeavePosts(channels map[string]common_export.MetadataChannel, membersByChannel common_export.MembersByChannel, db store.Store) ([]*model.MessageExport, *model.AppError) {
+	joinLeavePosts := []*model.MessageExport{}
+	for _, channel := range channels {
+		channelMembersHistory, err := db.ChannelMemberHistory().GetUsersInChannelDuring(channel.StartTime, channel.EndTime, channel.ChannelId)
+		if err != nil {
+			return nil, model.NewAppError("getJoinLeavePosts", "ent.get_users_in_channel_during", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+
+		joins, leaves := common_export.GetJoinsAndLeavesForChannel(channel.StartTime, channel.EndTime, channelMembersHistory, membersByChannel[channel.ChannelId])
+
+		for _, join := range joins {
+			enterMessage := fmt.Sprintf("User %s (%s) joined the channel", join.Username, join.Email)
+			enterPostType := EnterPostType
+			createAt := model.NewPointer(join.Datetime)
+			channelCopy := channel
+			if join.Datetime <= channel.StartTime {
+				enterPostType = PreviouslyJoinedPostType
+				enterMessage = fmt.Sprintf("User %s (%s) was already in the channel", join.Username, join.Email)
+				createAt = model.NewPointer(channel.StartTime)
+			}
+			joinLeavePosts = append(
+				joinLeavePosts,
+				&model.MessageExport{
+					TeamId:          channel.TeamId,
+					TeamName:        channel.TeamName,
+					TeamDisplayName: channel.TeamDisplayName,
+
+					ChannelId:          &channelCopy.ChannelId,
+					ChannelName:        &channelCopy.ChannelName,
+					ChannelDisplayName: &channelCopy.ChannelDisplayName,
+					ChannelType:        &channelCopy.ChannelType,
+
+					UserId:    model.NewPointer(join.UserId),
+					UserEmail: model.NewPointer(join.Email),
+					Username:  model.NewPointer(join.Username),
+					IsBot:     join.IsBot,
+
+					PostId:         model.NewPointer(""),
+					PostCreateAt:   createAt,
+					PostMessage:    &enterMessage,
+					PostType:       &enterPostType,
+					PostOriginalId: model.NewPointer(""),
+					PostFileIds:    []string{},
+				},
+			)
+		}
+		for _, leave := range leaves {
+			leaveMessage := fmt.Sprintf("User %s (%s) leaved the channel", leave.Username, leave.Email)
+			leavePostType := LeavePostType
+			channelCopy := channel
+
+			joinLeavePosts = append(
+				joinLeavePosts,
+				&model.MessageExport{
+					TeamId:          channel.TeamId,
+					TeamName:        channel.TeamName,
+					TeamDisplayName: channel.TeamDisplayName,
+
+					ChannelId:          &channelCopy.ChannelId,
+					ChannelName:        &channelCopy.ChannelName,
+					ChannelDisplayName: &channelCopy.ChannelDisplayName,
+					ChannelType:        &channelCopy.ChannelType,
+
+					UserId:    model.NewPointer(leave.UserId),
+					UserEmail: model.NewPointer(leave.Email),
+					Username:  model.NewPointer(leave.Username),
+					IsBot:     leave.IsBot,
+
+					PostId:         model.NewPointer(""),
+					PostCreateAt:   model.NewPointer(leave.Datetime),
+					PostMessage:    &leaveMessage,
+					PostType:       &leavePostType,
+					PostOriginalId: model.NewPointer(""),
+					PostFileIds:    []string{},
+				},
+			)
+		}
+	}
+
+	sort.Slice(joinLeavePosts, func(i, j int) bool {
+		return *joinLeavePosts[i].PostCreateAt < *joinLeavePosts[j].PostCreateAt
+	})
+	return joinLeavePosts, nil
+}
+
+func getPostAttachments(db store.Store, post *model.MessageExport) ([]*model.FileInfo, *model.AppError) {
+	// if the post included any files, we need to add special elements to the export.
+	if len(post.PostFileIds) == 0 {
+		return []*model.FileInfo{}, nil
+	}
+
+	attachments, err := db.FileInfo().GetForPost(*post.PostId, true, true, false)
+	if err != nil {
+		return nil, model.NewAppError("getPostAttachments", "ent.message_export.csv_export.get_attachment_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return attachments, nil
+}
+
+func postToRow(post *model.MessageExport, createTime *int64, message string) []string {
+	teamId := ""
+	teamName := ""
+	teamDisplayName := ""
+	if post.TeamId != nil {
+		teamId = *post.TeamId
+	}
+	if post.TeamName != nil {
+		teamName = *post.TeamName
+	}
+	if post.TeamDisplayName != nil {
+		teamDisplayName = *post.TeamDisplayName
+	}
+	postType := "message"
+	if post.PostType != nil && *post.PostType != "" {
+		postType = *post.PostType
+	}
+	postRootId := ""
+	if post.PostRootId != nil {
+		postRootId = *post.PostRootId
+	}
+	userType := "user"
+	if post.IsBot {
+		userType = "bot"
+	}
+
+	return []string{
+		strconv.FormatInt(*createTime, 10),
+		teamId,
+		teamName,
+		teamDisplayName,
+		*post.ChannelId,
+		*post.ChannelName,
+		*post.ChannelDisplayName,
+		common_export.ChannelTypeDisplayName(*post.ChannelType),
+		*post.UserId,
+		*post.UserEmail,
+		*post.Username,
+		*post.PostId,
+		*post.PostOriginalId,
+		postRootId,
+		message,
+		postType,
+		userType,
+		post.PreviewID(),
+	}
+}
+
+func attachmentToRow(post *model.MessageExport, attachment *model.FileInfo) []string {
+	row := postToRow(post, post.PostCreateAt, *post.PostMessage)
+
+	attachmentEntry := fmt.Sprintf("%s (files/%s/%s-%s)", attachment.Name, *post.PostId, attachment.Id, path.Base(attachment.Path))
+	attachmentMessage := "attachment"
+	userType := row[len(row)-2]
+
+	if attachment.DeleteAt > 0 && post.PostDeleteAt != nil {
+		deleteRow := postToRow(post, post.PostDeleteAt, *post.PostMessage)
+		row = append(
+			deleteRow[:len(deleteRow)-4],
+			attachmentEntry,
+			"deleted "+attachmentMessage,
+			userType,
+		)
+	} else {
+		row = append(
+			row[:len(row)-4],
+			attachmentEntry,
+			attachmentMessage,
+			userType,
+		)
+	}
+	return row
+}

server/enterprise/message_export/global_relay_export/deliver.go

@@ -0,0 +1,110 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package global_relay_export
+
+import (
+	"archive/zip"
+	"context"
+	"io"
+	"net/http"
+	"net/mail"
+	"net/smtp"
+	"os"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+)
+
+func Deliver(export *os.File, config *model.Config) *model.AppError {
+	info, err := export.Stat()
+	if err != nil {
+		return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.unable_to_get_file_info.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	zipFile, err := zip.NewReader(export, info.Size())
+	if err != nil {
+		return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.unable_to_open_zip_file_data.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	to := *config.MessageExportSettings.GlobalRelaySettings.EmailAddress
+	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(ctx, time.Duration(*config.EmailSettings.SMTPServerTimeout)*time.Second)
+	defer cancel()
+
+	conn, err := connectToSMTPServer(ctx, config)
+	if err != nil {
+		return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.unable_to_connect_smtp_server.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	defer conn.Close()
+
+	mailsCount := 0
+	for _, mail := range zipFile.File {
+		from, err := getFrom(mail)
+		if err != nil {
+			return err
+		}
+		if err := deliverEmail(conn, mail, from, to); err != nil {
+			return err
+		}
+
+		mailsCount++
+		if mailsCount == MaxEmailsPerConnection {
+			mailsCount = 0
+			conn.Close()
+
+			var nErr error
+			conn, nErr = connectToSMTPServer(context.Background(), config)
+			if nErr != nil {
+				return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.unable_to_connect_smtp_server.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
+			}
+		}
+	}
+	return nil
+}
+
+func deliverEmail(c *smtp.Client, mailFile *zip.File, from string, to string) *model.AppError {
+	mailData, err := mailFile.Open()
+	if err != nil {
+		return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.unable_to_open_email_file.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	defer mailData.Close()
+
+	err = c.Mail(from)
+	if err != nil {
+		return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.from_address.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	err = c.Rcpt(to)
+	if err != nil {
+		return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.to_address.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	w, err := c.Data()
+	if err != nil {
+		return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.msg_data.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	_, err = io.Copy(w, mailData)
+	if err != nil {
+		return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.msg.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	err = w.Close()
+	if err != nil {
+		return model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.close.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func getFrom(mailFile *zip.File) (string, *model.AppError) {
+	mailData, err := mailFile.Open()
+	if err != nil {
+		return "", model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.unable_to_open_email_file.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	defer mailData.Close()
+
+	message, err := mail.ReadMessage(mailData)
+	if err != nil {
+		return "", model.NewAppError("GlobalRelayDelivery", "ent.message_export.global_relay_export.deliver.parse_mail.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return message.Header.Get("From"), nil
+}

server/enterprise/message_export/global_relay_export/deliver_test.go

@@ -0,0 +1,140 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package global_relay_export
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	gomail "gopkg.in/mail.v2"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/mail"
+)
+
+func TestDeliver(t *testing.T) {
+	config := &model.Config{}
+	config.SetDefaults()
+	config.MessageExportSettings.GlobalRelaySettings.CustomerType = model.NewPointer("INBUCKET")
+	config.MessageExportSettings.GlobalRelaySettings.EmailAddress = model.NewPointer("test-globalrelay-mailbox@test")
+
+	t.Run("Testing invalid zip file", func(t *testing.T) {
+		emptyFile, err := os.CreateTemp("", "export")
+		require.NoError(t, err)
+		defer emptyFile.Close()
+		defer os.Remove(emptyFile.Name())
+
+		appErr := Deliver(emptyFile, config)
+		assert.NotNil(t, appErr)
+	})
+
+	t.Run("Testing empty zip file", func(t *testing.T) {
+		emptyZipFile, err := os.CreateTemp("", "export")
+		require.NoError(t, err)
+		zipFile := zip.NewWriter(emptyZipFile)
+		err = zipFile.Close()
+		require.NoError(t, err)
+		defer emptyZipFile.Close()
+		defer os.Remove(emptyZipFile.Name())
+
+		err = mail.DeleteMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
+		require.NoError(t, err)
+
+		appErr := Deliver(emptyZipFile, config)
+		assert.Nil(t, appErr)
+
+		_, err = mail.GetMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
+		require.Error(t, err)
+	})
+
+	t.Run("Testing zip file with one email", func(t *testing.T) {
+		headers := map[string][]string{
+			"From":                       {"test@test.com"},
+			"To":                         {*config.MessageExportSettings.GlobalRelaySettings.EmailAddress},
+			"Subject":                    {encodeRFC2047Word("test")},
+			"Content-Transfer-Encoding":  {"8bit"},
+			"Auto-Submitted":             {"auto-generated"},
+			"Precedence":                 {"bulk"},
+			GlobalRelayMsgTypeHeader:     {"Mattermost"},
+			GlobalRelayChannelNameHeader: {encodeRFC2047Word("test")},
+			GlobalRelayChannelIDHeader:   {encodeRFC2047Word("test")},
+			GlobalRelayChannelTypeHeader: {encodeRFC2047Word("test")},
+		}
+
+		m := gomail.NewMessage(gomail.SetCharset("UTF-8"))
+		m.SetHeaders(headers)
+		m.SetBody("text/plain", "test")
+
+		emptyZipFile, err := os.CreateTemp("", "export")
+		require.NoError(t, err)
+		zipFile := zip.NewWriter(emptyZipFile)
+		file, err := zipFile.Create("test")
+		require.NoError(t, err)
+		_, err = m.WriteTo(file)
+		require.NoError(t, err)
+
+		err = zipFile.Close()
+		require.NoError(t, err)
+		defer emptyZipFile.Close()
+		defer os.Remove(emptyZipFile.Name())
+
+		err = mail.DeleteMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
+		require.NoError(t, err)
+
+		appErr := Deliver(emptyZipFile, config)
+		assert.Nil(t, appErr)
+
+		mailbox, err := mail.GetMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
+		require.NoError(t, err)
+		require.Len(t, mailbox, 1)
+	})
+
+	t.Run("Testing zip file with 50 emails", func(t *testing.T) {
+		headers := map[string][]string{
+			"From":                       {"test@test.com"},
+			"To":                         {*config.MessageExportSettings.GlobalRelaySettings.EmailAddress},
+			"Subject":                    {encodeRFC2047Word("test")},
+			"Content-Transfer-Encoding":  {"8bit"},
+			"Auto-Submitted":             {"auto-generated"},
+			"Precedence":                 {"bulk"},
+			GlobalRelayMsgTypeHeader:     {"Mattermost"},
+			GlobalRelayChannelNameHeader: {encodeRFC2047Word("test")},
+			GlobalRelayChannelIDHeader:   {encodeRFC2047Word("test")},
+			GlobalRelayChannelTypeHeader: {encodeRFC2047Word("test")},
+		}
+		m := gomail.NewMessage(gomail.SetCharset("UTF-8"))
+		m.SetHeaders(headers)
+		m.SetBody("text/plain", "test")
+
+		emptyZipFile, err := os.CreateTemp("", "export")
+		require.NoError(t, err)
+		zipFile := zip.NewWriter(emptyZipFile)
+		for x := 0; x < 50; x++ {
+			var file io.Writer
+			file, err = zipFile.Create(fmt.Sprintf("test-%d", x))
+			require.NoError(t, err)
+			_, err = m.WriteTo(file)
+			require.NoError(t, err)
+		}
+		err = zipFile.Close()
+		require.NoError(t, err)
+		defer emptyZipFile.Close()
+		defer os.Remove(emptyZipFile.Name())
+
+		err = mail.DeleteMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
+		require.NoError(t, err)
+
+		appErr := Deliver(emptyZipFile, config)
+		assert.Nil(t, appErr)
+
+		mailbox, err := mail.GetMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
+		require.NoError(t, err)
+		require.Len(t, mailbox, 50)
+	})
+}

server/enterprise/message_export/global_relay_export/global_relay_export.go

@@ -0,0 +1,420 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package global_relay_export
+
+import (
+	"archive/zip"
+	"encoding/json"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/jaytaylor/html2text"
+	gomail "gopkg.in/mail.v2"
+
+	"github.com/mattermost/mattermost/server/v8/enterprise/message_export/common_export"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/templates"
+)
+
+const (
+	GlobalRelayMsgTypeHeader     = "X-GlobalRelay-MsgType"
+	GlobalRelayChannelNameHeader = "X-Mattermost-ChannelName"
+	GlobalRelayChannelIDHeader   = "X-Mattermost-ChannelID"
+	GlobalRelayChannelTypeHeader = "X-Mattermost-ChannelType"
+	MaxEmailBytes                = 250 << (10 * 2)
+	MaxEmailsPerConnection       = 400
+)
+
+type AllExport map[string][]*ChannelExport
+
+type ChannelExport struct {
+	ChannelId       string            // the unique id of the channel
+	ChannelName     string            // the name of the channel
+	ChannelType     model.ChannelType // the channel type
+	StartTime       int64             // utc timestamp (seconds), start of export period or create time of channel, whichever is greater. Example: 1366611728.
+	EndTime         int64             // utc timestamp (seconds), end of export period or delete time of channel, whichever is lesser. Example: 1366611728.
+	Participants    []ParticipantRow  // summary information about the conversation participants
+	Messages        []Message         // the messages that were sent during the conversation
+	ExportedOn      int64             // utc timestamp (seconds), when this export was generated
+	numUserMessages map[string]int    // key is user id, value is number of messages that they sent during this period
+	uploadedFiles   []*model.FileInfo // any files that were uploaded to the channel during the export period
+	bytes           int64
+}
+
+// a row in the summary table at the top of the export
+type ParticipantRow struct {
+	Username     string
+	UserType     string
+	Email        string
+	JoinTime     int64
+	LeaveTime    int64
+	MessagesSent int
+}
+
+type Message struct {
+	SentTime       int64
+	SenderUsername string
+	PostType       string
+	PostUsername   string
+	SenderUserType string
+	SenderEmail    string
+	Message        string
+	PreviewsPost   string
+}
+
+func GlobalRelayExport(rctx request.CTX, posts []*model.MessageExport, db store.Store, fileAttachmentBackend filestore.FileBackend, dest io.Writer, templates *templates.Container) ([]string, int64, *model.AppError) {
+	var warningCount int64
+	attachmentsRemovedPostIDs := []string{}
+	allExports := make(map[string][]*ChannelExport)
+
+	zipFile := zip.NewWriter(dest)
+
+	membersByChannel := common_export.MembersByChannel{}
+	metadata := common_export.Metadata{
+		Channels:         map[string]common_export.MetadataChannel{},
+		MessagesCount:    0,
+		AttachmentsCount: 0,
+		StartTime:        0,
+		EndTime:          0,
+	}
+
+	for _, post := range posts {
+		if _, ok := membersByChannel[*post.ChannelId]; !ok {
+			membersByChannel[*post.ChannelId] = common_export.ChannelMembers{}
+		}
+
+		membersByChannel[*post.ChannelId][*post.UserId] = common_export.ChannelMember{
+			UserId:   *post.UserId,
+			Username: *post.Username,
+			IsBot:    post.IsBot,
+			Email:    *post.UserEmail,
+		}
+
+		attachments := []*model.FileInfo{}
+		if len(post.PostFileIds) > 0 {
+			var err error
+			attachments, err = db.FileInfo().GetForPost(*post.PostId, true, true, false)
+			if err != nil {
+				return nil, warningCount, model.NewAppError("GlobalRelayExport", "ent.message_export.global_relay_export.get_attachment_error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+		}
+
+		attachmentsRemoved := addToExports(rctx, attachments, allExports, post)
+		attachmentsRemovedPostIDs = append(attachmentsRemovedPostIDs, attachmentsRemoved...)
+
+		metadata.Update(post, len(attachments))
+	}
+
+	for _, channelExportList := range allExports {
+		for batchId, channelExport := range channelExportList {
+			participants, appErr := getParticipants(db, channelExport, membersByChannel[channelExport.ChannelId])
+			if appErr != nil {
+				return nil, warningCount, appErr
+			}
+			channelExport.Participants = participants
+			channelExport.ExportedOn = time.Now().Unix() * 1000
+
+			channelExportFile, err := zipFile.Create(fmt.Sprintf("%s - (%s) - %d.eml", channelExport.ChannelName, channelExport.ChannelId, batchId))
+			if err != nil {
+				return nil, warningCount, model.NewAppError("GlobalRelayExport", "ent.message_export.global_relay.create_file_in_zip.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+
+			if appErr, warningCount = generateEmail(rctx, fileAttachmentBackend, channelExport, templates, channelExportFile); appErr != nil {
+				return nil, warningCount, appErr
+			}
+		}
+	}
+
+	err := zipFile.Close()
+	if err != nil {
+		return nil, warningCount, model.NewAppError("GlobalRelayExport", "ent.message_export.global_relay.close_zip_file.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	return attachmentsRemovedPostIDs, warningCount, nil
+}
+
+func addToExports(rctx request.CTX, attachments []*model.FileInfo, exports map[string][]*ChannelExport, post *model.MessageExport) []string {
+	var channelExport *ChannelExport
+	attachmentsRemovedPostIDs := []string{}
+	if channelExports, present := exports[*post.ChannelId]; !present {
+		// we found a new channel
+		channelExport = &ChannelExport{
+			ChannelId:       *post.ChannelId,
+			ChannelName:     *post.ChannelDisplayName,
+			ChannelType:     *post.ChannelType,
+			StartTime:       *post.PostCreateAt,
+			Messages:        make([]Message, 0),
+			Participants:    make([]ParticipantRow, 0),
+			numUserMessages: make(map[string]int),
+			uploadedFiles:   make([]*model.FileInfo, 0),
+			bytes:           0,
+		}
+		exports[*post.ChannelId] = []*ChannelExport{channelExport}
+	} else {
+		// we already know about this channel
+		channelExport = channelExports[len(channelExports)-1]
+	}
+
+	// Create a new ChannelExport if it would be too many bytes to add the post
+	fileBytes := fileInfoListBytes(attachments)
+	msgBytes := int64(len(*post.PostMessage))
+	postBytes := fileBytes + msgBytes
+	postTooLargeForChannelBatch := channelExport.bytes+postBytes > MaxEmailBytes
+	postAloneTooLargeToSend := postBytes > MaxEmailBytes // Attachments must be removed from export, they're too big to send.
+
+	if postAloneTooLargeToSend {
+		attachmentsRemovedPostIDs = append(attachmentsRemovedPostIDs, *post.PostId)
+	}
+
+	if postTooLargeForChannelBatch && !postAloneTooLargeToSend {
+		channelExport = &ChannelExport{
+			ChannelId:       *post.ChannelId,
+			ChannelName:     *post.ChannelDisplayName,
+			ChannelType:     *post.ChannelType,
+			StartTime:       *post.PostCreateAt,
+			Messages:        make([]Message, 0),
+			Participants:    make([]ParticipantRow, 0),
+			numUserMessages: make(map[string]int),
+			uploadedFiles:   make([]*model.FileInfo, 0),
+			bytes:           0,
+		}
+		exports[*post.ChannelId] = append(exports[*post.ChannelId], channelExport)
+	}
+
+	addPostToChannelExport(rctx, channelExport, post)
+
+	// if this post includes files, add them to the collection
+	for _, fileInfo := range attachments {
+		addAttachmentToChannelExport(channelExport, post, fileInfo, postAloneTooLargeToSend)
+	}
+	channelExport.bytes += postBytes
+	return attachmentsRemovedPostIDs
+}
+
+func getParticipants(db store.Store, channelExport *ChannelExport, members common_export.ChannelMembers) ([]ParticipantRow, *model.AppError) {
+	participantsMap := map[string]ParticipantRow{}
+	channelMembersHistory, err := db.ChannelMemberHistory().GetUsersInChannelDuring(channelExport.StartTime, channelExport.EndTime, channelExport.ChannelId)
+	if err != nil {
+		return nil, model.NewAppError("getParticipants", "ent.get_users_in_channel_during", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	joins, leaves := common_export.GetJoinsAndLeavesForChannel(channelExport.StartTime, channelExport.EndTime, channelMembersHistory, members)
+
+	for _, join := range joins {
+		userType := "user"
+		if join.IsBot {
+			userType = "bot"
+		}
+
+		if _, ok := participantsMap[join.UserId]; !ok {
+			participantsMap[join.UserId] = ParticipantRow{
+				Username:     join.Username,
+				UserType:     userType,
+				Email:        join.Email,
+				JoinTime:     join.Datetime,
+				LeaveTime:    channelExport.EndTime,
+				MessagesSent: channelExport.numUserMessages[join.UserId],
+			}
+		}
+	}
+	for _, leave := range leaves {
+		if participantRow, ok := participantsMap[leave.UserId]; ok {
+			participantRow.LeaveTime = leave.Datetime //nolint:govet
+		}
+	}
+
+	participants := []ParticipantRow{}
+	for _, participant := range participantsMap {
+		participants = append(participants, participant)
+	}
+
+	sort.Slice(participants, func(i, j int) bool {
+		return participants[i].Username < participants[j].Username
+	})
+	return participants, nil
+}
+
+func generateEmail(rctx request.CTX, fileAttachmentBackend filestore.FileBackend, channelExport *ChannelExport, templates *templates.Container, w io.Writer) (*model.AppError, int64) {
+	var warningCount int64
+	participantEmailAddresses := getParticipantEmails(channelExport)
+
+	// GlobalRelay expects the email to come from the person that initiated the conversation.
+	// our conversations aren't really initiated, so we just use the first person we find
+	from := participantEmailAddresses[0]
+
+	// it also expects the email to be addressed to the other participants in the conversation
+	mimeTo := strings.Join(participantEmailAddresses, ",")
+
+	htmlBody, err := channelExportToHTML(rctx, channelExport, templates)
+	if err != nil {
+		return model.NewAppError("GlobalRelayExport", "ent.message_export.global_relay.generate_email.app_error", nil, "", http.StatusInternalServerError).Wrap(err), warningCount
+	}
+
+	subject := fmt.Sprintf("Mattermost Compliance Export: %s", channelExport.ChannelName)
+	htmlMessage := "\r\n<html><body>" + htmlBody + "</body></html>"
+
+	txtBody, err := html2text.FromString(htmlBody)
+	if err != nil {
+		rctx.Logger().Warn("Error transforming html to plain text for GlobalRelay email", mlog.Err(err))
+		txtBody = ""
+	}
+
+	headers := map[string][]string{
+		"From":                       {from},
+		"To":                         {mimeTo},
+		"Subject":                    {encodeRFC2047Word(subject)},
+		"Content-Transfer-Encoding":  {"8bit"},
+		"Auto-Submitted":             {"auto-generated"},
+		"Precedence":                 {"bulk"},
+		GlobalRelayMsgTypeHeader:     {"Mattermost"},
+		GlobalRelayChannelNameHeader: {encodeRFC2047Word(channelExport.ChannelName)},
+		GlobalRelayChannelIDHeader:   {encodeRFC2047Word(channelExport.ChannelId)},
+		GlobalRelayChannelTypeHeader: {encodeRFC2047Word(common_export.ChannelTypeDisplayName(channelExport.ChannelType))},
+	}
+
+	m := gomail.NewMessage(gomail.SetCharset("UTF-8"))
+	m.SetHeaders(headers)
+	m.SetDateHeader("Date", time.Unix(channelExport.EndTime/1000, 0).UTC())
+	m.SetBody("text/plain", txtBody)
+	m.AddAlternative("text/html", htmlMessage)
+
+	for _, fileInfo := range channelExport.uploadedFiles {
+		path := fileInfo.Path
+
+		m.Attach(fileInfo.Name, gomail.SetCopyFunc(func(writer io.Writer) error {
+			reader, appErr := fileAttachmentBackend.Reader(path)
+			if appErr != nil {
+				rctx.Logger().Warn("File not found for export", mlog.String("Filename", path))
+				warningCount += 1
+				return nil
+			}
+			defer reader.Close()
+
+			_, err = io.Copy(writer, reader)
+			if err != nil {
+				return model.NewAppError("GlobalRelayExport", "ent.message_export.global_relay.attach_file.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+			return nil
+		}))
+	}
+
+	_, err = m.WriteTo(w)
+	if err != nil {
+		return model.NewAppError("GlobalRelayExport", "ent.message_export.global_relay.generate_email.app_error", nil, "", http.StatusInternalServerError).Wrap(err), warningCount
+	}
+	return nil, warningCount
+}
+
+func getParticipantEmails(channelExport *ChannelExport) []string {
+	participantEmails := make([]string, len(channelExport.Participants))
+	for i, participant := range channelExport.Participants {
+		participantEmails[i] = participant.Email
+	}
+	return participantEmails
+}
+
+func fileInfoListBytes(fileInfoList []*model.FileInfo) int64 {
+	totalBytes := int64(0)
+	for _, fileInfo := range fileInfoList {
+		totalBytes += fileInfo.Size
+	}
+	return totalBytes
+}
+
+func addPostToChannelExport(rctx request.CTX, channelExport *ChannelExport, post *model.MessageExport) {
+	userType := "user"
+	if post.IsBot {
+		userType = "bot"
+	}
+
+	strPostProps := post.PostProps
+	bytPostProps := []byte(*strPostProps)
+
+	// Added to show the username if overridden by a webhook or API integration
+	postUserName := ""
+	var postPropsLocal map[string]any
+	err := json.Unmarshal(bytPostProps, &postPropsLocal)
+	if err != nil {
+		rctx.Logger().Warn("Failed to unmarshal post Props into JSON. Ignoring username override.", mlog.Err(err))
+	} else {
+		if overrideUsername, ok := postPropsLocal["override_username"]; ok {
+			postUserName = overrideUsername.(string)
+		}
+
+		if postUserName == "" {
+			if overrideUsername, ok := postPropsLocal["webhook_display_name"]; ok {
+				postUserName = overrideUsername.(string)
+			}
+		}
+	}
+
+	element := Message{
+		SentTime:       *post.PostCreateAt,
+		Message:        *post.PostMessage,
+		SenderUserType: userType,
+		PostType:       *post.PostType,
+		PostUsername:   postUserName,
+		SenderUsername: *post.Username,
+		SenderEmail:    *post.UserEmail,
+		PreviewsPost:   post.PreviewID(),
+	}
+	channelExport.Messages = append(channelExport.Messages, element)
+	channelExport.EndTime = *post.PostCreateAt
+	channelExport.numUserMessages[*post.UserId] += 1
+}
+
+func addAttachmentToChannelExport(channelExport *ChannelExport, post *model.MessageExport, fileInfo *model.FileInfo, removeAttachments bool) {
+	var uploadElement Message
+	userType := "user"
+	if post.IsBot {
+		userType = "bot"
+	}
+	if removeAttachments {
+		// add "post" message indicating that attachments were not sent
+		uploadElement = Message{
+			SentTime:       fileInfo.CreateAt,
+			Message:        fmt.Sprintf("Uploaded file '%s' (id '%s') was removed because it was too large to send.", fileInfo.Name, fileInfo.Id),
+			SenderUsername: *post.Username,
+			SenderUserType: userType,
+			SenderEmail:    *post.UserEmail,
+		}
+
+		if fileInfo.DeleteAt != 0 {
+			uploadElement.SentTime = fileInfo.DeleteAt
+			uploadElement.Message = fmt.Sprintf("Deleted file '%s' (id '%s') was removed because it was too large to send.", fileInfo.Name, fileInfo.Id)
+		}
+	} else {
+		channelExport.uploadedFiles = append(channelExport.uploadedFiles, fileInfo)
+
+		// add an implicit "post" to the export that includes the filename so GlobalRelay knows who uploaded each file
+		uploadElement = Message{
+			SentTime:       fileInfo.CreateAt,
+			Message:        fmt.Sprintf("Uploaded file %s", fileInfo.Name),
+			SenderUsername: *post.Username,
+			SenderUserType: userType,
+			SenderEmail:    *post.UserEmail,
+		}
+
+		if fileInfo.DeleteAt != 0 {
+			uploadElement.SentTime = fileInfo.DeleteAt
+			uploadElement.Message = fmt.Sprintf("Deleted file %s", fileInfo.Name)
+		}
+	}
+
+	channelExport.Messages = append(channelExport.Messages, uploadElement)
+}
+
+func encodeRFC2047Word(s string) string {
+	return mime.BEncoding.Encode("utf-8", s)
+}

server/enterprise/message_export/global_relay_export/main_test.go

@@ -0,0 +1,23 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package global_relay_export
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost/server/v8/channels/testlib"
+)
+
+var mainHelper *testlib.MainHelper
+
+func TestMain(m *testing.M) {
+	var options = testlib.HelperOptions{
+		EnableResources: true,
+	}
+
+	mainHelper = testlib.NewMainHelperWithOptions(&options)
+	defer mainHelper.Close()
+
+	mainHelper.Main(m)
+}

server/enterprise/message_export/global_relay_export/smtp.go

@@ -0,0 +1,93 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package global_relay_export
+
+import (
+	"context"
+	"net/smtp"
+	"os"
+	"strconv"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/utils"
+	"github.com/mattermost/mattermost/server/v8/channels/utils/testutils"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/mail"
+)
+
+const (
+	GlobalRelayA9Server  = "mailarchivespool1.globalrelay.com"
+	GlobalRelayA10Server = "feeds.globalrelay.com"
+	GlobalRelayA9IP      = "208.81.212.70"
+	GlobalRelayA10IP     = "208.81.213.24"
+
+	defaultSMTPPort         = "25"
+	defaultInbucketSMTPPort = "10025"
+)
+
+func connectToSMTPServer(ctx context.Context, config *model.Config) (*smtp.Client, error) {
+	smtpServerName := ""
+	smtpServerHost := ""
+	smtpPort := defaultSMTPPort
+	security := model.ConnSecurityStarttls
+	auth := true
+	if *config.MessageExportSettings.GlobalRelaySettings.CustomerType == "A10" {
+		smtpServerName = GlobalRelayA10Server
+		smtpServerHost = GlobalRelayA10IP
+	} else if *config.MessageExportSettings.GlobalRelaySettings.CustomerType == "A9" {
+		smtpServerName = GlobalRelayA9Server
+		smtpServerHost = GlobalRelayA9IP
+	} else if *config.MessageExportSettings.GlobalRelaySettings.CustomerType == "INBUCKET" {
+		inbucketSMTPPort := os.Getenv("CI_INBUCKET_SMTP_PORT")
+		if inbucketSMTPPort == "" {
+			inbucketSMTPPort = defaultInbucketSMTPPort
+		}
+		inbucketHost := os.Getenv("CI_INBUCKET_HOST")
+		if inbucketHost == "" {
+			intPort, err := strconv.Atoi(inbucketSMTPPort)
+			if err != nil {
+				intPort = 0
+			}
+			inbucketHost = testutils.GetInterface(intPort)
+		}
+		smtpServerName = inbucketHost
+		smtpServerHost = inbucketHost
+		smtpPort = inbucketSMTPPort
+		auth = false
+	} else if *config.MessageExportSettings.GlobalRelaySettings.CustomerType == model.GlobalrelayCustomerTypeCustom {
+		customSMTPPort := *config.MessageExportSettings.GlobalRelaySettings.CustomSMTPPort
+		if customSMTPPort != "" {
+			smtpPort = customSMTPPort
+		}
+		smtpServerName = *config.MessageExportSettings.GlobalRelaySettings.CustomSMTPServerName
+		smtpServerHost = *config.MessageExportSettings.GlobalRelaySettings.CustomSMTPServerName
+	}
+
+	smtpConfig := &mail.SMTPConfig{
+		ConnectionSecurity:                security,
+		SkipServerCertificateVerification: false,
+		Hostname:                          utils.GetHostnameFromSiteURL(*config.ServiceSettings.SiteURL),
+		ServerName:                        smtpServerName,
+		Server:                            smtpServerHost,
+		Port:                              smtpPort,
+		EnableSMTPAuth:                    auth,
+		Username:                          *config.MessageExportSettings.GlobalRelaySettings.SMTPUsername,
+		Password:                          *config.MessageExportSettings.GlobalRelaySettings.SMTPPassword,
+		ServerTimeout:                     *config.MessageExportSettings.GlobalRelaySettings.SMTPServerTimeout,
+	}
+	conn, err1 := mail.ConnectToSMTPServerAdvanced(smtpConfig)
+	if err1 != nil {
+		return nil, err1
+	}
+
+	c, err2 := mail.NewSMTPClientAdvanced(
+		ctx,
+		conn,
+		smtpConfig,
+	)
+	if err2 != nil {
+		conn.Close()
+		return nil, err2
+	}
+	return c, nil
+}

server/enterprise/message_export/global_relay_export/to_html.go

@@ -0,0 +1,105 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package global_relay_export
+
+import (
+	"bytes"
+	"html/template"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/hako/durafmt"
+
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/templates"
+)
+
+func channelExportToHTML(rctx request.CTX, channelExport *ChannelExport, t *templates.Container) (string, error) {
+	durationMilliseconds := channelExport.EndTime - channelExport.StartTime
+	// TODO CHECK IF WE NEED THE MILISECONS HERE OR WE CAN ROUND IT DIRECTLY HERE
+	duration := time.Duration(durationMilliseconds) * time.Millisecond
+
+	var participantRowsBuffer bytes.Buffer
+	for i := range channelExport.Participants {
+		participantHTML, err := participantToHTML(&channelExport.Participants[i], t)
+		if err != nil {
+			rctx.Logger().Error("Unable to render participant html for compliance export", mlog.Err(err))
+			continue
+		}
+		participantRowsBuffer.WriteString(participantHTML)
+	}
+
+	var messagesBuffer bytes.Buffer
+	sort.Slice(channelExport.Messages, func(i, j int) bool {
+		if channelExport.Messages[i].SentTime == channelExport.Messages[j].SentTime {
+			return !strings.HasPrefix(channelExport.Messages[i].Message, "Uploaded file") && !strings.HasPrefix(channelExport.Messages[i].Message, "Deleted file")
+		}
+		return channelExport.Messages[i].SentTime < channelExport.Messages[j].SentTime
+	})
+	for i := range channelExport.Messages {
+		messageHTML, err := messageToHTML(&channelExport.Messages[i], t)
+		if err != nil {
+			rctx.Logger().Error("Unable to render message html for compliance export", mlog.Err(err))
+			continue
+		}
+		messagesBuffer.WriteString(messageHTML)
+	}
+
+	data := templates.Data{
+		Props: map[string]any{
+			"ChannelName":     channelExport.ChannelName,
+			"Started":         time.Unix(channelExport.StartTime/1000, 0).UTC().Format(time.RFC3339),
+			"Ended":           time.Unix(channelExport.EndTime/1000, 0).UTC().Format(time.RFC3339),
+			"Duration":        durafmt.Parse(duration.Round(time.Minute)).String(),
+			"ParticipantRows": template.HTML(participantRowsBuffer.String()),
+			"Messages":        template.HTML(messagesBuffer.String()),
+			"ExportDate":      time.Unix(channelExport.ExportedOn/1000, 0).UTC().Format(time.RFC3339),
+		},
+	}
+
+	return t.RenderToString("globalrelay_compliance_export", data)
+}
+
+func participantToHTML(participant *ParticipantRow, t *templates.Container) (string, error) {
+	durationMilliseconds := participant.LeaveTime - participant.JoinTime
+	// TODO CHECK IF WE NEED THE MILISECONS HERE OR WE CAN ROUND IT DIRECTLY HERE
+	duration := time.Duration(durationMilliseconds) * time.Millisecond
+
+	data := templates.Data{
+		Props: map[string]any{
+			"Username":    participant.Username,
+			"UserType":    participant.UserType,
+			"Email":       participant.Email,
+			"Joined":      time.Unix(participant.JoinTime/1000, 0).UTC().Format(time.RFC3339),
+			"Left":        time.Unix(participant.LeaveTime/1000, 0).UTC().Format(time.RFC3339),
+			"Duration":    durafmt.Parse(duration.Round(time.Minute)).String(),
+			"NumMessages": participant.MessagesSent,
+		},
+	}
+	return t.RenderToString("globalrelay_compliance_export_participant_row", data)
+}
+
+func messageToHTML(message *Message, t *templates.Container) (string, error) {
+	postUsername := message.PostUsername
+	// Added to improve readability
+	if postUsername != "" {
+		postUsername = "@" + postUsername
+	}
+	data := templates.Data{
+		Props: map[string]any{
+			"SentTime":     time.Unix(message.SentTime/1000, 0).UTC().Format(time.RFC3339),
+			"Username":     message.SenderUsername,
+			"PostUsername": postUsername,
+			"UserType":     message.SenderUserType,
+			"PostType":     message.PostType,
+			"Email":        message.SenderEmail,
+			"Message":      message.Message,
+			"PreviewsPost": message.PreviewsPost,
+		},
+	}
+
+	return t.RenderToString("globalrelay_compliance_export_message", data)
+}

server/enterprise/message_export/main_test.go

@@ -0,0 +1,21 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package message_export
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost/server/v8/channels/api4"
+	"github.com/mattermost/mattermost/server/v8/channels/testlib"
+)
+
+var mainHelper *testlib.MainHelper
+
+func TestMain(m *testing.M) {
+	mainHelper = testlib.NewMainHelper()
+	defer mainHelper.Close()
+	api4.SetMainHelper(mainHelper)
+
+	mainHelper.Main(m)
+}

server/enterprise/message_export/membership_map.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package message_export
+
+type MembershipMapUser struct {
+	userId   string
+	email    string
+	username string
+}
+
+// Provides a clean interface for tracking the users that are present in any number of channels by channel id and user email
+type MembershipMap map[string]map[string]MembershipMapUser
+
+func (m *MembershipMap) init(channelId string) {
+	if *m == nil {
+		*m = make(map[string]map[string]MembershipMapUser)
+	}
+	if (*m)[channelId] == nil {
+		(*m)[channelId] = make(map[string]MembershipMapUser)
+	}
+}
+
+func (m *MembershipMap) AddUserToChannel(channelId string, user MembershipMapUser) {
+	m.init(channelId)
+	if !m.IsUserInChannel(channelId, user.email) {
+		(*m)[channelId][user.email] = user
+	}
+}
+
+func (m *MembershipMap) RemoveUserFromChannel(channelId string, userEmail string) {
+	m.init(channelId)
+	delete((*m)[channelId], userEmail)
+}
+
+func (m *MembershipMap) IsUserInChannel(channelId string, userEmail string) bool {
+	m.init(channelId)
+	_, exists := (*m)[channelId][userEmail]
+	return exists
+}
+
+func (m *MembershipMap) GetUserEmailsInChannel(channelId string) []string {
+	m.init(channelId)
+	users := make([]string, 0, len((*m)[channelId]))
+	for k := range (*m)[channelId] {
+		users = append(users, k)
+	}
+	return users
+}
+
+func (m *MembershipMap) GetUsersInChannel(channelId string) []MembershipMapUser {
+	m.init(channelId)
+	users := make([]MembershipMapUser, 0, len((*m)[channelId]))
+	for _, v := range (*m)[channelId] {
+		users = append(users, v)
+	}
+	return users
+}

server/enterprise/message_export/membership_map_test.go

@@ -0,0 +1,79 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package message_export
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/mattermost/mattermost/server/public/model"
+)
+
+func TestMembershipMap(t *testing.T) {
+	membershipMap := make(MembershipMap)
+
+	channelId := model.NewId()
+
+	user1 := &MembershipMapUser{
+		email:    model.NewId() + "@mattermost.com",
+		username: model.NewId(),
+		userId:   model.NewId(),
+	}
+	user2 := &MembershipMapUser{
+		email:    model.NewId() + "@mattermost.com",
+		username: model.NewId(),
+		userId:   model.NewId(),
+	}
+
+	assert.False(t, membershipMap.IsUserInChannel(channelId, user1.email))
+	membershipMap.AddUserToChannel(channelId, *user1)
+	assert.True(t, membershipMap.IsUserInChannel(channelId, user1.email))
+
+	assert.False(t, membershipMap.IsUserInChannel(channelId, user2.email))
+	membershipMap.AddUserToChannel(channelId, *user2)
+	assert.True(t, membershipMap.IsUserInChannel(channelId, user2.email))
+
+	// ensure that the correct user emails are returned
+	emails := membershipMap.GetUserEmailsInChannel(channelId)
+	assert.Len(t, emails, 2)
+	assert.Contains(t, emails, user1.email)
+	assert.Contains(t, emails, user2.email)
+
+	// ensure that the correct user objects are returned
+	users := membershipMap.GetUsersInChannel(channelId)
+	assert.Len(t, users, 2)
+	if users[0].userId == user1.userId {
+		assert.Equal(t, user1.username, users[0].username)
+		assert.Equal(t, user1.email, users[0].email)
+		assert.Equal(t, user2.userId, users[1].userId)
+		assert.Equal(t, user2.username, users[1].username)
+		assert.Equal(t, user2.email, users[1].email)
+	} else if users[0].userId == user2.userId {
+		assert.Equal(t, user2.username, users[0].username)
+		assert.Equal(t, user2.email, users[0].email)
+		assert.Equal(t, user1.userId, users[1].userId)
+		assert.Equal(t, user1.username, users[1].username)
+		assert.Equal(t, user1.email, users[1].email)
+	} else {
+		assert.Fail(t, "First returned user is not recognized")
+	}
+
+	// remove user1 from the channel
+	membershipMap.RemoveUserFromChannel(channelId, user1.email)
+	assert.False(t, membershipMap.IsUserInChannel(channelId, user1.email))
+	assert.True(t, membershipMap.IsUserInChannel(channelId, user2.email))
+
+	// ensure that user2's email is returned
+	emails = membershipMap.GetUserEmailsInChannel(channelId)
+	assert.Len(t, emails, 1)
+	assert.Contains(t, emails, user2.email)
+
+	// ensure that only user2 is returned
+	users = membershipMap.GetUsersInChannel(channelId)
+	assert.Len(t, users, 1)
+	assert.Equal(t, user2.userId, users[0].userId)
+	assert.Equal(t, user2.username, users[0].username)
+	assert.Equal(t, user2.email, users[0].email)
+}

server/enterprise/message_export/message_export.go

@@ -0,0 +1,234 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package message_export
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math"
+	"net/http"
+	"os"
+	"path"
+	"time"
+
+	"strconv"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/app"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/mattermost/mattermost/server/v8/channels/utils/fileutils"
+	"github.com/mattermost/mattermost/server/v8/einterfaces"
+	ejobs "github.com/mattermost/mattermost/server/v8/einterfaces/jobs"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/templates"
+
+	"github.com/mattermost/mattermost/server/v8/enterprise/message_export/actiance_export"
+	"github.com/mattermost/mattermost/server/v8/enterprise/message_export/csv_export"
+	"github.com/mattermost/mattermost/server/v8/enterprise/message_export/global_relay_export"
+)
+
+const (
+	GlobalRelayExportFilename = "global-relay.zip"
+)
+
+type MessageExportInterfaceImpl struct {
+	Server *app.Server
+}
+
+type MessageExportJobInterfaceImpl struct {
+	Server *app.Server
+}
+
+func init() {
+	app.RegisterJobsMessageExportJobInterface(func(s *app.Server) ejobs.MessageExportJobInterface {
+		return &MessageExportJobInterfaceImpl{s}
+	})
+	app.RegisterMessageExportInterface(func(app *app.App) einterfaces.MessageExportInterface {
+		return &MessageExportInterfaceImpl{app.Srv()}
+	})
+}
+
+func (m *MessageExportInterfaceImpl) StartSynchronizeJob(rctx request.CTX, exportFromTimestamp int64) (*model.Job, *model.AppError) {
+	// if a valid export time was specified, put it in the job data
+	jobData := make(map[string]string)
+	if exportFromTimestamp >= 0 {
+		jobData[JobDataBatchStartTimestamp] = strconv.FormatInt(exportFromTimestamp, 10)
+	}
+
+	// passing nil for job data will cause the worker to inherit start time from previously successful job
+	job, err := m.Server.Jobs.CreateJob(rctx, model.JobTypeMessageExport, jobData)
+	if err != nil {
+		return nil, err
+	}
+
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+
+	for job.Status == model.JobStatusPending ||
+		job.Status == model.JobStatusInProgress ||
+		job.Status == model.JobStatusCancelRequested {
+		select {
+		case <-ticker.C:
+			job, err = m.Server.Jobs.GetJob(rctx, job.Id)
+			if err != nil {
+				return nil, err
+			}
+		case <-rctx.Context().Done():
+			return nil, model.NewAppError("StartSynchronizeJob", "ent.jobs.start_synchronize_job.timeout", nil, "", 0).Wrap(rctx.Context().Err())
+		}
+	}
+
+	return job, nil
+}
+
+func (m *MessageExportInterfaceImpl) RunExport(rctx request.CTX, exportType string, since int64, limit int) (warningCount int64, appErr *model.AppError) {
+	if limit < 0 {
+		limit = math.MaxInt64
+	}
+	postsToExport, _, err := m.Server.Store().Compliance().MessageExport(rctx, model.MessageExportCursor{LastPostUpdateAt: since}, limit)
+	if err != nil {
+		return warningCount, model.NewAppError("RunExport", "ent.message_export.run_export.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	rctx.Logger().Debug("Found posts to export", mlog.Int("number_of_posts", len(postsToExport)))
+
+	fileBackend := m.Server.FileBackend()
+	templatesDir, ok := fileutils.FindDir("templates")
+	if !ok {
+		return warningCount, model.NewAppError("RunExport", "ent.compliance.run_export.template_watcher.appError", nil, "", http.StatusAccepted)
+	}
+
+	t, err2 := templates.New(templatesDir)
+	if err2 != nil {
+		return warningCount, model.NewAppError("RunExport", "ent.compliance.run_export.template_watcher.appError", nil, "", http.StatusAccepted).Wrap(err2)
+	}
+
+	exportDirectory := getOutputDirectoryPath(since, model.GetMillis())
+	return runExportByType(rctx, exportType, postsToExport, exportDirectory, m.Server.Store(), fileBackend, fileBackend, t, m.Server.Config())
+}
+
+func runExportByType(rctx request.CTX, exportType string, postsToExport []*model.MessageExport, exportDirectory string, db store.Store, exportBackend filestore.FileBackend, fileAttachmentBackend filestore.FileBackend, htmlTemplates *templates.Container, config *model.Config) (warningCount int64, appErr *model.AppError) {
+	// go through all the posts and if the post's props contain 'from_bot' - override the IsBot field, since it's possible that the sender is not a user, but was a Bot and vise-versa
+	for _, post := range postsToExport {
+		if post.PostProps != nil {
+			props := map[string]any{}
+
+			if json.Unmarshal([]byte(*post.PostProps), &props) == nil {
+				if val, ok := props["from_bot"]; ok {
+					post.IsBot = val == "true"
+				}
+			}
+		}
+
+		// Team info can be null for DM/GM channels.
+		if post.TeamId == nil {
+			post.TeamId = new(string)
+		}
+		if post.TeamName == nil {
+			post.TeamName = new(string)
+		}
+		if post.TeamDisplayName == nil {
+			post.TeamDisplayName = new(string)
+		}
+
+		// make sure user information is present. Set defaults and log an error otherwise.
+		if post.ChannelId == nil {
+			rctx.Logger().Warn("ChannelId is missing for post", mlog.String("post_id", *post.PostId))
+			post.ChannelId = new(string)
+		}
+		if post.ChannelName == nil {
+			rctx.Logger().Warn("ChannelName is missing for post", mlog.String("post_id", *post.PostId))
+			post.ChannelName = new(string)
+		}
+		if post.ChannelDisplayName == nil {
+			rctx.Logger().Warn("ChannelDisplayName is missing for post", mlog.String("post_id", *post.PostId))
+			post.ChannelDisplayName = new(string)
+		}
+		if post.ChannelType == nil {
+			rctx.Logger().Warn("ChannelType is missing for post", mlog.String("post_id", *post.PostId))
+			post.ChannelType = new(model.ChannelType)
+		}
+
+		if post.UserId == nil {
+			rctx.Logger().Warn("UserId is missing for post", mlog.String("post_id", *post.PostId))
+			post.UserId = new(string)
+		}
+		if post.UserEmail == nil {
+			rctx.Logger().Warn("UserEmail is missing for post", mlog.String("post_id", *post.PostId))
+			post.UserEmail = new(string)
+		}
+		if post.Username == nil {
+			rctx.Logger().Warn("Username is missing for post", mlog.String("post_id", *post.PostId))
+			post.Username = new(string)
+		}
+
+		if post.PostType == nil {
+			rctx.Logger().Warn("Type is missing for post", mlog.String("post_id", *post.PostId))
+			post.PostType = new(string)
+		}
+		if post.PostMessage == nil {
+			rctx.Logger().Warn("Message is missing for post", mlog.String("post_id", *post.PostId))
+			post.PostMessage = new(string)
+		}
+		if post.PostCreateAt == nil {
+			rctx.Logger().Warn("CreateAt is missing for post", mlog.String("post_id", *post.PostId))
+			post.PostCreateAt = new(int64)
+		}
+	}
+
+	switch exportType {
+	case model.ComplianceExportTypeCsv:
+		rctx.Logger().Debug("Exporting CSV")
+		return csv_export.CsvExport(rctx, postsToExport, db, exportBackend, fileAttachmentBackend, exportDirectory)
+
+	case model.ComplianceExportTypeActiance:
+		rctx.Logger().Debug("Exporting Actiance")
+		return actiance_export.ActianceExport(rctx, postsToExport, db, exportBackend, fileAttachmentBackend, exportDirectory)
+
+	case model.ComplianceExportTypeGlobalrelay, model.ComplianceExportTypeGlobalrelayZip:
+		rctx.Logger().Debug("Exporting GlobalRelay")
+		f, err := os.CreateTemp("", "")
+		if err != nil {
+			return warningCount, model.NewAppError("RunExport", "ent.compliance.global_relay.open_temporary_file.appError", nil, "", http.StatusAccepted).Wrap(err)
+		}
+		defer f.Close()
+		defer os.Remove(f.Name())
+
+		attachmentsRemovedPostIDs, warnings, appErr := global_relay_export.GlobalRelayExport(rctx, postsToExport, db, fileAttachmentBackend, f, htmlTemplates)
+		if appErr != nil {
+			return warningCount, appErr
+		}
+		warningCount = warnings
+		_, err = f.Seek(0, 0)
+		if err != nil {
+			return warningCount, model.NewAppError("RunExport", "ent.compliance.global_relay.rewind_temporary_file.appError", nil, "", http.StatusAccepted).Wrap(err)
+		}
+
+		if exportType == model.ComplianceExportTypeGlobalrelayZip {
+			// Try to disable the write timeout for the potentially big export file.
+			_, nErr := filestore.TryWriteFileContext(rctx.Context(), exportBackend, f, path.Join(exportDirectory, GlobalRelayExportFilename))
+			if nErr != nil {
+				return warningCount, model.NewAppError("runExportByType", "ent.compliance.global_relay.write_file.appError", nil, "", http.StatusInternalServerError).Wrap(nErr)
+			}
+		} else {
+			appErr = global_relay_export.Deliver(f, config)
+			if appErr != nil {
+				return warningCount, appErr
+			}
+		}
+
+		if len(attachmentsRemovedPostIDs) > 0 {
+			rctx.Logger().Debug("Global Relay Attachments Removed because they were too large to send to Global Relay", mlog.Array("attachment_ids", attachmentsRemovedPostIDs))
+			description := fmt.Sprintf("Attachments to post IDs %v were removed because they were too large to send to Global Relay.", attachmentsRemovedPostIDs)
+			appErr = model.NewAppError("RunExport", "ent.compliance.global_relay.attachments_removed.appError", map[string]any{"Description": description}, description, http.StatusAccepted)
+			return warningCount, appErr
+		}
+	default:
+		err := errors.New("Unknown output format " + exportType)
+		return warningCount, model.NewAppError("RunExport", "ent.compliance.bad_export_type.appError", map[string]any{"ExportType": exportType}, "", http.StatusBadRequest).Wrap(err)
+	}
+	return warningCount, nil
+}

server/enterprise/message_export/message_export_test.go

@@ -0,0 +1,145 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package message_export
+
+import (
+	"context"
+	"os"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/api4"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRunExportByType(t *testing.T) {
+	tempDir, err := os.MkdirTemp("", "")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		err = os.RemoveAll(tempDir)
+		assert.NoError(t, err)
+	})
+
+	config := filestore.FileBackendSettings{
+		DriverName: model.ImageDriverLocal,
+		Directory:  tempDir,
+	}
+
+	fileBackend, err := filestore.NewFileBackend(config)
+	require.NoError(t, err)
+
+	rctx := request.TestContext(t)
+
+	chanTypeDirect := model.ChannelTypeDirect
+	t.Run("missing user info", func(t *testing.T) {
+		posts := []*model.MessageExport{
+			{
+				PostId:             model.NewPointer("post-id"),
+				PostOriginalId:     model.NewPointer("post-original-id"),
+				TeamId:             model.NewPointer("team-id"),
+				TeamName:           model.NewPointer("team-name"),
+				TeamDisplayName:    model.NewPointer("team-display-name"),
+				ChannelId:          model.NewPointer("channel-id"),
+				ChannelName:        model.NewPointer("channel-name"),
+				ChannelDisplayName: model.NewPointer("channel-display-name"),
+				PostCreateAt:       model.NewPointer(int64(1)),
+				PostUpdateAt:       model.NewPointer(int64(1)),
+				PostMessage:        model.NewPointer("message"),
+				ChannelType:        &chanTypeDirect,
+				PostFileIds:        []string{},
+			},
+		}
+
+		mockStore := &storetest.Store{}
+		defer mockStore.AssertExpectations(t)
+		mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", int64(1), int64(1), "channel-id").Return([]*model.ChannelMemberHistoryResult{}, nil)
+
+		warnings, err := runExportByType(rctx, model.ComplianceExportTypeActiance, posts, tempDir, mockStore, fileBackend, fileBackend, nil, nil)
+		require.Nil(t, err)
+		require.Zero(t, warnings)
+	})
+}
+
+func runJobForTest(t *testing.T, th *api4.TestHelper) *model.Job {
+	job, _, err := th.SystemAdminClient.CreateJob(context.Background(), &model.Job{Type: "message_export"})
+	require.NoError(t, err)
+	// poll until completion
+	doneChan := make(chan bool)
+	go func() {
+		defer close(doneChan)
+		for {
+			jobs, _, err := th.SystemAdminClient.GetJobsByType(context.Background(), "message_export", 0, 1)
+			require.NoError(t, err)
+			require.Len(t, jobs, 1)
+			require.Equal(t, job.Id, jobs[0].Id)
+			job = jobs[0]
+			if job.Status != "pending" && job.Status != "in_progress" {
+				break
+			}
+			time.Sleep(1 * time.Second)
+		}
+		require.Equal(t, "success", job.Status)
+	}()
+	select {
+	case <-doneChan:
+	case <-time.After(10 * time.Second):
+		require.True(t, false, "job is taking too long")
+	}
+	return job
+}
+
+func TestRunExportJob(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	jobs.DefaultWatcherPollingInterval = 100
+	th := api4.SetupEnterprise(t).InitBasic()
+	th.App.Srv().SetLicense(model.NewTestLicense("message_export"))
+	defer th.TearDown()
+	messageExportImpl := MessageExportJobInterfaceImpl{th.App.Srv()}
+	th.App.Srv().Jobs.RegisterJobType(model.JobTypeMessageExport, messageExportImpl.MakeWorker(), messageExportImpl.MakeScheduler())
+
+	err := th.App.Srv().Jobs.StartWorkers()
+	require.NoError(t, err)
+
+	err = th.App.Srv().Jobs.StartSchedulers()
+	require.NoError(t, err)
+
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.MessageExportSettings.EnableExport = true
+	})
+
+	t.Run("conflicting timestamps", func(t *testing.T) {
+		time.Sleep(100 * time.Millisecond)
+		now := model.GetMillis()
+		th.App.UpdateConfig(func(cfg *model.Config) {
+			*cfg.MessageExportSettings.ExportFromTimestamp = now - 1
+			*cfg.MessageExportSettings.BatchSize = 2
+		})
+
+		for i := 0; i < 3; i++ {
+			_, err := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
+				ChannelId: th.BasicChannel.Id,
+				UserId:    model.NewId(),
+				Message:   "zz" + model.NewId() + "b",
+				CreateAt:  now,
+			})
+			require.NoError(t, err)
+		}
+
+		job := runJobForTest(t, th)
+		numExported, err := strconv.ParseInt(job.Data["messages_exported"], 0, 64)
+		require.NoError(t, err)
+		require.Equal(t, int64(3), numExported)
+	})
+}

server/enterprise/message_export/scheduler.go

@@ -0,0 +1,76 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package message_export
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	ejobs "github.com/mattermost/mattermost/server/v8/einterfaces/jobs"
+)
+
+type MessageExportScheduler struct {
+	jobServer   *jobs.JobServer
+	enabledFunc func(cfg *model.Config) bool
+}
+
+var _ jobs.Scheduler = (*MessageExportScheduler)(nil)
+
+func NewMessageExportScheduler(jobServer *jobs.JobServer, enabledFunc func(cfg *model.Config) bool) *MessageExportScheduler {
+	return &MessageExportScheduler{
+		enabledFunc: enabledFunc,
+		jobServer:   jobServer,
+	}
+}
+
+func (s *MessageExportScheduler) Enabled(cfg *model.Config) bool {
+	return s.enabledFunc(cfg)
+}
+
+func (s *MessageExportScheduler) NextScheduleTime(cfg *model.Config, now time.Time, _ bool, _ *model.Job) *time.Time {
+	// We set the next scheduled time regardless of whether there is a running or pending job
+	// In ScheduleJob we check pending or running jobs, before actually scheduling a job
+	parsedTime, err := time.Parse("15:04", *cfg.MessageExportSettings.DailyRunTime)
+	if err != nil {
+		s.jobServer.Logger().Error(
+			"Cannot determine next schedule time for message export. DailyRunTime config value is invalid.",
+			mlog.String("DailyRunTime", *cfg.MessageExportSettings.DailyRunTime),
+		)
+		return nil
+	}
+	return jobs.GenerateNextStartDateTime(now, parsedTime)
+}
+
+func (s *MessageExportScheduler) ScheduleJob(rctx request.CTX, _ *model.Config, havePendingJobs bool, _ *model.Job) (*model.Job, *model.AppError) {
+	// Don't schedule a job if we already have a pending job
+	if havePendingJobs {
+		return nil, nil
+	}
+	// Don't schedule a job if we already have a running job
+	count, err := s.jobServer.Store.Job().GetCountByStatusAndType(model.JobStatusInProgress, model.JobTypeMessageExport)
+	if err != nil {
+		return nil, model.NewAppError(
+			"ScheduleJob",
+			"app.job.get_count_by_status_and_type.app_error",
+			map[string]any{"jobtype": model.JobTypeMessageExport, "status": model.JobStatusInProgress},
+			"",
+			http.StatusInternalServerError).Wrap(err)
+	}
+	if count > 0 {
+		return nil, nil
+	}
+	return s.jobServer.CreateJob(rctx, model.JobTypeMessageExport, nil)
+}
+
+func (dr *MessageExportJobInterfaceImpl) MakeScheduler() ejobs.Scheduler {
+	enabled := func(cfg *model.Config) bool {
+		license := dr.Server.License()
+		return license != nil && *license.Features.MessageExport && *cfg.MessageExportSettings.EnableExport
+	}
+	return NewMessageExportScheduler(dr.Server.Jobs, enabled)
+}

server/enterprise/message_export/scheduler_test.go

@@ -0,0 +1,98 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package message_export
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/api4"
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
+)
+
+func TestMessageExportJobEnabled(t *testing.T) {
+	t.Run("MessageExport job is enabled only if feature is enabled", func(t *testing.T) {
+		th := api4.SetupEnterpriseWithStoreMock(t)
+		defer th.TearDown()
+
+		th.Server.SetLicense(model.NewTestLicense("message_export"))
+
+		messageExport := &MessageExportJobInterfaceImpl{th.App.Srv()}
+
+		config := &model.Config{
+			MessageExportSettings: model.MessageExportSettings{
+				EnableExport: model.NewPointer(true),
+			},
+		}
+		scheduler := messageExport.MakeScheduler()
+		result := scheduler.Enabled(config)
+		assert.True(t, result)
+	})
+
+	t.Run("MessageExport job is disabled if there is no license", func(t *testing.T) {
+		th := api4.SetupEnterpriseWithStoreMock(t)
+		defer th.TearDown()
+
+		th.Server.SetLicense(nil)
+
+		messageExport := &MessageExportJobInterfaceImpl{th.App.Srv()}
+
+		config := &model.Config{
+			MessageExportSettings: model.MessageExportSettings{
+				EnableExport: model.NewPointer(true),
+			},
+		}
+		scheduler := messageExport.MakeScheduler()
+		result := scheduler.Enabled(config)
+		assert.False(t, result)
+	})
+}
+
+func TestMessageExportJobPending(t *testing.T) {
+	th := api4.SetupEnterpriseWithStoreMock(t)
+	defer th.TearDown()
+
+	mockStore := th.App.Srv().Platform().Store.(*mocks.Store)
+	mockUserStore := mocks.UserStore{}
+	mockUserStore.On("Count", mock.Anything).Return(int64(10), nil)
+	mockPostStore := mocks.PostStore{}
+	mockPostStore.On("GetMaxPostSize").Return(65535, nil)
+	mockSystemStore := mocks.SystemStore{}
+	mockSystemStore.On("GetByName", "UpgradedFromTE").Return(&model.System{Name: "UpgradedFromTE", Value: "false"}, nil)
+	mockSystemStore.On("GetByName", "InstallationDate").Return(&model.System{Name: "InstallationDate", Value: "10"}, nil)
+	mockSystemStore.On("GetByName", "FirstServerRunTimestamp").Return(&model.System{Name: "FirstServerRunTimestamp", Value: "10"}, nil)
+	mockStore.On("User").Return(&mockUserStore)
+	mockStore.On("Post").Return(&mockPostStore)
+	mockStore.On("System").Return(&mockSystemStore)
+	mockStore.On("GetDBSchemaVersion").Return(1, nil)
+
+	mockJobServerStore := th.App.Srv().Jobs.Store.(*mocks.Store)
+	mockJobStore := mocks.JobStore{}
+	// Mock that we have an in-progress message export job
+	mockJobStore.On("GetCountByStatusAndType", model.JobStatusInProgress, model.JobTypeMessageExport).Return(int64(1), nil)
+	mockJobServerStore.On("Job").Return(&mockJobStore)
+
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.MessageExportSettings.EnableExport = true
+		*cfg.MessageExportSettings.DailyRunTime = "10:40"
+	})
+
+	th.App.Srv().SetLicense(model.NewTestLicense("message_export"))
+
+	messageExport := &MessageExportJobInterfaceImpl{th.App.Srv()}
+	scheduler := messageExport.MakeScheduler()
+
+	// Confirm that job is not scheduled if we have pending jobs
+	job, err := scheduler.ScheduleJob(th.Context, th.App.Config(), true, nil)
+	assert.Nil(t, err)
+	assert.Nil(t, job)
+
+	// Confirm that job is not scheduled if we have an inprogress job
+	job, err = scheduler.ScheduleJob(th.Context, th.App.Config(), false, nil)
+	assert.Nil(t, err)
+	assert.Nil(t, job)
+}

server/enterprise/message_export/worker.go

@@ -0,0 +1,579 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package message_export
+
+import (
+	"archive/zip"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/utils/fileutils"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/templates"
+)
+
+const (
+	JobDataBatchStartTimestamp = "batch_start_timestamp" // message export uses keyset pagination sorted by (posts.updateat, posts.id). batch_start_timestamp is the posts.updateat value from the previous batch.
+	JobDataBatchStartId        = "batch_start_id"        // message export uses keyset pagination sorted by (posts.updateat, posts.id). batch_start_id is the posts.id value from the previous batch.
+
+	JobDataStartTimestamp   = "start_timestamp"
+	JobDataStartId          = "start_id"
+	JobDataExportType       = "export_type"
+	JOB_DATA_BatchSize      = "batch_size"
+	JobDataMessagesExported = "messages_exported"
+	JobDataWarningCount     = "warning_count"
+	JobDataIsDownloadable   = "is_downloadable"
+	JobDirectories          = "job_directories"
+	TimeBetweenBatches      = 100
+
+	estimatedPostCount = 10_000_000
+)
+
+const exportPath = "export"
+
+type MessageExportWorker struct {
+	name string
+	// stateMut protects stopCh, cancel, and stopped and helps enforce
+	// ordering in case subsequent Run or Stop calls are made.
+	stateMut            sync.Mutex
+	stopCh              chan struct{}
+	stopped             bool
+	stoppedCh           chan struct{}
+	jobs                chan model.Job
+	jobServer           *jobs.JobServer
+	logger              mlog.LoggerIFace
+	htmlTemplateWatcher *templates.Container
+	license             func() *model.License
+
+	context context.Context
+	cancel  func()
+}
+
+func (dr *MessageExportJobInterfaceImpl) MakeWorker() model.Worker {
+	const workerName = "MessageExportWorker"
+	logger := dr.Server.Jobs.Logger().With(mlog.String("worker_name", workerName))
+
+	templatesDir, ok := fileutils.FindDir("templates")
+	if !ok {
+		logger.Error("Failed to initialize HTMLTemplateWatcher, templates directory not found")
+		return nil
+	}
+	htmlTemplateWatcher, err := templates.New(templatesDir)
+	if err != nil {
+		logger.Error("Failed to initialize HTMLTemplateWatcher", mlog.Err(err))
+		return nil
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	return &MessageExportWorker{
+		name:                workerName,
+		stoppedCh:           make(chan struct{}, 1),
+		jobs:                make(chan model.Job),
+		jobServer:           dr.Server.Jobs,
+		logger:              logger,
+		htmlTemplateWatcher: htmlTemplateWatcher,
+		// It is not a best practice to store context inside a struct,
+		// however we need to cancel a SQL query during a job execution.
+		// There is no other good way.
+		context: ctx,
+		cancel:  cancel,
+		license: dr.Server.License,
+		stopped: true,
+	}
+}
+
+func (worker *MessageExportWorker) IsEnabled(cfg *model.Config) bool {
+	return worker.license() != nil && *worker.license().Features.MessageExport && *cfg.MessageExportSettings.EnableExport
+}
+
+func (worker *MessageExportWorker) Run() {
+	worker.stateMut.Lock()
+	// We have to re-assign the stop channel again, because
+	// it might happen that the job was restarted due to a config change.
+	if worker.stopped {
+		worker.stopped = false
+		worker.stopCh = make(chan struct{})
+		worker.context, worker.cancel = context.WithCancel(context.Background())
+	} else {
+		worker.stateMut.Unlock()
+		return
+	}
+	// Run is called from a separate goroutine and doesn't return.
+	// So we cannot Unlock in a defer clause.
+	worker.stateMut.Unlock()
+
+	worker.logger.Debug("Worker Started")
+
+	defer func() {
+		worker.logger.Debug("Worker finished")
+		worker.stoppedCh <- struct{}{}
+	}()
+
+	for {
+		select {
+		case <-worker.stopCh:
+			worker.logger.Debug("Worker: Received stop signal")
+			return
+		case job := <-worker.jobs:
+			worker.DoJob(&job)
+		}
+	}
+}
+
+func (worker *MessageExportWorker) Stop() {
+	worker.stateMut.Lock()
+	defer worker.stateMut.Unlock()
+
+	// Set to close, and if already closed before, then return.
+	if worker.stopped {
+		return
+	}
+	worker.stopped = true
+
+	worker.logger.Debug("Worker: Stopping")
+	worker.cancel()
+	close(worker.stopCh)
+	<-worker.stoppedCh
+}
+
+func (worker *MessageExportWorker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+// getExportBackend returns the file backend where the export will be created.
+func (worker *MessageExportWorker) getExportBackend(rctx request.CTX) (filestore.FileBackend, *model.AppError) {
+	config := worker.jobServer.Config()
+	insecure := config.ServiceSettings.EnableInsecureOutgoingConnections
+
+	if config.FileSettings.DedicatedExportStore != nil && *config.FileSettings.DedicatedExportStore {
+		rctx.Logger().Debug("Worker: using dedicated export filestore", mlog.String("driver_name", *config.FileSettings.ExportDriverName))
+		backend, errFileBack := filestore.NewExportFileBackend(filestore.NewExportFileBackendSettingsFromConfig(&config.FileSettings, true, insecure != nil && *insecure))
+		if errFileBack != nil {
+			return nil, model.NewAppError("getFileBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError).Wrap(errFileBack)
+		}
+
+		return backend, nil
+	}
+
+	backend, err := filestore.NewFileBackend(filestore.NewFileBackendSettingsFromConfig(&config.FileSettings, true, insecure != nil && *insecure))
+	if err != nil {
+		return nil, model.NewAppError("getFileBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return backend, nil
+}
+
+// getFileAttachmentBackend returns the file backend where file attachments are
+// located for messages that will be exported. This may be the same backend
+// where the export will be created.
+func (worker *MessageExportWorker) getFileAttachmentBackend(rctx request.CTX) (filestore.FileBackend, *model.AppError) {
+	config := worker.jobServer.Config()
+	insecure := config.ServiceSettings.EnableInsecureOutgoingConnections
+
+	backend, err := filestore.NewFileBackend(filestore.NewFileBackendSettingsFromConfig(&config.FileSettings, true, insecure != nil && *insecure))
+	if err != nil {
+		return nil, model.NewAppError("getFileBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return backend, nil
+}
+
+func (worker *MessageExportWorker) DoJob(job *model.Job) {
+	logger := worker.logger.With(jobs.JobLoggerFields(job)...)
+	logger.Debug("Worker: Received a new candidate job.")
+	defer worker.jobServer.HandleJobPanic(logger, job)
+
+	claimed, appErr := worker.jobServer.ClaimJob(job)
+	if appErr != nil {
+		logger.Info("Worker: Error occurred while trying to claim job", mlog.Err(appErr))
+		return
+	}
+
+	if !claimed {
+		return
+	}
+
+	var cancelContext request.CTX = request.EmptyContext(worker.logger)
+	cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background())
+	cancelWatcherChan := make(chan struct{}, 1)
+	cancelContext = cancelContext.WithContext(cancelCtx)
+	go worker.jobServer.CancellationWatcher(cancelContext, job.Id, cancelWatcherChan)
+	defer cancelCancelWatcher()
+
+	// if job data is missing, we'll do our best to recover
+	worker.initJobData(logger, job)
+
+	// the initJobData call above populates the create_at timestamp of the first post that we should export
+	// incase of job resumption or new job
+	batchStartTime, err := strconv.ParseInt(job.Data[JobDataBatchStartTimestamp], 10, 64)
+	if err != nil {
+		worker.setJobError(logger, job, model.NewAppError("Job.DoJob", model.NoTranslation, nil, "", http.StatusBadRequest).Wrap((err)))
+		return
+	}
+	batchStartId := job.Data[JobDataBatchStartId]
+
+	jobStartTime, err := strconv.ParseInt(job.Data[JobDataStartTimestamp], 10, 64)
+	if err != nil {
+		worker.setJobError(logger, job, model.NewAppError("Job.DoJob", model.NoTranslation, nil, "", http.StatusBadRequest).Wrap((err)))
+		return
+	}
+	jobStartId := job.Data[JobDataStartId]
+
+	batchSize, err := strconv.Atoi(job.Data[JOB_DATA_BatchSize])
+	if err != nil {
+		worker.setJobError(logger, job, model.NewAppError("Job.DoJob", model.NoTranslation, nil, "", http.StatusBadRequest).Wrap((err)))
+		return
+	}
+
+	totalPostsExported, err := strconv.ParseInt(job.Data[JobDataMessagesExported], 10, 64)
+	if err != nil {
+		worker.setJobError(logger, job, model.NewAppError("Job.DoJob", model.NoTranslation, nil, "", http.StatusBadRequest).Wrap((err)))
+		return
+	}
+
+	var directories []string
+	err = json.Unmarshal([]byte(job.Data[JobDirectories]), &directories)
+	if err != nil {
+		worker.setJobError(logger, job, model.NewAppError("Job.DoJob", model.NoTranslation, nil, "", http.StatusBadRequest).Wrap((err)))
+		return
+	}
+
+	// Counting all posts may fail or timeout when the posts table is large. If this happens, log a warning, but carry
+	// on with the job anyway. The only issue is that the progress % reporting will be inaccurate.
+	var totalPosts int64
+	if count, err := worker.jobServer.Store.Post().AnalyticsPostCount(&model.PostCountOptions{ExcludeSystemPosts: true, SincePostID: jobStartId, SinceUpdateAt: jobStartTime}); err != nil {
+		logger.Warn("Worker: Failed to fetch total post count for job. An estimated value will be used for progress reporting.", mlog.Err(err))
+		totalPosts = estimatedPostCount
+	} else {
+		totalPosts = count
+	}
+
+	var totalWarningCount int64
+	cursor := model.MessageExportCursor{LastPostUpdateAt: batchStartTime, LastPostId: batchStartId}
+	for {
+		select {
+		case <-cancelWatcherChan:
+			logger.Debug("Worker: Job has been canceled via CancellationWatcher")
+			worker.setJobCanceled(logger, job)
+			return
+
+		case <-worker.stopCh:
+			logger.Debug("Worker: Job has been canceled via Worker Stop. Setting the job back to pending")
+			worker.SetJobPending(logger, job)
+			return
+
+		case <-time.After(TimeBetweenBatches * time.Millisecond):
+			logger.Debug("Starting batch export", mlog.Int("last_post_update_at", cursor.LastPostUpdateAt))
+			rctx := request.EmptyContext(logger).WithContext(worker.context)
+			prevPostUpdateAt := cursor.LastPostUpdateAt
+
+			var postsExported []*model.MessageExport
+			var nErr error
+			postsExported, cursor, nErr = worker.jobServer.Store.Compliance().MessageExport(rctx, cursor, batchSize)
+			if nErr != nil {
+				// We ignore error if the job was explicitly cancelled
+				// and let it
+				if worker.context.Err() == context.Canceled {
+					logger.Debug("Worker: Job has been canceled via worker's context. Setting the job back to pending")
+					worker.SetJobPending(logger, job)
+				} else {
+					worker.setJobError(logger, job, model.NewAppError("DoJob", "ent.message_export.run_export.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr))
+				}
+				return
+			}
+			logger.Debug("Found posts to export", mlog.Int("number_of_posts", len(postsExported)))
+			totalPostsExported += int64(len(postsExported))
+			job.Data[JobDataMessagesExported] = strconv.FormatInt(totalPostsExported, 10)
+			job.Data[JobDataBatchStartTimestamp] = strconv.FormatInt(cursor.LastPostUpdateAt, 10)
+			job.Data[JobDataBatchStartId] = cursor.LastPostId
+
+			if len(postsExported) == 0 {
+				job.Data[JobDataWarningCount] = strconv.FormatInt(totalWarningCount, 10)
+				// we've exported everything up to the current time
+				logger.Debug("FormatExport complete")
+
+				// Create downloadable zip file of all batches.
+				if job.Data[JobDataExportType] != model.ComplianceExportTypeGlobalrelay {
+					exportBackend, err := worker.getExportBackend(rctx)
+					if err != nil {
+						worker.setJobError(logger, job, err)
+						return
+					}
+
+					zipErr := createZipFile(rctx, exportBackend, job.Id, directories)
+					if zipErr != nil {
+						logger.Error("Error creating zip file for export", mlog.Err(zipErr))
+						job.Data[JobDataIsDownloadable] = "false"
+					} else {
+						job.Data[JobDataIsDownloadable] = "true"
+					}
+				}
+				if totalWarningCount > 0 {
+					worker.setJobWarning(logger, job)
+				} else {
+					worker.setJobSuccess(logger, job)
+				}
+				return
+			}
+
+			exportBackend, err := worker.getExportBackend(rctx)
+			if err != nil {
+				worker.setJobError(logger, job, err)
+				return
+			}
+
+			fileAttachmentBackend, err := worker.getFileAttachmentBackend(rctx)
+			if err != nil {
+				worker.setJobError(logger, job, err)
+				return
+			}
+
+			batchDirectory := getOutputDirectoryPath(prevPostUpdateAt, cursor.LastPostUpdateAt)
+			warningCount, err := runExportByType(
+				rctx,
+				job.Data[JobDataExportType],
+				postsExported,
+				batchDirectory,
+				worker.jobServer.Store,
+				exportBackend,
+				fileAttachmentBackend,
+				worker.htmlTemplateWatcher,
+				worker.jobServer.Config(),
+			)
+			if err != nil {
+				worker.setJobError(logger, job, err)
+				return
+			}
+
+			totalWarningCount += warningCount
+
+			directories = append(directories, batchDirectory)
+			directoriesBytes, e := json.Marshal(directories)
+			if e != nil {
+				worker.setJobError(logger, job, model.NewAppError("Job.DoJob", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap((e)))
+				return
+			}
+			job.Data[JobDirectories] = string(directoriesBytes)
+
+			// also saves the last post create time
+			if err := worker.jobServer.SetJobProgress(job, getJobProgress(totalPostsExported, totalPosts)); err != nil {
+				worker.setJobError(logger, job, err)
+				return
+			}
+		}
+	}
+}
+
+func createZipFile(rctx request.CTX, fileBackend filestore.FileBackend, jobId string, directories []string) error {
+	zipFileName := jobId + ".zip"
+
+	dest, err := os.CreateTemp("", zipFileName)
+	if err != nil {
+		return err
+	}
+	defer os.Remove(dest.Name())
+
+	// Create a new zip archive.
+	w := zip.NewWriter(dest)
+
+	// create a 32 KiB buffer for copying files
+	buf := make([]byte, 32*1024)
+
+	// Add directories to the archive.
+	for _, directory := range directories {
+		err = addFiles(w, fileBackend, directory, buf)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Make sure to check the error on Close.
+	err = w.Close()
+	if err != nil {
+		return fmt.Errorf("error closing zip file: %s %v", dest.Name(), err)
+	}
+
+	_, err = dest.Seek(0, 0)
+	if err != nil {
+		return fmt.Errorf("error seeking zip file: %s %v", dest.Name(), err)
+	}
+
+	zipPath := path.Join(exportPath, zipFileName)
+
+	// If the file backend allows it, we want to upload without a timeout
+	_, err = filestore.TryWriteFileContext(rctx.Context(), fileBackend, dest, zipPath)
+	return err
+}
+
+func addFiles(w *zip.Writer, fileBackend filestore.FileBackend, basePath string, buf []byte) error {
+	// Open the Directory
+	files, err := fileBackend.ListDirectoryRecursively(basePath)
+	if err != nil {
+		return err
+	}
+
+	for _, file := range files {
+		err = addFile(w, fileBackend, file, basePath, buf)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func addFile(w *zip.Writer, fileBackend filestore.FileBackend, file, basePath string, buf []byte) error {
+	// In some storage backends like Hitachi HCP, the first entry
+	// from a ListObjects API is always the dir entry itself.
+	if file == basePath {
+		return nil
+	}
+
+	size, err := fileBackend.FileSize(file)
+	if err != nil {
+		return fmt.Errorf("error reading file size for %s: %w", file, err)
+	}
+	if size == 0 {
+		// skip empty files
+		return nil
+	}
+
+	r, err := fileBackend.Reader(file)
+	if err != nil {
+		return fmt.Errorf("error opening file %s: %w", file, err)
+	}
+	defer r.Close()
+
+	// Add some files to the archive.
+	f, err := w.Create(file)
+	if err != nil {
+		return fmt.Errorf("error creating file %s in the archive: %w", file, err)
+	}
+	_, err = io.CopyBuffer(f, r, buf)
+	if err != nil {
+		return fmt.Errorf("error copying file %s into the archive: %w", file, err)
+	}
+
+	return nil
+}
+
+// initializes job data if it's missing, allows us to recover from failed or improperly configured jobs
+func (worker *MessageExportWorker) initJobData(logger mlog.LoggerIFace, job *model.Job) {
+	if job.Data == nil {
+		job.Data = make(map[string]string)
+	}
+	if _, exists := job.Data[JobDataMessagesExported]; !exists {
+		job.Data[JobDataMessagesExported] = "0"
+	}
+	if _, exists := job.Data[JobDirectories]; !exists {
+		// json null value
+		job.Data[JobDirectories] = "null"
+	}
+	if _, exists := job.Data[JobDataExportType]; !exists {
+		// for now, we'll default to Actiance. When we support multiple export types, we'll have to fetch it from config instead
+		logger.Info("Worker: Defaulting to configured export format")
+		job.Data[JobDataExportType] = *worker.jobServer.Config().MessageExportSettings.ExportFormat
+	}
+	if _, exists := job.Data[JOB_DATA_BatchSize]; !exists {
+		logger.Info("Worker: Defaulting to configured batch size")
+		job.Data[JOB_DATA_BatchSize] = strconv.Itoa(*worker.jobServer.Config().MessageExportSettings.BatchSize)
+	}
+	if _, exists := job.Data[JobDataBatchStartTimestamp]; !exists {
+		previousJob, err := worker.jobServer.Store.Job().GetNewestJobByStatusesAndType([]string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport)
+		if err != nil {
+			logger.Info("Worker: No previously successful job found, falling back to configured MessageExportSettings.ExportFromTimestamp")
+			job.Data[JobDataBatchStartTimestamp] = strconv.FormatInt(*worker.jobServer.Config().MessageExportSettings.ExportFromTimestamp, 10)
+			job.Data[JobDataBatchStartId] = ""
+			job.Data[JobDataStartTimestamp] = job.Data[JobDataBatchStartTimestamp]
+			job.Data[JobDataStartId] = job.Data[JobDataBatchStartId]
+			return
+		}
+
+		logger.Info("Worker: Implicitly resuming export from where previously successful job left off")
+		if previousJob == nil {
+			previousJob = &model.Job{}
+		}
+		if previousJob.Data == nil {
+			previousJob.Data = make(map[string]string)
+		}
+		if _, prevExists := previousJob.Data[JobDataBatchStartTimestamp]; !prevExists {
+			logger.Info("Worker: Previously successful job lacks job data, falling back to configured MessageExportSettings.ExportFromTimestamp")
+			job.Data[JobDataBatchStartTimestamp] = strconv.FormatInt(*worker.jobServer.Config().MessageExportSettings.ExportFromTimestamp, 10)
+		} else {
+			job.Data[JobDataBatchStartTimestamp] = previousJob.Data[JobDataBatchStartTimestamp]
+		}
+		if _, prevExists := previousJob.Data[JobDataBatchStartId]; !prevExists {
+			logger.Info("Worker: Previously successful job lacks post ID, falling back to empty string")
+			job.Data[JobDataBatchStartId] = ""
+		} else {
+			job.Data[JobDataBatchStartId] = previousJob.Data[JobDataBatchStartId]
+		}
+		job.Data[JobDataStartTimestamp] = job.Data[JobDataBatchStartTimestamp]
+		job.Data[JobDataStartId] = job.Data[JobDataBatchStartId]
+	} else {
+		logger.Info("Worker: FormatExport start time explicitly set", mlog.String("new_start_time", job.Data[JobDataBatchStartTimestamp]))
+	}
+}
+
+func getJobProgress(totalExportedPosts, totalPosts int64) int64 {
+	return totalExportedPosts * 100 / totalPosts
+}
+
+func (worker *MessageExportWorker) setJobSuccess(logger mlog.LoggerIFace, job *model.Job) {
+	// setting progress causes the job data to be saved, which is necessary if we want the next job to pick up where this one left off
+	if err := worker.jobServer.SetJobProgress(job, 100); err != nil {
+		logger.Error("Worker: Failed to update progress for job", mlog.Err(err))
+		worker.setJobError(logger, job, err)
+	}
+	if err := worker.jobServer.SetJobSuccess(job); err != nil {
+		logger.Error("Worker: Failed to set success for job", mlog.Err(err))
+		worker.setJobError(logger, job, err)
+	}
+}
+
+func (worker *MessageExportWorker) setJobWarning(logger mlog.LoggerIFace, job *model.Job) {
+	// setting progress causes the job data to be saved, which is necessary if we want the next job to pick up where this one left off
+	if err := worker.jobServer.SetJobProgress(job, 100); err != nil {
+		logger.Error("Worker: Failed to update progress for job", mlog.Err(err))
+		worker.setJobError(logger, job, err)
+	}
+	if err := worker.jobServer.SetJobWarning(job); err != nil {
+		logger.Error("Worker: Failed to set warning for job", mlog.Err(err))
+		worker.setJobError(logger, job, err)
+	}
+}
+
+func (worker *MessageExportWorker) setJobError(logger mlog.LoggerIFace, job *model.Job, appError *model.AppError) {
+	logger.Error("Worker: Job error", mlog.Err(appError))
+	if err := worker.jobServer.SetJobError(job, appError); err != nil {
+		logger.Error("Worker: Failed to set job errorv", mlog.Err(err), mlog.NamedErr("set_error", appError))
+	}
+}
+
+func (worker *MessageExportWorker) setJobCanceled(logger mlog.LoggerIFace, job *model.Job) {
+	if err := worker.jobServer.SetJobCanceled(job); err != nil {
+		logger.Error("Worker: Failed to mark job as canceled", mlog.Err(err))
+	}
+}
+
+func (worker *MessageExportWorker) SetJobPending(logger mlog.LoggerIFace, job *model.Job) {
+	if err := worker.jobServer.SetJobPending(job); err != nil {
+		logger.Error("Worker: Failed to mark job as pending", mlog.Err(err))
+	}
+}
+
+func getOutputDirectoryPath(exportStartTime int64, exportEndTime int64) string {
+	return path.Join(exportPath, strconv.FormatInt(exportStartTime, 10)+"-"+strconv.FormatInt(exportEndTime, 10))
+}

server/enterprise/message_export/worker_test.go

@@ -0,0 +1,499 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package message_export
+
+import (
+	"archive/zip"
+	"bytes"
+	"context"
+	"net/http"
+	"os"
+	"path"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	tmock "github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/plugin/plugintest/mock"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/app"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
+	"github.com/mattermost/mattermost/server/v8/channels/utils/testutils"
+	"github.com/mattermost/mattermost/server/v8/einterfaces/mocks"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+	fmocks "github.com/mattermost/mattermost/server/v8/platform/shared/filestore/mocks"
+)
+
+func TestInitJobDataNoJobData(t *testing.T) {
+	logger := mlog.CreateConsoleTestLogger(t)
+	mockStore := &storetest.Store{}
+	defer mockStore.AssertExpectations(t)
+
+	job := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: model.GetMillis(),
+		Status:   model.JobStatusPending,
+		Type:     model.JobTypeMessageExport,
+	}
+
+	// mock job store doesn't return a previously successful job, forcing fallback to config
+	mockStore.JobStore.On("GetNewestJobByStatusesAndType", []string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport).Return(nil, model.NewAppError("", "", nil, "", http.StatusBadRequest))
+
+	worker := &MessageExportWorker{
+		jobServer: &jobs.JobServer{
+			Store: mockStore,
+			ConfigService: &testutils.StaticConfigService{
+				Cfg: &model.Config{
+					// mock config
+					MessageExportSettings: model.MessageExportSettings{
+						EnableExport:        model.NewPointer(true),
+						ExportFormat:        model.NewPointer(model.ComplianceExportTypeActiance),
+						DailyRunTime:        model.NewPointer("01:00"),
+						ExportFromTimestamp: model.NewPointer(int64(0)),
+						BatchSize:           model.NewPointer(10000),
+					},
+				},
+			},
+		},
+		logger: logger,
+	}
+
+	// actually execute the code under test
+	worker.initJobData(logger, job)
+
+	assert.Equal(t, model.ComplianceExportTypeActiance, job.Data[JobDataExportType])
+	assert.Equal(t, strconv.Itoa(*worker.jobServer.Config().MessageExportSettings.BatchSize), job.Data[JOB_DATA_BatchSize])
+	assert.Equal(t, strconv.FormatInt(*worker.jobServer.Config().MessageExportSettings.ExportFromTimestamp, 10), job.Data[JobDataBatchStartTimestamp])
+}
+
+func TestInitJobDataPreviousJobNoJobData(t *testing.T) {
+	logger := mlog.CreateConsoleTestLogger(t)
+	mockStore := &storetest.Store{}
+	defer mockStore.AssertExpectations(t)
+
+	previousJob := &model.Job{
+		Id:             model.NewId(),
+		CreateAt:       model.GetMillis(),
+		Status:         model.JobStatusSuccess,
+		Type:           model.JobTypeMessageExport,
+		StartAt:        model.GetMillis() - 1000,
+		LastActivityAt: model.GetMillis() - 1000,
+	}
+
+	job := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: model.GetMillis(),
+		Status:   model.JobStatusPending,
+		Type:     model.JobTypeMessageExport,
+	}
+
+	// mock job store returns a previously successful job, but it doesn't have job data either, so we still fall back to config
+	mockStore.JobStore.On("GetNewestJobByStatusesAndType", []string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport).Return(previousJob, nil)
+
+	worker := &MessageExportWorker{
+		jobServer: &jobs.JobServer{
+			Store: mockStore,
+			ConfigService: &testutils.StaticConfigService{
+				Cfg: &model.Config{
+					// mock config
+					MessageExportSettings: model.MessageExportSettings{
+						EnableExport:        model.NewPointer(true),
+						ExportFormat:        model.NewPointer(model.ComplianceExportTypeActiance),
+						DailyRunTime:        model.NewPointer("01:00"),
+						ExportFromTimestamp: model.NewPointer(int64(0)),
+						BatchSize:           model.NewPointer(10000),
+					},
+				},
+			},
+		},
+		logger: logger,
+	}
+
+	// actually execute the code under test
+	worker.initJobData(logger, job)
+
+	assert.Equal(t, model.ComplianceExportTypeActiance, job.Data[JobDataExportType])
+	assert.Equal(t, strconv.Itoa(*worker.jobServer.Config().MessageExportSettings.BatchSize), job.Data[JOB_DATA_BatchSize])
+	assert.Equal(t, strconv.FormatInt(*worker.jobServer.Config().MessageExportSettings.ExportFromTimestamp, 10), job.Data[JobDataBatchStartTimestamp])
+}
+
+func TestInitJobDataPreviousJobWithJobData(t *testing.T) {
+	logger := mlog.CreateConsoleTestLogger(t)
+	mockStore := &storetest.Store{}
+	defer mockStore.AssertExpectations(t)
+
+	previousJob := &model.Job{
+		Id:             model.NewId(),
+		CreateAt:       model.GetMillis(),
+		Status:         model.JobStatusSuccess,
+		Type:           model.JobTypeMessageExport,
+		StartAt:        model.GetMillis() - 1000,
+		LastActivityAt: model.GetMillis() - 1000,
+		Data:           map[string]string{JobDataBatchStartTimestamp: "123"},
+	}
+
+	job := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: model.GetMillis(),
+		Status:   model.JobStatusPending,
+		Type:     model.JobTypeMessageExport,
+	}
+
+	// mock job store returns a previously successful job that has the config that we're looking for, so we use it
+	mockStore.JobStore.On("GetNewestJobByStatusesAndType", []string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport).Return(previousJob, nil)
+
+	worker := &MessageExportWorker{
+		jobServer: &jobs.JobServer{
+			Store: mockStore,
+			ConfigService: &testutils.StaticConfigService{
+				Cfg: &model.Config{
+					// mock config
+					MessageExportSettings: model.MessageExportSettings{
+						EnableExport:        model.NewPointer(true),
+						ExportFormat:        model.NewPointer(model.ComplianceExportTypeActiance),
+						DailyRunTime:        model.NewPointer("01:00"),
+						ExportFromTimestamp: model.NewPointer(int64(0)),
+						BatchSize:           model.NewPointer(10000),
+					},
+				},
+			},
+		},
+		logger: logger,
+	}
+
+	// actually execute the code under test
+	worker.initJobData(logger, job)
+
+	assert.Equal(t, model.ComplianceExportTypeActiance, job.Data[JobDataExportType])
+	assert.Equal(t, strconv.Itoa(*worker.jobServer.Config().MessageExportSettings.BatchSize), job.Data[JOB_DATA_BatchSize])
+	assert.Equal(t, previousJob.Data[JobDataBatchStartTimestamp], job.Data[JobDataBatchStartTimestamp])
+}
+
+func TestDoJobNoPostsToExport(t *testing.T) {
+	logger := mlog.CreateConsoleTestLogger(t)
+
+	mockStore := &storetest.Store{}
+	defer mockStore.AssertExpectations(t)
+
+	mockMetrics := &mocks.MetricsInterface{}
+	defer mockMetrics.AssertExpectations(t)
+
+	job := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: model.GetMillis(),
+		Status:   model.JobStatusPending,
+		Type:     model.JobTypeMessageExport,
+	}
+
+	// claim job succeeds
+	mockStore.JobStore.On("UpdateStatusOptimistically", job.Id, model.JobStatusPending, model.JobStatusInProgress).Return(true, nil)
+	mockMetrics.On("IncrementJobActive", model.JobTypeMessageExport)
+
+	// no previous job, data will be loaded from config
+	mockStore.JobStore.On("GetNewestJobByStatusesAndType", []string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport).Return(nil, model.NewAppError("", "", nil, "", http.StatusBadRequest))
+
+	// no posts found to export
+	mockStore.ComplianceStore.On("MessageExport", mock.Anything, mock.AnythingOfType("model.MessageExportCursor"), 10000).Return(
+		make([]*model.MessageExport, 0), model.MessageExportCursor{}, nil,
+	)
+
+	mockStore.PostStore.On("AnalyticsPostCount", mock.Anything).Return(
+		int64(estimatedPostCount), nil,
+	)
+
+	// job completed successfully
+	mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(true, nil)
+	mockStore.JobStore.On("UpdateStatus", job.Id, model.JobStatusSuccess).Return(job, nil)
+	mockMetrics.On("DecrementJobActive", model.JobTypeMessageExport)
+
+	tempDir, err := os.MkdirTemp("", "")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		err = os.RemoveAll(tempDir)
+		assert.NoError(t, err)
+	})
+
+	worker := &MessageExportWorker{
+		jobServer: jobs.NewJobServer(
+			&testutils.StaticConfigService{
+				Cfg: &model.Config{
+					// mock config
+					FileSettings: model.FileSettings{
+						DriverName: model.NewPointer(model.ImageDriverLocal),
+						Directory:  model.NewPointer(tempDir),
+					},
+					MessageExportSettings: model.MessageExportSettings{
+						EnableExport:        model.NewPointer(true),
+						ExportFormat:        model.NewPointer(model.ComplianceExportTypeActiance),
+						DailyRunTime:        model.NewPointer("01:00"),
+						ExportFromTimestamp: model.NewPointer(int64(0)),
+						BatchSize:           model.NewPointer(10000),
+					},
+				},
+			},
+			mockStore,
+			mockMetrics,
+			logger,
+		),
+		logger: logger,
+	}
+
+	// actually execute the code under test
+	worker.DoJob(job)
+}
+
+func TestDoJobWithDedicatedExportBackend(t *testing.T) {
+	logger := mlog.CreateConsoleTestLogger(t)
+
+	mockStore := &storetest.Store{}
+	defer mockStore.AssertExpectations(t)
+
+	mockMetrics := &mocks.MetricsInterface{}
+	defer mockMetrics.AssertExpectations(t)
+
+	job := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: model.GetMillis(),
+		Status:   model.JobStatusPending,
+		Type:     model.JobTypeMessageExport,
+	}
+
+	// claim job succeeds
+	mockStore.JobStore.On("UpdateStatusOptimistically", job.Id, model.JobStatusPending, model.JobStatusInProgress).Return(true, nil)
+	mockMetrics.On("IncrementJobActive", model.JobTypeMessageExport)
+
+	// no previous job, data will be loaded from config
+	mockStore.JobStore.On("GetNewestJobByStatusesAndType", []string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport).Return(nil, model.NewAppError("", "", nil, "", http.StatusBadRequest))
+
+	// no posts found to export
+	mockStore.ComplianceStore.On("MessageExport", mock.Anything, mock.AnythingOfType("model.MessageExportCursor"), 10000).Return(
+		make([]*model.MessageExport, 0), model.MessageExportCursor{}, nil,
+	)
+
+	mockStore.PostStore.On("AnalyticsPostCount", mock.Anything).Return(
+		int64(estimatedPostCount), nil,
+	)
+
+	// job completed successfully
+	mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(true, nil)
+	mockStore.JobStore.On("UpdateStatus", job.Id, model.JobStatusSuccess).Return(job, nil)
+	mockMetrics.On("DecrementJobActive", model.JobTypeMessageExport)
+
+	// create primary filestore directory
+	tempPrimaryDir, err := os.MkdirTemp("", "")
+	require.NoError(t, err)
+	defer os.RemoveAll(tempPrimaryDir)
+
+	// create dedicated filestore directory
+	tempDedicatedDir, err := os.MkdirTemp("", "")
+	require.NoError(t, err)
+	defer os.RemoveAll(tempDedicatedDir)
+
+	// setup worker with primary and dedicated filestores.
+	worker := &MessageExportWorker{
+		jobServer: jobs.NewJobServer(
+			&testutils.StaticConfigService{
+				Cfg: &model.Config{
+					// mock config
+					FileSettings: model.FileSettings{
+						DriverName:           model.NewPointer(model.ImageDriverLocal),
+						Directory:            model.NewPointer(tempPrimaryDir),
+						DedicatedExportStore: model.NewPointer(true),
+						ExportDriverName:     model.NewPointer(model.ImageDriverLocal),
+						ExportDirectory:      model.NewPointer(tempDedicatedDir),
+					},
+					MessageExportSettings: model.MessageExportSettings{
+						EnableExport:        model.NewPointer(true),
+						ExportFormat:        model.NewPointer(model.ComplianceExportTypeActiance),
+						DailyRunTime:        model.NewPointer("01:00"),
+						ExportFromTimestamp: model.NewPointer(int64(0)),
+						BatchSize:           model.NewPointer(10000),
+					},
+				},
+			},
+			mockStore,
+			mockMetrics,
+			logger,
+		),
+		logger: logger,
+	}
+
+	// actually execute the code under test
+	worker.DoJob(job)
+
+	// ensure no primary filestore files exist
+	files, err := os.ReadDir(tempPrimaryDir)
+	require.NoError(t, err)
+	assert.Zero(t, len(files))
+
+	// ensure some dedicated filestore files exist
+	files, err = os.ReadDir(tempDedicatedDir)
+	require.NoError(t, err)
+	assert.NotZero(t, len(files))
+}
+
+func TestDoJobCancel(t *testing.T) {
+	logger := mlog.CreateConsoleTestLogger(t)
+
+	mockStore := &storetest.Store{}
+	t.Cleanup(func() { mockStore.AssertExpectations(t) })
+	mockMetrics := &mocks.MetricsInterface{}
+	t.Cleanup(func() { mockMetrics.AssertExpectations(t) })
+
+	job := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: model.GetMillis(),
+		Status:   model.JobStatusPending,
+		Type:     model.JobTypeMessageExport,
+	}
+
+	tempDir, err := os.MkdirTemp("", "")
+	require.NoError(t, err)
+	t.Cleanup(func() { os.RemoveAll(tempDir) })
+
+	impl := MessageExportJobInterfaceImpl{
+		Server: &app.Server{
+			Jobs: jobs.NewJobServer(
+				&testutils.StaticConfigService{
+					Cfg: &model.Config{
+						// mock config
+						FileSettings: model.FileSettings{
+							DriverName: model.NewPointer(model.ImageDriverLocal),
+							Directory:  model.NewPointer(tempDir),
+						},
+						MessageExportSettings: model.MessageExportSettings{
+							EnableExport:        model.NewPointer(true),
+							ExportFormat:        model.NewPointer(model.ComplianceExportTypeActiance),
+							DailyRunTime:        model.NewPointer("01:00"),
+							ExportFromTimestamp: model.NewPointer(int64(0)),
+							BatchSize:           model.NewPointer(10000),
+						},
+					},
+				},
+				mockStore,
+				mockMetrics,
+				logger,
+			),
+		},
+	}
+	worker, ok := impl.MakeWorker().(*MessageExportWorker)
+	require.True(t, ok)
+
+	// Claim job succeeds
+	mockStore.JobStore.On("UpdateStatusOptimistically", job.Id, model.JobStatusPending, model.JobStatusInProgress).Return(true, nil)
+	mockMetrics.On("IncrementJobActive", model.JobTypeMessageExport)
+
+	// No previous job, data will be loaded from config
+	mockStore.JobStore.On("GetNewestJobByStatusesAndType", []string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport).Return(nil, model.NewAppError("", "", nil, "", http.StatusBadRequest))
+
+	cancelled := make(chan struct{})
+	// Cancel the worker and return an error
+	mockStore.ComplianceStore.On("MessageExport", mock.Anything, mock.AnythingOfType("model.MessageExportCursor"), 10000).Run(func(args tmock.Arguments) {
+		worker.cancel()
+
+		rctx, ok := args.Get(0).(request.CTX)
+		require.True(t, ok)
+		assert.Error(t, rctx.Context().Err())
+		assert.ErrorIs(t, rctx.Context().Err(), context.Canceled)
+
+		cancelled <- struct{}{}
+	}).Return(
+		nil, model.MessageExportCursor{}, context.Canceled,
+	)
+
+	mockStore.PostStore.On("AnalyticsPostCount", mock.Anything).Return(
+		int64(estimatedPostCount), nil,
+	)
+
+	// Job marked as pending
+	mockStore.JobStore.On("UpdateStatus", job.Id, model.JobStatusPending).Return(job, nil)
+	mockMetrics.On("DecrementJobActive", model.JobTypeMessageExport)
+
+	go worker.Run()
+
+	worker.JobChannel() <- *job
+
+	// Wait for the cancelation
+	<-cancelled
+
+	// Cleanup
+	worker.Stop()
+}
+
+func TestCreateZipFile(t *testing.T) {
+	rctx := request.TestContext(t)
+
+	tempDir, ioErr := os.MkdirTemp("", "")
+	require.NoError(t, ioErr)
+	defer os.RemoveAll(tempDir)
+
+	config := filestore.FileBackendSettings{
+		DriverName: model.ImageDriverLocal,
+		Directory:  tempDir,
+	}
+
+	fileBackend, err := filestore.NewFileBackend(config)
+	assert.NoError(t, err)
+	_ = fileBackend
+
+	b := []byte("test")
+	path1 := path.Join(exportPath, "19700101")
+	path2 := path.Join(exportPath, "19800101/subdir")
+
+	// We test with a mock to test the Hitachi HCP case
+	// where ListDirectory returns the dir itself as the first entry.
+	// Note: If the mocks fail, that means the logic in createZipFile has
+	// gone wrong and needs to be verified.
+	mock := &fmocks.FileBackend{}
+	defer mock.AssertExpectations(t)
+
+	mock.On("WriteFile", tmock.Anything, tmock.AnythingOfType("string")).Return(int64(4), nil)
+	mock.On("FileSize", tmock.Anything).Return(int64(4), nil)
+	mock.On("FileSize", tmock.Anything).Return(int64(4), nil)
+	mock.On("Reader", path.Join(path1, "testid")).Return(mockReadSeekCloser{bytes.NewReader([]byte("test"))}, nil)
+	mock.On("Reader", path.Join(path2, "testid")).Return(mockReadSeekCloser{bytes.NewReader([]byte("test"))}, nil)
+	mock.On("ListDirectoryRecursively", path1).Return([]string{path1, path.Join(path1, "testid")}, nil)
+	mock.On("ListDirectoryRecursively", path2).Return([]string{path2, path.Join(path2, "testid")}, nil)
+
+	for i, backend := range []filestore.FileBackend{fileBackend, mock} {
+		written, err := backend.WriteFile(bytes.NewReader(b), path1+"/"+model.NewId())
+		assert.NoError(t, err)
+		assert.Equal(t, int64(len(b)), written)
+
+		written, err = backend.WriteFile(bytes.NewReader(b), path2+"/"+model.NewId())
+		assert.NoError(t, err)
+		assert.Equal(t, int64(len(b)), written)
+
+		written, err = backend.WriteFile(bytes.NewReader(b), path2+"/"+model.NewId())
+		assert.NoError(t, err)
+		assert.Equal(t, int64(len(b)), written)
+
+		err = createZipFile(rctx, backend, "testjob", []string{path1, path2})
+		assert.NoError(t, err)
+
+		// Skip checking the zip file in mock case.
+		if i == 1 {
+			continue
+		}
+		r, err := zip.OpenReader(path.Join(tempDir, exportPath) + "/testjob.zip")
+		assert.NoError(t, err)
+		err = r.Close()
+		require.NoError(t, err)
+
+		assert.Equal(t, 3, len(r.File))
+	}
+}
+
+type mockReadSeekCloser struct {
+	*bytes.Reader
+}
+
+func (r mockReadSeekCloser) Close() error {
+	return nil
+}