MM-60987: Move message_export code to monorepo (#28674)
* move message_export code to monorepo * update license headers * moving over `dedicated filestore` (#1772) from enterprise master
Этот коммит содержится в:
Christopher Poile
коммит произвёл
GitHub
GitHub
родитель
1d04a2292d
Коммит
b8e462e198
234
server/enterprise/message_export/message_export.go
Обычный файл
234
server/enterprise/message_export/message_export.go
Обычный файл
@@ -0,0 +1,234 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.enterprise for license information.
|
||||
|
||||
package message_export
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"strconv"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
"github.com/mattermost/mattermost/server/public/shared/mlog"
|
||||
"github.com/mattermost/mattermost/server/public/shared/request"
|
||||
"github.com/mattermost/mattermost/server/v8/channels/app"
|
||||
"github.com/mattermost/mattermost/server/v8/channels/store"
|
||||
"github.com/mattermost/mattermost/server/v8/channels/utils/fileutils"
|
||||
"github.com/mattermost/mattermost/server/v8/einterfaces"
|
||||
ejobs "github.com/mattermost/mattermost/server/v8/einterfaces/jobs"
|
||||
"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
|
||||
"github.com/mattermost/mattermost/server/v8/platform/shared/templates"
|
||||
|
||||
"github.com/mattermost/mattermost/server/v8/enterprise/message_export/actiance_export"
|
||||
"github.com/mattermost/mattermost/server/v8/enterprise/message_export/csv_export"
|
||||
"github.com/mattermost/mattermost/server/v8/enterprise/message_export/global_relay_export"
|
||||
)
|
||||
|
||||
const (
|
||||
GlobalRelayExportFilename = "global-relay.zip"
|
||||
)
|
||||
|
||||
type MessageExportInterfaceImpl struct {
|
||||
Server *app.Server
|
||||
}
|
||||
|
||||
type MessageExportJobInterfaceImpl struct {
|
||||
Server *app.Server
|
||||
}
|
||||
|
||||
func init() {
|
||||
app.RegisterJobsMessageExportJobInterface(func(s *app.Server) ejobs.MessageExportJobInterface {
|
||||
return &MessageExportJobInterfaceImpl{s}
|
||||
})
|
||||
app.RegisterMessageExportInterface(func(app *app.App) einterfaces.MessageExportInterface {
|
||||
return &MessageExportInterfaceImpl{app.Srv()}
|
||||
})
|
||||
}
|
||||
|
||||
func (m *MessageExportInterfaceImpl) StartSynchronizeJob(rctx request.CTX, exportFromTimestamp int64) (*model.Job, *model.AppError) {
|
||||
// if a valid export time was specified, put it in the job data
|
||||
jobData := make(map[string]string)
|
||||
if exportFromTimestamp >= 0 {
|
||||
jobData[JobDataBatchStartTimestamp] = strconv.FormatInt(exportFromTimestamp, 10)
|
||||
}
|
||||
|
||||
// passing nil for job data will cause the worker to inherit start time from previously successful job
|
||||
job, err := m.Server.Jobs.CreateJob(rctx, model.JobTypeMessageExport, jobData)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for job.Status == model.JobStatusPending ||
|
||||
job.Status == model.JobStatusInProgress ||
|
||||
job.Status == model.JobStatusCancelRequested {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
job, err = m.Server.Jobs.GetJob(rctx, job.Id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case <-rctx.Context().Done():
|
||||
return nil, model.NewAppError("StartSynchronizeJob", "ent.jobs.start_synchronize_job.timeout", nil, "", 0).Wrap(rctx.Context().Err())
|
||||
}
|
||||
}
|
||||
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (m *MessageExportInterfaceImpl) RunExport(rctx request.CTX, exportType string, since int64, limit int) (warningCount int64, appErr *model.AppError) {
|
||||
if limit < 0 {
|
||||
limit = math.MaxInt64
|
||||
}
|
||||
postsToExport, _, err := m.Server.Store().Compliance().MessageExport(rctx, model.MessageExportCursor{LastPostUpdateAt: since}, limit)
|
||||
if err != nil {
|
||||
return warningCount, model.NewAppError("RunExport", "ent.message_export.run_export.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
|
||||
}
|
||||
rctx.Logger().Debug("Found posts to export", mlog.Int("number_of_posts", len(postsToExport)))
|
||||
|
||||
fileBackend := m.Server.FileBackend()
|
||||
templatesDir, ok := fileutils.FindDir("templates")
|
||||
if !ok {
|
||||
return warningCount, model.NewAppError("RunExport", "ent.compliance.run_export.template_watcher.appError", nil, "", http.StatusAccepted)
|
||||
}
|
||||
|
||||
t, err2 := templates.New(templatesDir)
|
||||
if err2 != nil {
|
||||
return warningCount, model.NewAppError("RunExport", "ent.compliance.run_export.template_watcher.appError", nil, "", http.StatusAccepted).Wrap(err2)
|
||||
}
|
||||
|
||||
exportDirectory := getOutputDirectoryPath(since, model.GetMillis())
|
||||
return runExportByType(rctx, exportType, postsToExport, exportDirectory, m.Server.Store(), fileBackend, fileBackend, t, m.Server.Config())
|
||||
}
|
||||
|
||||
func runExportByType(rctx request.CTX, exportType string, postsToExport []*model.MessageExport, exportDirectory string, db store.Store, exportBackend filestore.FileBackend, fileAttachmentBackend filestore.FileBackend, htmlTemplates *templates.Container, config *model.Config) (warningCount int64, appErr *model.AppError) {
|
||||
// go through all the posts and if the post's props contain 'from_bot' - override the IsBot field, since it's possible that the sender is not a user, but was a Bot and vise-versa
|
||||
for _, post := range postsToExport {
|
||||
if post.PostProps != nil {
|
||||
props := map[string]any{}
|
||||
|
||||
if json.Unmarshal([]byte(*post.PostProps), &props) == nil {
|
||||
if val, ok := props["from_bot"]; ok {
|
||||
post.IsBot = val == "true"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Team info can be null for DM/GM channels.
|
||||
if post.TeamId == nil {
|
||||
post.TeamId = new(string)
|
||||
}
|
||||
if post.TeamName == nil {
|
||||
post.TeamName = new(string)
|
||||
}
|
||||
if post.TeamDisplayName == nil {
|
||||
post.TeamDisplayName = new(string)
|
||||
}
|
||||
|
||||
// make sure user information is present. Set defaults and log an error otherwise.
|
||||
if post.ChannelId == nil {
|
||||
rctx.Logger().Warn("ChannelId is missing for post", mlog.String("post_id", *post.PostId))
|
||||
post.ChannelId = new(string)
|
||||
}
|
||||
if post.ChannelName == nil {
|
||||
rctx.Logger().Warn("ChannelName is missing for post", mlog.String("post_id", *post.PostId))
|
||||
post.ChannelName = new(string)
|
||||
}
|
||||
if post.ChannelDisplayName == nil {
|
||||
rctx.Logger().Warn("ChannelDisplayName is missing for post", mlog.String("post_id", *post.PostId))
|
||||
post.ChannelDisplayName = new(string)
|
||||
}
|
||||
if post.ChannelType == nil {
|
||||
rctx.Logger().Warn("ChannelType is missing for post", mlog.String("post_id", *post.PostId))
|
||||
post.ChannelType = new(model.ChannelType)
|
||||
}
|
||||
|
||||
if post.UserId == nil {
|
||||
rctx.Logger().Warn("UserId is missing for post", mlog.String("post_id", *post.PostId))
|
||||
post.UserId = new(string)
|
||||
}
|
||||
if post.UserEmail == nil {
|
||||
rctx.Logger().Warn("UserEmail is missing for post", mlog.String("post_id", *post.PostId))
|
||||
post.UserEmail = new(string)
|
||||
}
|
||||
if post.Username == nil {
|
||||
rctx.Logger().Warn("Username is missing for post", mlog.String("post_id", *post.PostId))
|
||||
post.Username = new(string)
|
||||
}
|
||||
|
||||
if post.PostType == nil {
|
||||
rctx.Logger().Warn("Type is missing for post", mlog.String("post_id", *post.PostId))
|
||||
post.PostType = new(string)
|
||||
}
|
||||
if post.PostMessage == nil {
|
||||
rctx.Logger().Warn("Message is missing for post", mlog.String("post_id", *post.PostId))
|
||||
post.PostMessage = new(string)
|
||||
}
|
||||
if post.PostCreateAt == nil {
|
||||
rctx.Logger().Warn("CreateAt is missing for post", mlog.String("post_id", *post.PostId))
|
||||
post.PostCreateAt = new(int64)
|
||||
}
|
||||
}
|
||||
|
||||
switch exportType {
|
||||
case model.ComplianceExportTypeCsv:
|
||||
rctx.Logger().Debug("Exporting CSV")
|
||||
return csv_export.CsvExport(rctx, postsToExport, db, exportBackend, fileAttachmentBackend, exportDirectory)
|
||||
|
||||
case model.ComplianceExportTypeActiance:
|
||||
rctx.Logger().Debug("Exporting Actiance")
|
||||
return actiance_export.ActianceExport(rctx, postsToExport, db, exportBackend, fileAttachmentBackend, exportDirectory)
|
||||
|
||||
case model.ComplianceExportTypeGlobalrelay, model.ComplianceExportTypeGlobalrelayZip:
|
||||
rctx.Logger().Debug("Exporting GlobalRelay")
|
||||
f, err := os.CreateTemp("", "")
|
||||
if err != nil {
|
||||
return warningCount, model.NewAppError("RunExport", "ent.compliance.global_relay.open_temporary_file.appError", nil, "", http.StatusAccepted).Wrap(err)
|
||||
}
|
||||
defer f.Close()
|
||||
defer os.Remove(f.Name())
|
||||
|
||||
attachmentsRemovedPostIDs, warnings, appErr := global_relay_export.GlobalRelayExport(rctx, postsToExport, db, fileAttachmentBackend, f, htmlTemplates)
|
||||
if appErr != nil {
|
||||
return warningCount, appErr
|
||||
}
|
||||
warningCount = warnings
|
||||
_, err = f.Seek(0, 0)
|
||||
if err != nil {
|
||||
return warningCount, model.NewAppError("RunExport", "ent.compliance.global_relay.rewind_temporary_file.appError", nil, "", http.StatusAccepted).Wrap(err)
|
||||
}
|
||||
|
||||
if exportType == model.ComplianceExportTypeGlobalrelayZip {
|
||||
// Try to disable the write timeout for the potentially big export file.
|
||||
_, nErr := filestore.TryWriteFileContext(rctx.Context(), exportBackend, f, path.Join(exportDirectory, GlobalRelayExportFilename))
|
||||
if nErr != nil {
|
||||
return warningCount, model.NewAppError("runExportByType", "ent.compliance.global_relay.write_file.appError", nil, "", http.StatusInternalServerError).Wrap(nErr)
|
||||
}
|
||||
} else {
|
||||
appErr = global_relay_export.Deliver(f, config)
|
||||
if appErr != nil {
|
||||
return warningCount, appErr
|
||||
}
|
||||
}
|
||||
|
||||
if len(attachmentsRemovedPostIDs) > 0 {
|
||||
rctx.Logger().Debug("Global Relay Attachments Removed because they were too large to send to Global Relay", mlog.Array("attachment_ids", attachmentsRemovedPostIDs))
|
||||
description := fmt.Sprintf("Attachments to post IDs %v were removed because they were too large to send to Global Relay.", attachmentsRemovedPostIDs)
|
||||
appErr = model.NewAppError("RunExport", "ent.compliance.global_relay.attachments_removed.appError", map[string]any{"Description": description}, description, http.StatusAccepted)
|
||||
return warningCount, appErr
|
||||
}
|
||||
default:
|
||||
err := errors.New("Unknown output format " + exportType)
|
||||
return warningCount, model.NewAppError("RunExport", "ent.compliance.bad_export_type.appError", map[string]any{"ExportType": exportType}, "", http.StatusBadRequest).Wrap(err)
|
||||
}
|
||||
return warningCount, nil
|
||||
}
|
||||
Ссылка в новой задаче
Block a user